strepsiades/ecm_sync_system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

调整了pipeline执行顺序。现在按domain顺序读取数据源数据，防止前面domain的更新影响后面domain.

Browse Source 
将target_project_ids替换为更通用的data_id_filter
This commit is contained in:
strepsiades
2026-03-20 14:38:56 +08:00
parent f9f175ee79
commit 2c09c61165
29 changed files with 296 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sync_state_machine/common/collection.py
+3 -3
View File
@@ -514,9 +514,9 @@ class DataCollection:
await self.persistence.save_nodes_bulk(self.collection_id, list(self._nodes.values()))
async def filter_by_project_ids(self, target_project_ids: List[str]) -> int:
"""按目标项目集合过滤当前 collection，返回删除数量。"""
target_set = set(target_project_ids or [])
async def filter_by_project_ids(self, data_id_filter: List[str]) -> int:
"""按目标项目 data_id 集合过滤当前 collection，返回删除数量。"""
target_set = set(data_id_filter or [])
if not target_set:
return 0
sync_state_machine/config/datasource_config.py
-2
View File
@@ -11,7 +11,6 @@ class BaseDataSourceConfig(BaseModel):
model_config = ConfigDict(extra="forbid", validate_assignment=True)
type: str
target_project_ids: List[str] = Field(default_factory=list)
handler_configs: Dict[str, Dict[str, Any]] = Field(default_factory=dict)
@@ -33,7 +32,6 @@ class ApiDataSourceConfig(BaseDataSourceConfig):
poll_interval: float = 0.5
api_rate_limit_max_requests: int = 10
api_rate_limit_window_seconds: float = 10.0
target_project_ids: List[str] = Field(default_factory=list)
DataSourceConfig = Annotated[
sync_state_machine/datasource/api/handler.py
+7 -3
View File
@@ -69,9 +69,13 @@ class BaseApiHandler(NodeHandler[T]):
def set_handler_config(self, config: Dict[str, Any]) -> None:
self.handler_config = dict(config)
def set_target_project_ids(self, target_project_ids: List[str]) -> None:
"""默认忽略项目过滤列表，具体子类按需覆盖。"""
return
def get_data_id_filter(self) -> List[str]:
value = self.handler_config.get("data_id_filter", [])
if isinstance(value, str):
return [value] if value else []
if isinstance(value, list):
return [str(item) for item in value if str(item)]
return []
@property
def node_type(self) -> str:
sync_state_machine/datasource/datasource.py
-19
View File
@@ -74,7 +74,6 @@ class BaseDataSource(ABC):
def __init__(self, poll_max_retries: int = 1, poll_interval: float = 0.5):
self._handlers: Dict[str, "NodeHandler"] = {}
self._collection: Optional["DataCollection"] = None
self._target_project_ids: List[str] = []
# 按节点类型统计信息
# {"company": {"loaded": 10, "synced": 10, "success": 10, "failed": 0}, ...}
@@ -87,13 +86,6 @@ class BaseDataSource(ABC):
self._poll_interval = max(0, poll_interval)
self._sm_runtime: Optional[StateMachineRuntime] = None
def set_target_project_ids(self, target_project_ids: List[str]) -> None:
self._target_project_ids = [str(item) for item in target_project_ids if str(item)]
@property
def target_project_ids(self) -> List[str]:
return list(self._target_project_ids)
def _ensure_sm_runtime(self) -> StateMachineRuntime:
if self._sm_runtime is None:
cfg_path = Path(__file__).resolve().parents[1] / "config" / "node_state_machine.yaml"
@@ -379,17 +371,6 @@ class BaseDataSource(ABC):
datasource.register_handler(ContractNodeHandler(api_client))
"""
self._handlers[handler.node_type] = handler
self._apply_target_project_ids_to_handler(handler)
def set_target_project_ids(self, target_project_ids: Optional[List[str]]) -> None:
"""设置项目过滤列表，并在注册阶段下发给所有 handler。"""
normalized = [str(pid) for pid in (target_project_ids or []) if str(pid)]
self._target_project_ids = normalized
for handler in self._handlers.values():
self._apply_target_project_ids_to_handler(handler)
def _apply_target_project_ids_to_handler(self, handler: "NodeHandler") -> None:
handler.set_target_project_ids(self._target_project_ids)
def get_handler(self, node_type: str) -> "NodeHandler":
"""获取 Handler"""
sync_state_machine/datasource/handler.py
+61 -8
View File
@@ -74,10 +74,6 @@ class NodeHandler(Protocol[T]):
"""注入 handler 级别配置。默认可忽略。"""
...
def set_target_project_ids(self, target_project_ids: List[str]) -> None:
"""注入项目过滤列表。默认可忽略。"""
...
def set_api_client(self, client: "ApiClient") -> None:
"""注入 API 客户端。默认可忽略。"""
...
@@ -360,14 +356,18 @@ class BaseNodeHandler(ABC, Generic[T]):
"""
self._collection = collection
def set_target_project_ids(self, target_project_ids: List[str]) -> None:
"""默认忽略项目过滤能力。"""
return
def set_handler_config(self, config: Dict[str, Any]) -> None:
"""默认保存 handler 级别配置，供子类按需读取。"""
self.handler_config = dict(config)
def get_data_id_filter(self) -> List[str]:
value = self.handler_config.get("data_id_filter", [])
if isinstance(value, str):
return [value] if value else []
if isinstance(value, list):
return [str(item) for item in value if str(item)]
return []
def set_api_client(self, client: "ApiClient") -> None:
"""默认忽略 API 客户端注入。"""
return
@@ -479,6 +479,59 @@ class BaseNodeHandler(ABC, Generic[T]):
return self.create_node(data)
class CompatibleNodeHandler(BaseNodeHandler[T]):
def __init__(self, handler: BaseNodeHandler[T]):
super().__init__(handler.node_type, handler.node_class, handler.schema)
self._handler = handler
def set_collection(self, collection: "DataCollection") -> None:
super().set_collection(collection)
self._handler.set_collection(collection)
def set_handler_config(self, config: Dict[str, Any]) -> None:
self._handler.set_handler_config(config)
self.handler_config = dict(self._handler.handler_config)
def set_api_client(self, client: "ApiClient") -> None:
self._handler.set_api_client(client)
def set_poll_mode(self, mode: str) -> None:
self._handler.set_poll_mode(mode)
def get_update_fields(self, *, exclude: frozenset[str] = frozenset({"id"})) -> List[str]:
return self._handler.get_update_fields(exclude=exclude)
async def load(self) -> List['SyncNode']:
return await self._handler.load()
async def create_all(
self,
nodes: List["SyncNode"]
) -> List[TaskResult]:
return await self._handler.create_all(nodes)
async def update_all(
self,
nodes: List["SyncNode"]
) -> List[TaskResult]:
return await self._handler.update_all(nodes)
async def delete_all(
self,
nodes: List["SyncNode"]
) -> List[TaskResult]:
return await self._handler.delete_all(nodes)
async def poll_tasks(self, task_ids: List[str]) -> Dict[str, TaskResult]:
return await self._handler.poll_tasks(task_ids)
async def validate(self, data: Dict[str, Any]) -> ValidationResult:
return await self._handler.validate(data)
def extract_id(self, data: Dict[str, Any]) -> str:
return self._handler.extract_id(data)
class NodeHandlerRegistry:
"""节点处理器注册表"""
sync_state_machine/datasource/jsonl/handler.py
+4 -9
View File
@@ -50,22 +50,17 @@ class BaseJsonlHandler(BaseNodeHandler):
):
super().__init__(node_type, node_class, schema)
self.datasource = datasource
self._target_project_ids: List[str] = []
def set_target_project_ids(self, target_project_ids: List[str]) -> None:
"""注入项目过滤列表的本地副本。真正的过滤来源仍然是 datasource。"""
self._target_project_ids = [str(pid) for pid in target_project_ids if str(pid)]
def _should_skip_item_by_project_filter(self, item: Dict[str, Any], item_id: str) -> bool:
target_project_ids = set(self.datasource.target_project_ids)
if not target_project_ids:
data_id_filter = set(self.get_data_id_filter())
if not data_id_filter:
return False
if self.node_type == "project":
return item_id not in target_project_ids
return item_id not in data_id_filter
project_id = item.get("project_id")
if project_id:
return str(project_id) not in target_project_ids
return str(project_id) not in data_id_filter
return False
async def load(self) -> List['SyncNode']:
sync_state_machine/domain/preparation/api_handler.py
-9
View File
@@ -27,17 +27,8 @@ class PreparationApiHandler(BaseApiHandler):
self._node_type = "preparation"
self._node_class = PreparationSyncNode
self._schema = PreparationDetail
self._filter_project_id = None
self.update_schemas = [PostPreparation]
def set_filter_project_id(self, project_id: str | List[str]):
"""设置要加载的项目ID过滤列表"""
self._filter_project_id = project_id
def set_target_project_ids(self, target_project_ids: List[str]) -> None:
"""设置目标项目ID列表"""
self.set_filter_project_id(target_project_ids)
async def load(self) -> List[SyncNode]:
"""从 API 加载里程碑数据（需要项目上下文）"""
from ..project.api_handler import ProjectApiHandler
sync_state_machine/domain/project/api_handler.py
+4 -22
View File
@@ -71,26 +71,11 @@ class ProjectApiHandler(BaseApiHandler):
# 类级别缓存：存储 all_info 数据
_all_info_cache: Dict[str, Dict[str, Any]] = {}
def __init__(self, filter_project_id: str | List[str] | None = None):
"""
初始化 Project Handler
Args:
filter_project_id: 指定只加载的项目ID列表（可选，用于多项目测试）
"""
def __init__(self):
super().__init__()
self._node_type = "project"
self._node_class = ProjectSyncNode
self._schema = ProjectResponseBase
if filter_project_id is None:
self._filter_project_id = None
elif isinstance(filter_project_id, str):
self._filter_project_id = [filter_project_id]
else:
self._filter_project_id = filter_project_id # 保存过滤条件
def set_target_project_ids(self, target_project_ids: List[str]) -> None:
self._filter_project_id = list(target_project_ids) if target_project_ids else None
@classmethod
async def get_all_info(cls, client, project_id: str) -> Dict[str, Any]:
@@ -127,15 +112,12 @@ class ProjectApiHandler(BaseApiHandler):
nodes = []
# 确定要加载的项目ID列表：参数 > 构造函数配置
target_project_ids = self._filter_project_id or []
if isinstance(target_project_ids, str):
target_project_ids = [target_project_ids]
project_id_filter = self.get_data_id_filter()
if target_project_ids:
if project_id_filter:
# 只加载指定项目列表
projects : List[ProjectResponseBase]= []
for project_id in target_project_ids:
for project_id in project_id_filter:
project_response = await api_get_project(self.api_client, project_id)
projects.append(project_response)
else:
sync_state_machine/pipeline/factory.py
+12 -20
View File
@@ -277,9 +277,6 @@ async def create_pipeline_from_config(
project_root=_project_root(),
)
await _initialize_datasource(remote_ds, endpoint_name="remote")
local_ds.set_target_project_ids([str(pid) for pid in config.local_datasource.target_project_ids if str(pid)])
remote_ds.set_target_project_ids([str(pid) for pid in config.remote_datasource.target_project_ids if str(pid)])
_register_handlers(config, local_ds, remote_ds, all_types)
local_collection = DataCollection("local", persistence, auto_persist=False)
@@ -434,8 +431,6 @@ async def create_jsonl_to_api_pipeline(
api_secret: str,
# 节点类型
node_types: List[str],
# 项目过滤
target_project_ids: Optional[List[str]] = None,
strategy_overrides: Optional[Dict[str, Dict[str, Any]]] = None,
# Pipeline 行为
enable_persistence: bool = True,
@@ -446,8 +441,8 @@ async def create_jsonl_to_api_pipeline(
poll_interval: float = 0.5,
api_rate_limit_max_requests: int = 30,
api_rate_limit_window_seconds: float = 10.0,
# Handler 自定义配置（可选，作用于 API 端）
handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
local_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
remote_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
# Logger 配置
initialize_logger: bool = True,
logger_file_path: Optional[str] = None,
@@ -459,7 +454,7 @@ async def create_jsonl_to_api_pipeline(
type="jsonl",
jsonl_dir=local_jsonl_dir,
read_only=False,
target_project_ids=target_project_ids or [],
handler_configs=local_handler_configs or {},
),
remote_datasource=ApiDataSourceConfig(
type="api",
@@ -471,8 +466,7 @@ async def create_jsonl_to_api_pipeline(
poll_interval=poll_interval,
api_rate_limit_max_requests=api_rate_limit_max_requests,
api_rate_limit_window_seconds=api_rate_limit_window_seconds,
target_project_ids=target_project_ids or [],
handler_configs=handler_configs or {},
handler_configs=remote_handler_configs or {},
),
strategies=[],
persist=PersistConfig(
@@ -502,8 +496,6 @@ async def create_api_to_jsonl_pipeline(
persistence_db: str,
# 节点类型
node_types: List[str],
# 项目过滤
target_project_ids: Optional[List[str]] = None,
strategy_overrides: Optional[Dict[str, Dict[str, Any]]] = None,
# Pipeline 行为
enable_persistence: bool = True,
@@ -514,8 +506,8 @@ async def create_api_to_jsonl_pipeline(
poll_interval: float = 0.5,
api_rate_limit_max_requests: int = 30,
api_rate_limit_window_seconds: float = 10.0,
# Handler 自定义配置（可选，作用于 API 端）
handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
local_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
remote_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
# Logger 配置
initialize_logger: bool = True,
logger_file_path: Optional[str] = None,
@@ -533,14 +525,13 @@ async def create_api_to_jsonl_pipeline(
poll_interval=poll_interval,
api_rate_limit_max_requests=api_rate_limit_max_requests,
api_rate_limit_window_seconds=api_rate_limit_window_seconds,
target_project_ids=target_project_ids or [],
handler_configs=handler_configs or {},
handler_configs=local_handler_configs or {},
),
remote_datasource=JsonlDataSourceConfig(
type="jsonl",
jsonl_dir=remote_jsonl_dir,
read_only=False,
target_project_ids=target_project_ids or [],
handler_configs=remote_handler_configs or {},
),
strategies=[],
persist=PersistConfig(
@@ -565,10 +556,11 @@ async def create_jsonl_to_jsonl_pipeline(
remote_jsonl_dir: str,
persistence_db: str,
node_types: List[str],
target_project_ids: Optional[List[str]] = None,
enable_persistence: bool = True,
wipe_persistence_on_start: bool = False,
strategy_overrides: Optional[Dict[str, Dict[str, Any]]] = None,
local_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
remote_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
initialize_logger: bool = True,
logger_file_path: Optional[str] = None,
logger_level: int = logging.INFO,
@@ -579,13 +571,13 @@ async def create_jsonl_to_jsonl_pipeline(
type="jsonl",
jsonl_dir=local_jsonl_dir,
read_only=False,
target_project_ids=target_project_ids or [],
handler_configs=local_handler_configs or {},
),
remote_datasource=JsonlDataSourceConfig(
type="jsonl",
jsonl_dir=remote_jsonl_dir,
read_only=False,
target_project_ids=target_project_ids or [],
handler_configs=remote_handler_configs or {},
),
strategies=[],
persist=PersistConfig(
sync_state_machine/pipeline/full_sync_pipeline.py
+24 -13
View File
@@ -90,6 +90,7 @@ class FullSyncPipeline:
self._consistency_by_type: Dict[str, Dict[str, Any]] = {}
self._logger = logger
self._closed = False
self._loaded_node_types: set[str] = set()
def _resolve_pipeline_runtime(self) -> StateMachineRuntime:
if self.strategies:
@@ -119,6 +120,7 @@ class FullSyncPipeline:
await self.close()
async def phase_sync_by_node_type(self) -> None:
self._prepare_datasources()
strategy_map = {s.node_type: s for s in self.strategies}
for node_type in self.sync_order:
strategy = strategy_map.get(node_type)
@@ -130,6 +132,8 @@ class FullSyncPipeline:
self._logger.info(f"🧩 Strategy start: {node_type}")
self._logger.info("-" * section_width)
await self._load_node_type(node_type, reason="pre-strategy refresh")
await strategy.bind()
self._logger.info(f"✅ Strategy bind: {node_type}")
@@ -169,10 +173,11 @@ class FullSyncPipeline:
await self._load_persistence_state()
self._logger.info("✅ Bootstrap: persistence loaded")
await self._load_from_datasource()
self._logger.info("✅ Bootstrap: datasource fetched")
self._prepare_datasources()
self._logger.info("✅ Bootstrap: datasource prepared")
async def phase_bind(self) -> None:
self._prepare_datasources()
strategy_map = {s.node_type: s for s in self.strategies}
for node_type in self.sync_order:
strategy = strategy_map.get(node_type)
@@ -186,6 +191,7 @@ class FullSyncPipeline:
continue
async def phase_create(self) -> None:
self._prepare_datasources()
strategy_map = {s.node_type: s for s in self.strategies}
for node_type in self.sync_order:
strategy = strategy_map.get(node_type)
@@ -207,6 +213,7 @@ class FullSyncPipeline:
continue
async def phase_update(self) -> None:
self._prepare_datasources()
strategy_map = {s.node_type: s for s in self.strategies}
for node_type in self.sync_order:
strategy = strategy_map.get(node_type)
@@ -271,26 +278,30 @@ class FullSyncPipeline:
self._logger.info(f"🧹 Reset cleanup: {total_cleaned} CREATE-failed zombies cleaned")
self._logger.info("🔄 Post-load reset cleanup completed")
async def _load_from_datasource(self) -> None:
"""从数据源加载数据"""
def _prepare_datasources(self) -> None:
"""为后续按 node_type 加载准备 collection 绑定。"""
self.local_datasource.set_collection(self.local_collection)
await self.local_datasource.load_all(order=self.load_order)
self.remote_datasource.set_collection(self.remote_collection)
await self.remote_datasource.load_all(order=self.load_order)
for node_type in self.node_types:
self.stats["loaded"] += len(self.local_collection.filter(node_type=node_type))
self.stats["loaded"] += len(self.remote_collection.filter(node_type=node_type))
async def _load_node_type(self, node_type: str, *, reason: str) -> None:
self._logger.info(f"🔄 Load node_type={node_type}, reason={reason}")
await self.local_datasource.load_all(order=[node_type])
await self.remote_datasource.load_all(order=[node_type])
if node_type in self._loaded_node_types:
return
self._loaded_node_types.add(node_type)
self.stats["loaded"] += len(self.local_collection.filter(node_type=node_type))
self.stats["loaded"] += len(self.remote_collection.filter(node_type=node_type))
def _get_action_success_count(self, datasource: "BaseDataSource", node_type: str, action_key: str) -> int:
summary = datasource.get_action_summary().get(node_type, {})
action_summary = summary.get(action_key, {}) if isinstance(summary, dict) else {}
return int(action_summary.get("success", 0)) if isinstance(action_summary, dict) else 0
async def _reload_node_type(self, node_type: str, *, reason: str) -> None:
self._logger.info(f"🔄 Reload after write: node_type={node_type}, reason={reason}")
await self.local_datasource.load_all(order=[node_type])
await self.remote_datasource.load_all(order=[node_type])
await self._load_node_type(node_type, reason=reason)
async def _evaluate_consistency_for_node_type(self, node_type: str) -> Dict[str, Any]:
strategy = next((item for item in self.strategies if item.node_type == node_type), None)