strepsiades/ecm_sync_system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

调整了pipeline执行顺序。现在按domain顺序读取数据源数据，防止前面domain的更新影响后面domain.

Browse Source 
将target_project_ids替换为更通用的data_id_filter
This commit is contained in:
strepsiades
2026-03-20 14:38:56 +08:00
parent f9f175ee79
commit 2c09c61165
29 changed files with 296 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sync_state_machine/pipeline/factory.py
+12 -20
View File
@@ -277,9 +277,6 @@ async def create_pipeline_from_config(
project_root=_project_root(),
)
await _initialize_datasource(remote_ds, endpoint_name="remote")
local_ds.set_target_project_ids([str(pid) for pid in config.local_datasource.target_project_ids if str(pid)])
remote_ds.set_target_project_ids([str(pid) for pid in config.remote_datasource.target_project_ids if str(pid)])
_register_handlers(config, local_ds, remote_ds, all_types)
local_collection = DataCollection("local", persistence, auto_persist=False)
@@ -434,8 +431,6 @@ async def create_jsonl_to_api_pipeline(
api_secret: str,
# 节点类型
node_types: List[str],
# 项目过滤
target_project_ids: Optional[List[str]] = None,
strategy_overrides: Optional[Dict[str, Dict[str, Any]]] = None,
# Pipeline 行为
enable_persistence: bool = True,
@@ -446,8 +441,8 @@ async def create_jsonl_to_api_pipeline(
poll_interval: float = 0.5,
api_rate_limit_max_requests: int = 30,
api_rate_limit_window_seconds: float = 10.0,
# Handler 自定义配置（可选，作用于 API 端）
handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
local_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
remote_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
# Logger 配置
initialize_logger: bool = True,
logger_file_path: Optional[str] = None,
@@ -459,7 +454,7 @@ async def create_jsonl_to_api_pipeline(
type="jsonl",
jsonl_dir=local_jsonl_dir,
read_only=False,
target_project_ids=target_project_ids or [],
handler_configs=local_handler_configs or {},
),
remote_datasource=ApiDataSourceConfig(
type="api",
@@ -471,8 +466,7 @@ async def create_jsonl_to_api_pipeline(
poll_interval=poll_interval,
api_rate_limit_max_requests=api_rate_limit_max_requests,
api_rate_limit_window_seconds=api_rate_limit_window_seconds,
target_project_ids=target_project_ids or [],
handler_configs=handler_configs or {},
handler_configs=remote_handler_configs or {},
),
strategies=[],
persist=PersistConfig(
@@ -502,8 +496,6 @@ async def create_api_to_jsonl_pipeline(
persistence_db: str,
# 节点类型
node_types: List[str],
# 项目过滤
target_project_ids: Optional[List[str]] = None,
strategy_overrides: Optional[Dict[str, Dict[str, Any]]] = None,
# Pipeline 行为
enable_persistence: bool = True,
@@ -514,8 +506,8 @@ async def create_api_to_jsonl_pipeline(
poll_interval: float = 0.5,
api_rate_limit_max_requests: int = 30,
api_rate_limit_window_seconds: float = 10.0,
# Handler 自定义配置（可选，作用于 API 端）
handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
local_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
remote_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
# Logger 配置
initialize_logger: bool = True,
logger_file_path: Optional[str] = None,
@@ -533,14 +525,13 @@ async def create_api_to_jsonl_pipeline(
poll_interval=poll_interval,
api_rate_limit_max_requests=api_rate_limit_max_requests,
api_rate_limit_window_seconds=api_rate_limit_window_seconds,
target_project_ids=target_project_ids or [],
handler_configs=handler_configs or {},
handler_configs=local_handler_configs or {},
),
remote_datasource=JsonlDataSourceConfig(
type="jsonl",
jsonl_dir=remote_jsonl_dir,
read_only=False,
target_project_ids=target_project_ids or [],
handler_configs=remote_handler_configs or {},
),
strategies=[],
persist=PersistConfig(
@@ -565,10 +556,11 @@ async def create_jsonl_to_jsonl_pipeline(
remote_jsonl_dir: str,
persistence_db: str,
node_types: List[str],
target_project_ids: Optional[List[str]] = None,
enable_persistence: bool = True,
wipe_persistence_on_start: bool = False,
strategy_overrides: Optional[Dict[str, Dict[str, Any]]] = None,
local_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
remote_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
initialize_logger: bool = True,
logger_file_path: Optional[str] = None,
logger_level: int = logging.INFO,
@@ -579,13 +571,13 @@ async def create_jsonl_to_jsonl_pipeline(
type="jsonl",
jsonl_dir=local_jsonl_dir,
read_only=False,
target_project_ids=target_project_ids or [],
handler_configs=local_handler_configs or {},
),
remote_datasource=JsonlDataSourceConfig(
type="jsonl",
jsonl_dir=remote_jsonl_dir,
read_only=False,
target_project_ids=target_project_ids or [],
handler_configs=remote_handler_configs or {},
),
strategies=[],
persist=PersistConfig(
sync_state_machine/pipeline/full_sync_pipeline.py
+24 -13
View File
@@ -90,6 +90,7 @@ class FullSyncPipeline:
self._consistency_by_type: Dict[str, Dict[str, Any]] = {}
self._logger = logger
self._closed = False
self._loaded_node_types: set[str] = set()
def _resolve_pipeline_runtime(self) -> StateMachineRuntime:
if self.strategies:
@@ -119,6 +120,7 @@ class FullSyncPipeline:
await self.close()
async def phase_sync_by_node_type(self) -> None:
self._prepare_datasources()
strategy_map = {s.node_type: s for s in self.strategies}
for node_type in self.sync_order:
strategy = strategy_map.get(node_type)
@@ -130,6 +132,8 @@ class FullSyncPipeline:
self._logger.info(f"🧩 Strategy start: {node_type}")
self._logger.info("-" * section_width)
await self._load_node_type(node_type, reason="pre-strategy refresh")
await strategy.bind()
self._logger.info(f"✅ Strategy bind: {node_type}")
@@ -169,10 +173,11 @@ class FullSyncPipeline:
await self._load_persistence_state()
self._logger.info("✅ Bootstrap: persistence loaded")
await self._load_from_datasource()
self._logger.info("✅ Bootstrap: datasource fetched")
self._prepare_datasources()
self._logger.info("✅ Bootstrap: datasource prepared")
async def phase_bind(self) -> None:
self._prepare_datasources()
strategy_map = {s.node_type: s for s in self.strategies}
for node_type in self.sync_order:
strategy = strategy_map.get(node_type)
@@ -186,6 +191,7 @@ class FullSyncPipeline:
continue
async def phase_create(self) -> None:
self._prepare_datasources()
strategy_map = {s.node_type: s for s in self.strategies}
for node_type in self.sync_order:
strategy = strategy_map.get(node_type)
@@ -207,6 +213,7 @@ class FullSyncPipeline:
continue
async def phase_update(self) -> None:
self._prepare_datasources()
strategy_map = {s.node_type: s for s in self.strategies}
for node_type in self.sync_order:
strategy = strategy_map.get(node_type)
@@ -271,26 +278,30 @@ class FullSyncPipeline:
self._logger.info(f"🧹 Reset cleanup: {total_cleaned} CREATE-failed zombies cleaned")
self._logger.info("🔄 Post-load reset cleanup completed")
async def _load_from_datasource(self) -> None:
"""从数据源加载数据"""
def _prepare_datasources(self) -> None:
"""为后续按 node_type 加载准备 collection 绑定。"""
self.local_datasource.set_collection(self.local_collection)
await self.local_datasource.load_all(order=self.load_order)
self.remote_datasource.set_collection(self.remote_collection)
await self.remote_datasource.load_all(order=self.load_order)
for node_type in self.node_types:
self.stats["loaded"] += len(self.local_collection.filter(node_type=node_type))
self.stats["loaded"] += len(self.remote_collection.filter(node_type=node_type))
async def _load_node_type(self, node_type: str, *, reason: str) -> None:
self._logger.info(f"🔄 Load node_type={node_type}, reason={reason}")
await self.local_datasource.load_all(order=[node_type])
await self.remote_datasource.load_all(order=[node_type])
if node_type in self._loaded_node_types:
return
self._loaded_node_types.add(node_type)
self.stats["loaded"] += len(self.local_collection.filter(node_type=node_type))
self.stats["loaded"] += len(self.remote_collection.filter(node_type=node_type))
def _get_action_success_count(self, datasource: "BaseDataSource", node_type: str, action_key: str) -> int:
summary = datasource.get_action_summary().get(node_type, {})
action_summary = summary.get(action_key, {}) if isinstance(summary, dict) else {}
return int(action_summary.get("success", 0)) if isinstance(action_summary, dict) else 0
async def _reload_node_type(self, node_type: str, *, reason: str) -> None:
self._logger.info(f"🔄 Reload after write: node_type={node_type}, reason={reason}")
await self.local_datasource.load_all(order=[node_type])
await self.remote_datasource.load_all(order=[node_type])
await self._load_node_type(node_type, reason=reason)
async def _evaluate_consistency_for_node_type(self, node_type: str) -> Dict[str, Any]:
strategy = next((item for item in self.strategies if item.node_type == node_type), None)