""" Run FullSyncPipeline: JSONL -> JSONL (filtered_datasource -> remote_datasource) 直接运行脚本，不使用 pytest。 """ from __future__ import annotations import asyncio import sys from pathlib import Path from typing import List # 确保能导入项目模块 PROJECT_ROOT = Path(__file__).resolve().parent.parent if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) # 触发 DomainRegistry 注册 import sync_system_new.domain # noqa: F401, E402 from sync_system_new.common.binding import BindingManager # noqa: E402 from sync_system_new.common.collection import DataCollection # noqa: E402 from sync_system_new.common.persistence import PersistenceBackend # noqa: E402 from sync_system_new.common.registry import DomainRegistry # noqa: E402 from sync_system_new.datasource.jsonl.datasource import JsonlDataSource # noqa: E402 from sync_system_new.pipeline import FullSyncPipeline # noqa: E402 from sync_system_new.sync_system.config import OrphanAction, UpdateDirection # noqa: E402 LOCAL_DIR = PROJECT_ROOT / "filtered_datasource" / "datasource" / "ecm-2025-12-02" REMOTE_DIR = PROJECT_ROOT / "remote_datasource" / "datasource" PERSISTENCE_DB = PROJECT_ROOT / "test_results" / "full_sync_pipeline_jsonl_run.db" WIPE_DB_ON_START = False NODE_TYPES: List[str] = [ "company", # "supplier", "user", "project", "project_detail_data", "contract", "contract_change", "construction_task", "construction_task_detail", "material", "material_detail", "contract_settlement", "contract_settlement_detail", "personnel", "personnel_detail", "preparation", "production", "lar", "units", ] class StrategyLogger: """包装策略，打印执行完成信息。""" def __init__(self, inner): self.inner = inner self.node_type = inner.node_type async def bind(self): return await self.inner.bind() async def create(self): created = await self.inner.create() print(f"✅ Strategy create: {self.node_type} (created={len(created)})") return created async def update(self): updated = await self.inner.update() print(f"✅ Strategy update: {self.node_type} (updated={len(updated)})") return updated async def main() -> None: PERSISTENCE_DB.parent.mkdir(parents=True, exist_ok=True) if WIPE_DB_ON_START and PERSISTENCE_DB.exists(): PERSISTENCE_DB.unlink() persistence = PersistenceBackend(str(PERSISTENCE_DB)) await persistence.initialize() local_ds = JsonlDataSource(LOCAL_DIR, read_only=True) remote_ds = JsonlDataSource(REMOTE_DIR, read_only=False) for node_type in NODE_TYPES: handler_class = DomainRegistry.get_jsonl_handler(node_type) if handler_class is None: raise RuntimeError(f"Missing JSONL handler for node_type: {node_type}") local_ds.register_handler(handler_class(local_ds)) remote_ds.register_handler(handler_class(remote_ds)) local_collection = DataCollection("local", persistence, auto_persist=False) remote_collection = DataCollection("remote", persistence, auto_persist=False) binding_manager = BindingManager(persistence, auto_persist=False) strategies = [] for node_type in NODE_TYPES: strategy_class = DomainRegistry.get_strategy_class(node_type) if strategy_class is None: raise RuntimeError(f"Missing strategy for node_type: {node_type}") strategy = strategy_class(node_type, local_collection, remote_collection, binding_manager) # 运行时强制只推不拉（不覆盖原有 depend_fields/auto_bind_fields） strategy.update_config( local_orphan_action=OrphanAction.CREATE_REMOTE, remote_orphan_action=OrphanAction.NONE, update_direction=UpdateDirection.PUSH, ) if node_type == "contract": # contract 仅依赖 project_id，避免 company_id 为空导致依赖阻断 strategy.update_config(depend_fields={"project_id": "project"}) strategies.append(StrategyLogger(strategy)) pipeline = FullSyncPipeline( local_datasource=local_ds, remote_datasource=remote_ds, strategies=strategies, node_types=NODE_TYPES, load_order=NODE_TYPES, sync_order=NODE_TYPES, persistence=persistence, local_collection=local_collection, remote_collection=remote_collection, binding_manager=binding_manager, enable_persistence=True, fail_fast_on_precheck=False, ) stats = await pipeline.run() print(f"Pipeline finished. Stats: {stats}") binding_counts = {} for node_type in NODE_TYPES: records = await binding_manager.get_all_records(node_type) binding_counts[node_type] = len(records) def format_action(summary, key: str) -> str: action = summary.get(key, {}) return f"{action.get('total', 0)}({action.get('success', 0)}/{action.get('failed', 0)})" def format_total(summary) -> str: total = summary.get("total", {}) return f"{total.get('total', 0)}({total.get('success', 0)}/{total.get('failed', 0)})" def print_summary(title: str, action_summary: dict) -> None: print(f"\n=== {title} summary ===") print("doc_type: bindings | create(s/f) | update(s/f) | delete(s/f) | none | total(s/f)") total_bindings = 0 total_create = {"total": 0, "success": 0, "failed": 0} total_update = {"total": 0, "success": 0, "failed": 0} total_delete = {"total": 0, "success": 0, "failed": 0} total_none = 0 total_all = {"total": 0, "success": 0, "failed": 0} for node_type in NODE_TYPES: summary = action_summary.get(node_type, {}) bindings = binding_counts.get(node_type, 0) create_str = format_action(summary, "create") update_str = format_action(summary, "update") delete_str = format_action(summary, "delete") none_count = summary.get("none", {}).get("total", 0) total_str = format_total(summary) print( f"{node_type}: bindings={bindings} " f"create={create_str} update={update_str} delete={delete_str} " f"none={none_count} total={total_str}" ) total_bindings += bindings for bucket, key in ( (total_create, "create"), (total_update, "update"), (total_delete, "delete"), ): action = summary.get(key, {}) bucket["total"] += action.get("total", 0) bucket["success"] += action.get("success", 0) bucket["failed"] += action.get("failed", 0) total_none += none_count total_all["total"] += summary.get("total", {}).get("total", 0) total_all["success"] += summary.get("total", {}).get("success", 0) total_all["failed"] += summary.get("total", {}).get("failed", 0) print( "TOTAL: " f"bindings={total_bindings} " f"create={total_create['total']}({total_create['success']}/{total_create['failed']}) " f"update={total_update['total']}({total_update['success']}/{total_update['failed']}) " f"delete={total_delete['total']}({total_delete['success']}/{total_delete['failed']}) " f"none={total_none} " f"total={total_all['total']}({total_all['success']}/{total_all['failed']})" ) print_summary("Local", local_ds.get_action_summary()) print_summary("Remote", remote_ds.get_action_summary()) await persistence.close() if __name__ == "__main__": asyncio.run(main())