first commit

2026-03-09 16:31:42 +08:00
commit 4a9c444b10
486 changed files with 52479 additions and 0 deletions
@@ -0,0 +1,208 @@
+"""
+Run FullSyncPipeline: JSONL -> JSONL (filtered_datasource -> remote_datasource)
+
+直接运行脚本，不使用 pytest。
+"""
+
+from __future__ import annotations
+
+import asyncio
+import sys
+from pathlib import Path
+from typing import List
+
+# 确保能导入项目模块
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+# 触发 DomainRegistry 注册
+import sync_system_new.domain  # noqa: F401, E402
+
+from sync_system_new.common.binding import BindingManager  # noqa: E402
+from sync_system_new.common.collection import DataCollection  # noqa: E402
+from sync_system_new.common.persistence import PersistenceBackend  # noqa: E402
+from sync_system_new.common.registry import DomainRegistry  # noqa: E402
+from sync_system_new.datasource.jsonl.datasource import JsonlDataSource  # noqa: E402
+from sync_system_new.pipeline import FullSyncPipeline  # noqa: E402
+from sync_system_new.sync_system.config import OrphanAction, UpdateDirection  # noqa: E402
+
+
+LOCAL_DIR = PROJECT_ROOT / "filtered_datasource" / "datasource" / "ecm-2025-12-02"
+REMOTE_DIR = PROJECT_ROOT / "remote_datasource" / "datasource"
+PERSISTENCE_DB = PROJECT_ROOT / "test_results" / "full_sync_pipeline_jsonl_run.db"
+WIPE_DB_ON_START = False
+
+NODE_TYPES: List[str] = [
+    "company",
+    # "supplier",
+    "user",
+    "project",
+    "project_detail_data",
+    "contract",
+    "contract_change",
+    "construction_task",
+    "construction_task_detail",
+    "material",
+    "material_detail",
+    "contract_settlement",
+    "contract_settlement_detail",
+    "personnel",
+    "personnel_detail",
+    "preparation",
+    "production",
+    "lar",
+    "units",
+]
+
+
+class StrategyLogger:
+    """包装策略，打印执行完成信息。"""
+
+    def __init__(self, inner):
+        self.inner = inner
+        self.node_type = inner.node_type
+
+    async def bind(self):
+        return await self.inner.bind()
+
+    async def create(self):
+        created = await self.inner.create()
+        print(f"✅ Strategy create: {self.node_type} (created={len(created)})")
+        return created
+
+    async def update(self):
+        updated = await self.inner.update()
+        print(f"✅ Strategy update: {self.node_type} (updated={len(updated)})")
+        return updated
+
+
+async def main() -> None:
+    PERSISTENCE_DB.parent.mkdir(parents=True, exist_ok=True)
+    if WIPE_DB_ON_START and PERSISTENCE_DB.exists():
+        PERSISTENCE_DB.unlink()
+
+    persistence = PersistenceBackend(str(PERSISTENCE_DB))
+    await persistence.initialize()
+
+    local_ds = JsonlDataSource(LOCAL_DIR, read_only=True)
+    remote_ds = JsonlDataSource(REMOTE_DIR, read_only=False)
+
+    for node_type in NODE_TYPES:
+        handler_class = DomainRegistry.get_jsonl_handler(node_type)
+        if handler_class is None:
+            raise RuntimeError(f"Missing JSONL handler for node_type: {node_type}")
+        local_ds.register_handler(handler_class(local_ds))
+        remote_ds.register_handler(handler_class(remote_ds))
+
+    local_collection = DataCollection("local", persistence, auto_persist=False)
+    remote_collection = DataCollection("remote", persistence, auto_persist=False)
+    binding_manager = BindingManager(persistence, auto_persist=False)
+
+    strategies = []
+    for node_type in NODE_TYPES:
+        strategy_class = DomainRegistry.get_strategy_class(node_type)
+        if strategy_class is None:
+            raise RuntimeError(f"Missing strategy for node_type: {node_type}")
+        strategy = strategy_class(node_type, local_collection, remote_collection, binding_manager)
+        # 运行时强制只推不拉（不覆盖原有 depend_fields/auto_bind_fields）
+        strategy.update_config(
+            local_orphan_action=OrphanAction.CREATE_REMOTE,
+            remote_orphan_action=OrphanAction.NONE,
+            update_direction=UpdateDirection.PUSH,
+        )
+        if node_type == "contract":
+            # contract 仅依赖 project_id，避免 company_id 为空导致依赖阻断
+            strategy.update_config(depend_fields={"project_id": "project"})
+        strategies.append(StrategyLogger(strategy))
+
+    pipeline = FullSyncPipeline(
+        local_datasource=local_ds,
+        remote_datasource=remote_ds,
+        strategies=strategies,
+        node_types=NODE_TYPES,
+        load_order=NODE_TYPES,
+        sync_order=NODE_TYPES,
+        persistence=persistence,
+        local_collection=local_collection,
+        remote_collection=remote_collection,
+        binding_manager=binding_manager,
+        enable_persistence=True,
+        fail_fast_on_precheck=False,
+    )
+
+    stats = await pipeline.run()
+    print(f"Pipeline finished. Stats: {stats}")
+
+    binding_counts = {}
+    for node_type in NODE_TYPES:
+        records = await binding_manager.get_all_records(node_type)
+        binding_counts[node_type] = len(records)
+
+    def format_action(summary, key: str) -> str:
+        action = summary.get(key, {})
+        return f"{action.get('total', 0)}({action.get('success', 0)}/{action.get('failed', 0)})"
+
+    def format_total(summary) -> str:
+        total = summary.get("total", {})
+        return f"{total.get('total', 0)}({total.get('success', 0)}/{total.get('failed', 0)})"
+
+    def print_summary(title: str, action_summary: dict) -> None:
+        print(f"\n=== {title} summary ===")
+        print("doc_type: bindings | create(s/f) | update(s/f) | delete(s/f) | none | total(s/f)")
+
+        total_bindings = 0
+        total_create = {"total": 0, "success": 0, "failed": 0}
+        total_update = {"total": 0, "success": 0, "failed": 0}
+        total_delete = {"total": 0, "success": 0, "failed": 0}
+        total_none = 0
+        total_all = {"total": 0, "success": 0, "failed": 0}
+
+        for node_type in NODE_TYPES:
+            summary = action_summary.get(node_type, {})
+            bindings = binding_counts.get(node_type, 0)
+            create_str = format_action(summary, "create")
+            update_str = format_action(summary, "update")
+            delete_str = format_action(summary, "delete")
+            none_count = summary.get("none", {}).get("total", 0)
+            total_str = format_total(summary)
+
+            print(
+                f"{node_type}: bindings={bindings} "
+                f"create={create_str} update={update_str} delete={delete_str} "
+                f"none={none_count} total={total_str}"
+            )
+
+            total_bindings += bindings
+            for bucket, key in (
+                (total_create, "create"),
+                (total_update, "update"),
+                (total_delete, "delete"),
+            ):
+                action = summary.get(key, {})
+                bucket["total"] += action.get("total", 0)
+                bucket["success"] += action.get("success", 0)
+                bucket["failed"] += action.get("failed", 0)
+            total_none += none_count
+            total_all["total"] += summary.get("total", {}).get("total", 0)
+            total_all["success"] += summary.get("total", {}).get("success", 0)
+            total_all["failed"] += summary.get("total", {}).get("failed", 0)
+
+        print(
+            "TOTAL: "
+            f"bindings={total_bindings} "
+            f"create={total_create['total']}({total_create['success']}/{total_create['failed']}) "
+            f"update={total_update['total']}({total_update['success']}/{total_update['failed']}) "
+            f"delete={total_delete['total']}({total_delete['success']}/{total_delete['failed']}) "
+            f"none={total_none} "
+            f"total={total_all['total']}({total_all['success']}/{total_all['failed']})"
+        )
+
+    print_summary("Local", local_ds.get_action_summary())
+    print_summary("Remote", remote_ds.get_action_summary())
+
+    await persistence.close()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())