strepsiades
209 lines
7.6 KiB
Python
209 lines
7.6 KiB
Python
"""
|
||
Run FullSyncPipeline: JSONL -> JSONL (filtered_datasource -> remote_datasource)
|
||
|
||
直接运行脚本，不使用 pytest。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import sys
|
||
from pathlib import Path
|
||
from typing import List
|
||
|
||
# 确保能导入项目模块
|
||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||
if str(PROJECT_ROOT) not in sys.path:
|
||
sys.path.insert(0, str(PROJECT_ROOT))
|
||
|
||
# 触发 DomainRegistry 注册
|
||
import sync_system_new.domain # noqa: F401, E402
|
||
|
||
from sync_system_new.common.binding import BindingManager # noqa: E402
|
||
from sync_system_new.common.collection import DataCollection # noqa: E402
|
||
from sync_system_new.common.persistence import PersistenceBackend # noqa: E402
|
||
from sync_system_new.common.registry import DomainRegistry # noqa: E402
|
||
from sync_system_new.datasource.jsonl.datasource import JsonlDataSource # noqa: E402
|
||
from sync_system_new.pipeline import FullSyncPipeline # noqa: E402
|
||
from sync_system_new.sync_system.config import OrphanAction, UpdateDirection # noqa: E402
|
||
|
||
|
||
LOCAL_DIR = PROJECT_ROOT / "filtered_datasource" / "datasource" / "ecm-2025-12-02"
|
||
REMOTE_DIR = PROJECT_ROOT / "remote_datasource" / "datasource"
|
||
PERSISTENCE_DB = PROJECT_ROOT / "test_results" / "full_sync_pipeline_jsonl_run.db"
|
||
WIPE_DB_ON_START = False
|
||
|
||
NODE_TYPES: List[str] = [
|
||
"company",
|
||
# "supplier",
|
||
"user",
|
||
"project",
|
||
"project_detail_data",
|
||
"contract",
|
||
"contract_change",
|
||
"construction_task",
|
||
"construction_task_detail",
|
||
"material",
|
||
"material_detail",
|
||
"contract_settlement",
|
||
"contract_settlement_detail",
|
||
"personnel",
|
||
"personnel_detail",
|
||
"preparation",
|
||
"production",
|
||
"lar",
|
||
"units",
|
||
]
|
||
|
||
|
||
class StrategyLogger:
|
||
"""包装策略，打印执行完成信息。"""
|
||
|
||
def __init__(self, inner):
|
||
self.inner = inner
|
||
self.node_type = inner.node_type
|
||
|
||
async def bind(self):
|
||
return await self.inner.bind()
|
||
|
||
async def create(self):
|
||
created = await self.inner.create()
|
||
print(f"✅ Strategy create: {self.node_type} (created={len(created)})")
|
||
return created
|
||
|
||
async def update(self):
|
||
updated = await self.inner.update()
|
||
print(f"✅ Strategy update: {self.node_type} (updated={len(updated)})")
|
||
return updated
|
||
|
||
|
||
async def main() -> None:
|
||
PERSISTENCE_DB.parent.mkdir(parents=True, exist_ok=True)
|
||
if WIPE_DB_ON_START and PERSISTENCE_DB.exists():
|
||
PERSISTENCE_DB.unlink()
|
||
|
||
persistence = PersistenceBackend(str(PERSISTENCE_DB))
|
||
await persistence.initialize()
|
||
|
||
local_ds = JsonlDataSource(LOCAL_DIR, read_only=True)
|
||
remote_ds = JsonlDataSource(REMOTE_DIR, read_only=False)
|
||
|
||
for node_type in NODE_TYPES:
|
||
handler_class = DomainRegistry.get_jsonl_handler(node_type)
|
||
if handler_class is None:
|
||
raise RuntimeError(f"Missing JSONL handler for node_type: {node_type}")
|
||
local_ds.register_handler(handler_class(local_ds))
|
||
remote_ds.register_handler(handler_class(remote_ds))
|
||
|
||
local_collection = DataCollection("local", persistence, auto_persist=False)
|
||
remote_collection = DataCollection("remote", persistence, auto_persist=False)
|
||
binding_manager = BindingManager(persistence, auto_persist=False)
|
||
|
||
strategies = []
|
||
for node_type in NODE_TYPES:
|
||
strategy_class = DomainRegistry.get_strategy_class(node_type)
|
||
if strategy_class is None:
|
||
raise RuntimeError(f"Missing strategy for node_type: {node_type}")
|
||
strategy = strategy_class(node_type, local_collection, remote_collection, binding_manager)
|
||
# 运行时强制只推不拉（不覆盖原有 depend_fields/auto_bind_fields）
|
||
strategy.update_config(
|
||
local_orphan_action=OrphanAction.CREATE_REMOTE,
|
||
remote_orphan_action=OrphanAction.NONE,
|
||
update_direction=UpdateDirection.PUSH,
|
||
)
|
||
if node_type == "contract":
|
||
# contract 仅依赖 project_id，避免 company_id 为空导致依赖阻断
|
||
strategy.update_config(depend_fields={"project_id": "project"})
|
||
strategies.append(StrategyLogger(strategy))
|
||
|
||
pipeline = FullSyncPipeline(
|
||
local_datasource=local_ds,
|
||
remote_datasource=remote_ds,
|
||
strategies=strategies,
|
||
node_types=NODE_TYPES,
|
||
load_order=NODE_TYPES,
|
||
sync_order=NODE_TYPES,
|
||
persistence=persistence,
|
||
local_collection=local_collection,
|
||
remote_collection=remote_collection,
|
||
binding_manager=binding_manager,
|
||
enable_persistence=True,
|
||
fail_fast_on_precheck=False,
|
||
)
|
||
|
||
stats = await pipeline.run()
|
||
print(f"Pipeline finished. Stats: {stats}")
|
||
|
||
binding_counts = {}
|
||
for node_type in NODE_TYPES:
|
||
records = await binding_manager.get_all_records(node_type)
|
||
binding_counts[node_type] = len(records)
|
||
|
||
def format_action(summary, key: str) -> str:
|
||
action = summary.get(key, {})
|
||
return f"{action.get('total', 0)}({action.get('success', 0)}/{action.get('failed', 0)})"
|
||
|
||
def format_total(summary) -> str:
|
||
total = summary.get("total", {})
|
||
return f"{total.get('total', 0)}({total.get('success', 0)}/{total.get('failed', 0)})"
|
||
|
||
def print_summary(title: str, action_summary: dict) -> None:
|
||
print(f"\n=== {title} summary ===")
|
||
print("doc_type: bindings | create(s/f) | update(s/f) | delete(s/f) | none | total(s/f)")
|
||
|
||
total_bindings = 0
|
||
total_create = {"total": 0, "success": 0, "failed": 0}
|
||
total_update = {"total": 0, "success": 0, "failed": 0}
|
||
total_delete = {"total": 0, "success": 0, "failed": 0}
|
||
total_none = 0
|
||
total_all = {"total": 0, "success": 0, "failed": 0}
|
||
|
||
for node_type in NODE_TYPES:
|
||
summary = action_summary.get(node_type, {})
|
||
bindings = binding_counts.get(node_type, 0)
|
||
create_str = format_action(summary, "create")
|
||
update_str = format_action(summary, "update")
|
||
delete_str = format_action(summary, "delete")
|
||
none_count = summary.get("none", {}).get("total", 0)
|
||
total_str = format_total(summary)
|
||
|
||
print(
|
||
f"{node_type}: bindings={bindings} "
|
||
f"create={create_str} update={update_str} delete={delete_str} "
|
||
f"none={none_count} total={total_str}"
|
||
)
|
||
|
||
total_bindings += bindings
|
||
for bucket, key in (
|
||
(total_create, "create"),
|
||
(total_update, "update"),
|
||
(total_delete, "delete"),
|
||
):
|
||
action = summary.get(key, {})
|
||
bucket["total"] += action.get("total", 0)
|
||
bucket["success"] += action.get("success", 0)
|
||
bucket["failed"] += action.get("failed", 0)
|
||
total_none += none_count
|
||
total_all["total"] += summary.get("total", {}).get("total", 0)
|
||
total_all["success"] += summary.get("total", {}).get("success", 0)
|
||
total_all["failed"] += summary.get("total", {}).get("failed", 0)
|
||
|
||
print(
|
||
"TOTAL: "
|
||
f"bindings={total_bindings} "
|
||
f"create={total_create['total']}({total_create['success']}/{total_create['failed']}) "
|
||
f"update={total_update['total']}({total_update['success']}/{total_update['failed']}) "
|
||
f"delete={total_delete['total']}({total_delete['success']}/{total_delete['failed']}) "
|
||
f"none={total_none} "
|
||
f"total={total_all['total']}({total_all['success']}/{total_all['failed']})"
|
||
)
|
||
|
||
print_summary("Local", local_ds.get_action_summary())
|
||
print_summary("Remote", remote_ds.get_action_summary())
|
||
|
||
await persistence.close()
|
||
|
||
|
||
if __name__ == "__main__":
|
||
asyncio.run(main())
|