first commit
This commit is contained in:
strepsiades
@@ -0,0 +1,179 @@
|
||||
"""
|
||||
run_full_sync_pipeline_simple.py
|
||||
|
||||
极简测试脚本：演示使用工厂函数创建 Pipeline
|
||||
- 只需配置参数
|
||||
- 调用工厂函数
|
||||
- 执行 pipeline.run()
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# 项目路径
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
# 触发 Domain 注册
|
||||
import sync_system_new.domain # noqa: F401, E402
|
||||
|
||||
from sync_system_new.pipeline import create_jsonl_to_api_pipeline # noqa: E402
|
||||
|
||||
|
||||
# ========== 配置参数 ==========
|
||||
CONFIG = {
|
||||
# 路径
|
||||
"local_jsonl_dir": str(PROJECT_ROOT / "filtered_datasource" / "datasource" / "ecm-2025-12-02"),
|
||||
"persistence_db": str(PROJECT_ROOT / "_runtime" / "test_simple.db"),
|
||||
|
||||
# API
|
||||
"api_base_url": "http://localhost:8000/api/third/v2",
|
||||
"api_uid": "6b18c681-92d5-4c7f-802c-df413b6504c8",
|
||||
"api_secret": "1a28Az5dBD1Gc3URpmHwiR0XfzW7UtScQnqJQzgpzdg",
|
||||
|
||||
# 节点类型
|
||||
"node_types": [
|
||||
"company",
|
||||
"supplier",
|
||||
# "user",
|
||||
"project",
|
||||
"project_detail_data",
|
||||
"preparation",
|
||||
"production",
|
||||
"lar",
|
||||
"lar_change",
|
||||
"units",
|
||||
"contract",
|
||||
"contract_change",
|
||||
"construction_task",
|
||||
"construction_task_detail",
|
||||
"material",
|
||||
"material_detail",
|
||||
"contract_settlement",
|
||||
"contract_settlement_detail",
|
||||
"personnel",
|
||||
"personnel_detail",
|
||||
],
|
||||
|
||||
# 项目过滤和预绑定
|
||||
"target_project_ids": ["f3e02e84-e81c-4aa6-88d4-f5aa108c8227"],
|
||||
"project_id_bindings": {
|
||||
"f3e02e84-e81c-4aa6-88d4-f5aa108c8227": "f3e02e84-e81c-4aa6-88d4-f5aa108c8227",
|
||||
},
|
||||
|
||||
# Handler 自定义配置
|
||||
"handler_configs": {
|
||||
# supplier: 从持久化加载，不调用 API（后续运行时启用）
|
||||
"supplier": {"load_mode": "persisted_only"},
|
||||
},
|
||||
|
||||
# Skip/Force Sync 控制
|
||||
# skip_sync_types: 跳过同步（只执行 bind，不执行 create/update）
|
||||
# 后续运行时可配置: "skip_sync_types": ["supplier"]
|
||||
"skip_sync_types": ["supplier"],
|
||||
# 行为
|
||||
"wipe_persistence_on_start": False, # 改为 False，否则持久化数据会被清空
|
||||
"enable_persistence": True,
|
||||
|
||||
# API 调试
|
||||
"api_debug": True,
|
||||
"poll_max_retries": 1,
|
||||
"poll_interval": 2.0,
|
||||
}
|
||||
|
||||
|
||||
async def main():
|
||||
"""主函数 - 返回 True 表示成功，False 表示被中断"""
|
||||
from datetime import datetime
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
pipeline = None
|
||||
interrupted = False
|
||||
log_file = None
|
||||
original_stdout = None
|
||||
original_stderr = None
|
||||
|
||||
# 日志设置
|
||||
log_dir = Path(PROJECT_ROOT) / "test_results"
|
||||
log_dir.mkdir(parents=True, exist_ok=True)
|
||||
log_path = log_dir / f"simple_pipeline_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log"
|
||||
|
||||
try:
|
||||
# 重定向输出到日志文件
|
||||
log_file = open(log_path, "w", encoding="utf-8")
|
||||
|
||||
class Tee:
|
||||
def __init__(self, *streams):
|
||||
self._streams = streams
|
||||
def write(self, data):
|
||||
for stream in self._streams:
|
||||
stream.write(data)
|
||||
stream.flush()
|
||||
return len(data)
|
||||
def flush(self):
|
||||
for stream in self._streams:
|
||||
stream.flush()
|
||||
|
||||
original_stdout = sys.stdout
|
||||
original_stderr = sys.stderr
|
||||
sys.stdout = Tee(original_stdout, log_file)
|
||||
sys.stderr = Tee(original_stderr, log_file)
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("🚀 Creating Pipeline...")
|
||||
print(f"📝 Log file: {log_path}")
|
||||
print("=" * 60)
|
||||
|
||||
# 一行创建完整 Pipeline
|
||||
pipeline = await create_jsonl_to_api_pipeline(**CONFIG)
|
||||
|
||||
print(f"✅ Pipeline created!")
|
||||
print(f" - Node types: {len(pipeline.node_types)}")
|
||||
print(f" - Strategies: {len(pipeline.strategies)}")
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("🏃 Running Pipeline...")
|
||||
print("=" * 60)
|
||||
|
||||
# 执行同步（print_summary 已整合到 stage4_post_check 中）
|
||||
stats = await pipeline.run()
|
||||
|
||||
print("\n✅ Done!")
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n⚠️ KeyboardInterrupt received, shutting down gracefully...")
|
||||
interrupted = True
|
||||
except Exception as e:
|
||||
print(f"\n\n❌ Error: {type(e).__name__}: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
finally:
|
||||
if pipeline is not None:
|
||||
print("🧹 Cleaning up resources...")
|
||||
await pipeline.close()
|
||||
print("✅ Resources cleaned up")
|
||||
|
||||
# 恢复原始输出
|
||||
if original_stdout is not None:
|
||||
sys.stdout = original_stdout
|
||||
if original_stderr is not None:
|
||||
sys.stderr = original_stderr
|
||||
if log_file is not None:
|
||||
log_file.close()
|
||||
if not interrupted:
|
||||
print(f"\n📝 Log saved to: {log_path}")
|
||||
|
||||
return not interrupted # 返回 False 表示被中断
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
success = asyncio.run(main())
|
||||
sys.exit(0 if success else 130) # 130 是标准的 Ctrl+C 退出码
|
||||
except KeyboardInterrupt:
|
||||
# asyncio.run() 可能会重新抛出 KeyboardInterrupt
|
||||
print("\n⚠️ Program interrupted")
|
||||
sys.exit(130)
|
||||
Reference in New Issue
Block a user