first commit
This commit is contained in:
strepsiades
@@ -0,0 +1,57 @@
|
||||
"""Public package entrypoint for the ECM sync engine."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from importlib.metadata import PackageNotFoundError, version
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
_PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(_PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_PROJECT_ROOT))
|
||||
|
||||
from . import domain as _domain
|
||||
from .common.registry import DomainRegistry
|
||||
from .config import (
|
||||
ApiDataSourceConfig,
|
||||
JsonlDataSourceConfig,
|
||||
PipelineRunConfig,
|
||||
build_config,
|
||||
build_default_config,
|
||||
load_named_preset_overrides,
|
||||
load_overrides_from_file,
|
||||
)
|
||||
from .datasource import ApiClient, ApiDataSource, BaseApiHandler, BaseJsonlHandler, JsonlDataSource
|
||||
from .pipeline import FullSyncPipeline, create_pipeline_from_config, run_pipeline_from_config
|
||||
|
||||
try:
|
||||
__version__ = version("ecm-sync-system")
|
||||
except PackageNotFoundError:
|
||||
__version__ = "0.1.0"
|
||||
|
||||
|
||||
def ensure_domain_registry_loaded() -> None:
|
||||
"""Ensure built-in domain registrations are imported."""
|
||||
_domain # keep module import for side effects
|
||||
|
||||
|
||||
__all__ = [
|
||||
"__version__",
|
||||
"ensure_domain_registry_loaded",
|
||||
"DomainRegistry",
|
||||
"PipelineRunConfig",
|
||||
"ApiDataSourceConfig",
|
||||
"JsonlDataSourceConfig",
|
||||
"build_config",
|
||||
"build_default_config",
|
||||
"load_overrides_from_file",
|
||||
"load_named_preset_overrides",
|
||||
"ApiClient",
|
||||
"ApiDataSource",
|
||||
"JsonlDataSource",
|
||||
"BaseApiHandler",
|
||||
"BaseJsonlHandler",
|
||||
"FullSyncPipeline",
|
||||
"create_pipeline_from_config",
|
||||
"run_pipeline_from_config",
|
||||
]
|
||||
@@ -0,0 +1,65 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from .config import build_config, load_overrides_from_file
|
||||
from .pipeline import run_pipeline_from_config
|
||||
|
||||
|
||||
def package_project_root() -> Path:
|
||||
return Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def resolve_config_path(raw_path: str, *, project_root: Path) -> Path:
|
||||
path = Path(raw_path)
|
||||
if not path.is_absolute():
|
||||
path = project_root / path
|
||||
return path.resolve()
|
||||
|
||||
|
||||
async def _main() -> int:
|
||||
project_root = package_project_root()
|
||||
|
||||
parser = argparse.ArgumentParser(description="Run sync pipeline from config override file")
|
||||
parser.add_argument(
|
||||
"--config_path",
|
||||
required=True,
|
||||
help="Path to a JSON/YAML run profile. Relative paths are resolved from the repository root.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
cfg_path = resolve_config_path(args.config_path, project_root=project_root)
|
||||
if not cfg_path.exists() or not cfg_path.is_file():
|
||||
raise FileNotFoundError(f"Config file not found: {cfg_path}")
|
||||
|
||||
overrides = load_overrides_from_file(cfg_path, project_root)
|
||||
config = build_config(project_root=project_root, overrides=overrides)
|
||||
|
||||
mode = f"{config.local_datasource.type}->{config.remote_datasource.type}"
|
||||
print(f"🧩 Loaded config file: {cfg_path}")
|
||||
await run_pipeline_from_config(
|
||||
config,
|
||||
title=f"sync_state_machine pipeline ({mode})",
|
||||
print_summary=True,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
return asyncio.run(_main())
|
||||
except KeyboardInterrupt:
|
||||
print("\n⚠️ Interrupted by user")
|
||||
return 130
|
||||
except Exception as exc:
|
||||
print(f"\n❌ Pipeline failed: {type(exc).__name__}: {exc}")
|
||||
return 1
|
||||
finally:
|
||||
logging.shutdown()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,15 @@
|
||||
from .types import SyncAction, SyncStatus, BindingStatus
|
||||
from .sync_node import SyncNode
|
||||
from .persistence import PersistenceBackend
|
||||
from .collection import DataCollection
|
||||
from .binding import BindingManager
|
||||
|
||||
__all__ = [
|
||||
"SyncAction",
|
||||
"SyncStatus",
|
||||
"BindingStatus",
|
||||
"SyncNode",
|
||||
"PersistenceBackend",
|
||||
"DataCollection",
|
||||
"BindingManager",
|
||||
]
|
||||
@@ -0,0 +1,161 @@
|
||||
import json
|
||||
import logging
|
||||
from typing import Optional, Dict, Any, List
|
||||
from .persistence import PersistenceBackend
|
||||
from .types import BindingStatus, BindingRecord
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BindingManager:
|
||||
"""
|
||||
BindingManager 负责维护不同 DataCollection 间节点的绑定关系。
|
||||
所有查询均在内存中进行，持久化仅作为备份。
|
||||
"""
|
||||
def __init__(self, persistence: PersistenceBackend, auto_persist: bool = True):
|
||||
self.persistence = persistence
|
||||
self.auto_persist = auto_persist
|
||||
# { node_type: { local_id: remote_id } }
|
||||
self._bindings: Dict[str, Dict[str, Optional[str]]] = {}
|
||||
self._deleted: Dict[str, set[str]] = {}
|
||||
|
||||
def _get_type_bindings(self, node_type: str) -> Dict[str, Optional[str]]:
|
||||
if node_type not in self._bindings:
|
||||
self._bindings[node_type] = {}
|
||||
return self._bindings[node_type]
|
||||
|
||||
def _get_deleted(self, node_type: str) -> set[str]:
|
||||
if node_type not in self._deleted:
|
||||
self._deleted[node_type] = set()
|
||||
return self._deleted[node_type]
|
||||
|
||||
async def bind(self, node_type: str, local_id: str, remote_id: Optional[str], manual: bool = False):
|
||||
"""
|
||||
建立本地节点与远程节点间的映射（带中文日志）
|
||||
|
||||
Args:
|
||||
node_type: 节点类型
|
||||
local_id: 本地节点 ID
|
||||
remote_id: 远程节点 ID
|
||||
manual: 是否为手工绑定（用于日志区分）
|
||||
"""
|
||||
bindings = self._get_type_bindings(node_type)
|
||||
bindings[local_id] = remote_id
|
||||
self._get_deleted(node_type).discard(local_id)
|
||||
|
||||
# 记录日志：手工绑定保留 info，自动绑定降为 debug 避免日志过量
|
||||
op_type = "手工" if manual else "自动"
|
||||
log_line = (
|
||||
f"[{op_type}绑定] 节点类型={node_type}, "
|
||||
f"本地节点={local_id} <-> 远程节点={remote_id}"
|
||||
)
|
||||
if manual:
|
||||
logger.info(log_line)
|
||||
else:
|
||||
logger.debug(log_line)
|
||||
|
||||
if self.auto_persist:
|
||||
await self.persistence.save_binding(node_type, local_id, {"remote_id": remote_id})
|
||||
|
||||
async def unbind(self, node_type: str, local_id: str, manual: bool = False):
|
||||
"""
|
||||
解除映射关系并物理删除（带中文日志）
|
||||
|
||||
Args:
|
||||
node_type: 节点类型
|
||||
local_id: 本地节点 ID
|
||||
manual: 是否为手工解绑（用于日志区分）
|
||||
"""
|
||||
bindings = self._get_type_bindings(node_type)
|
||||
remote_id = bindings.get(local_id)
|
||||
|
||||
if local_id in bindings:
|
||||
del bindings[local_id]
|
||||
|
||||
# 记录日志（中文，区分手工/自动）
|
||||
op_type = "手工" if manual else "自动"
|
||||
logger.info(
|
||||
f"[{op_type}解绑] 节点类型={node_type}, "
|
||||
f"本地节点={local_id} (原远程节点={remote_id})"
|
||||
)
|
||||
|
||||
if self.auto_persist:
|
||||
await self.persistence.delete_binding(node_type, local_id)
|
||||
else:
|
||||
self._get_deleted(node_type).add(local_id)
|
||||
|
||||
async def get_remote_id(self, node_type: str, local_id: str) -> Optional[str]:
|
||||
"""查询本地节点对应的远程 ID"""
|
||||
bindings = self._get_type_bindings(node_type)
|
||||
return bindings.get(local_id)
|
||||
|
||||
async def get_local_id(self, node_type: str, remote_id: str) -> Optional[str]:
|
||||
"""查询远程节点对应的本地 ID (内存遍历)"""
|
||||
bindings = self._get_type_bindings(node_type)
|
||||
for lid, rid in bindings.items():
|
||||
if rid == remote_id:
|
||||
return lid
|
||||
return None
|
||||
|
||||
async def get_record(self, node_type: str, local_id: str) -> Optional[BindingRecord]:
|
||||
"""获取结构化绑定记录"""
|
||||
remote_id = self._get_type_bindings(node_type).get(local_id)
|
||||
if remote_id is None:
|
||||
return None
|
||||
return BindingRecord(
|
||||
local_id=local_id,
|
||||
remote_id=remote_id
|
||||
)
|
||||
|
||||
async def get_all_records(self, node_type: str) -> List[BindingRecord]:
|
||||
"""获取指定类型的所有完整绑定记录 (结构化对象)"""
|
||||
bindings = self._get_type_bindings(node_type)
|
||||
return [BindingRecord(local_id=lid, remote_id=rid) for lid, rid in bindings.items()]
|
||||
|
||||
async def get_all_bindings(self, node_type: str) -> List[tuple[str, Optional[str]]]:
|
||||
"""获取指定类型的所有 ID 映射关系 (local_id, remote_id)"""
|
||||
bindings = self._get_type_bindings(node_type)
|
||||
return [(lid, rid) for lid, rid in bindings.items()]
|
||||
|
||||
async def load_from_persistence(self, node_type: str):
|
||||
"""从持久化层加载特定类型的绑定关系到内存"""
|
||||
raw_bindings = await self.persistence.load_bindings(node_type)
|
||||
bindings = self._get_type_bindings(node_type)
|
||||
self._get_deleted(node_type).clear()
|
||||
for rb in raw_bindings:
|
||||
payload = rb.get("payload") or {}
|
||||
bindings[rb["local_id"]] = payload.get("remote_id")
|
||||
|
||||
async def persist(self) -> None:
|
||||
"""将所有绑定关系写回持久化层"""
|
||||
if not self._bindings and not self._deleted:
|
||||
logger.info("🔍 [BindingManager.persist] 跳过：无绑定变更")
|
||||
return
|
||||
|
||||
type_count = 0
|
||||
saved_total = 0
|
||||
deleted_total = 0
|
||||
touched_types: List[str] = []
|
||||
|
||||
for node_type, bindings in self._bindings.items():
|
||||
deleted = list(self._get_deleted(node_type))
|
||||
binding_count = len(bindings)
|
||||
deleted_count = len(deleted)
|
||||
|
||||
type_count += 1
|
||||
saved_total += binding_count
|
||||
deleted_total += deleted_count
|
||||
if binding_count > 0 or deleted_count > 0:
|
||||
touched_types.append(node_type)
|
||||
|
||||
if deleted:
|
||||
await self.persistence.delete_bindings_bulk(node_type, deleted)
|
||||
self._get_deleted(node_type).clear()
|
||||
|
||||
if bindings:
|
||||
await self.persistence.save_bindings_bulk(node_type, bindings)
|
||||
|
||||
touched_text = ",".join(touched_types) if touched_types else "none"
|
||||
logger.info(
|
||||
f"🔍 [BindingManager.persist] 完成: types={type_count}, saved={saved_total}, deleted={deleted_total}, touched=[{touched_text}]"
|
||||
)
|
||||
@@ -0,0 +1,540 @@
|
||||
import uuid
|
||||
from enum import Enum
|
||||
from typing import Dict, List, Optional, Any, Set, Callable, Literal
|
||||
from .sync_node import SyncNode
|
||||
from .types import SyncStatus
|
||||
from .persistence import PersistenceBackend
|
||||
|
||||
|
||||
class DataCollection:
|
||||
"""
|
||||
DataCollection 管理一组同步节点的内存集合。
|
||||
支持快速遍历、CRUD 以及依赖检查。
|
||||
|
||||
唯一性保证：
|
||||
- node_id: 全局唯一，由 Collection.generate_node_id() 生成
|
||||
- data_id: 业务主键，允许空字符串但不允许重复（非空时）
|
||||
|
||||
node_id 生成策略：
|
||||
- 由 Collection.generate_node_id() 统一生成
|
||||
- Collection.add() 会检查 node_id 唯一性，重复时抛出异常
|
||||
"""
|
||||
def __init__(
|
||||
self,
|
||||
collection_id: str,
|
||||
persistence: Optional[PersistenceBackend] = None,
|
||||
auto_persist: bool = False,
|
||||
sm_runtime: Optional[Any] = None,
|
||||
):
|
||||
self.collection_id = collection_id
|
||||
self.persistence = persistence
|
||||
self.auto_persist = auto_persist
|
||||
self._sm_runtime = sm_runtime
|
||||
self._nodes: Dict[str, SyncNode] = {} # 内存缓存
|
||||
# data_id 全局唯一（按 collection 维度），用于 O(1) 反查 node_id
|
||||
self._data_id_to_node_id: Dict[str, str] = {}
|
||||
# auto_persist=False 时，delete() 不会立即落库；这里记录删除集合，persist() 时统一写回
|
||||
self._deleted_node_ids: set[str] = set()
|
||||
|
||||
@staticmethod
|
||||
def is_bootstrap_blocked(node: SyncNode) -> bool:
|
||||
"""是否为加载异常隔离节点（不参与后续自动流程）。"""
|
||||
context = node.context
|
||||
if not isinstance(context, dict):
|
||||
return False
|
||||
return bool(context.get("bootstrap_blocked"))
|
||||
|
||||
def set_state_machine_runtime(self, runtime: Any) -> None:
|
||||
"""设置 Collection 默认状态机 runtime。"""
|
||||
self._sm_runtime = runtime
|
||||
|
||||
async def add(self, node: SyncNode):
|
||||
# 检查 node_id 唯一性
|
||||
if node.node_id in self._nodes:
|
||||
raise ValueError(
|
||||
f"Duplicate node_id detected: {node.node_id} already exists in collection"
|
||||
)
|
||||
|
||||
# 检查 data_id 唯一性（允许空字符串）
|
||||
if node.data_id and node.data_id != "":
|
||||
existing_node_id = self._data_id_to_node_id.get(node.data_id)
|
||||
if existing_node_id is not None:
|
||||
raise ValueError(
|
||||
f"Duplicate data_id detected: {node.data_id} already bound to node_id={existing_node_id}"
|
||||
)
|
||||
self._data_id_to_node_id[node.data_id] = node.node_id
|
||||
|
||||
self._nodes[node.node_id] = node
|
||||
self._deleted_node_ids.discard(node.node_id)
|
||||
if self.persistence and self.auto_persist:
|
||||
await self.persistence.save_node(self.collection_id, node)
|
||||
|
||||
async def update(self, node: SyncNode):
|
||||
old_node = self._nodes.get(node.node_id)
|
||||
if old_node and old_node.data_id and old_node.data_id != "" and old_node.data_id != node.data_id:
|
||||
# node_id 不变但 data_id 变更，清理旧索引
|
||||
self._data_id_to_node_id.pop(old_node.data_id, None)
|
||||
|
||||
if node.data_id and node.data_id != "":
|
||||
existing_node_id = self._data_id_to_node_id.get(node.data_id)
|
||||
if existing_node_id is not None and existing_node_id != node.node_id:
|
||||
raise ValueError(
|
||||
f"Duplicate data_id detected: {node.data_id} already bound to node_id={existing_node_id}"
|
||||
)
|
||||
self._data_id_to_node_id[node.data_id] = node.node_id
|
||||
self._nodes[node.node_id] = node
|
||||
self._deleted_node_ids.discard(node.node_id)
|
||||
if self.persistence and self.auto_persist:
|
||||
await self.persistence.save_node(self.collection_id, node)
|
||||
|
||||
async def delete(self, node_id: str):
|
||||
old_node = self._nodes.get(node_id)
|
||||
if old_node and old_node.data_id and old_node.data_id != "":
|
||||
self._data_id_to_node_id.pop(old_node.data_id, None)
|
||||
|
||||
if node_id in self._nodes:
|
||||
del self._nodes[node_id]
|
||||
if self.persistence:
|
||||
if self.auto_persist:
|
||||
await self.persistence.delete_node(self.collection_id, node_id)
|
||||
else:
|
||||
self._deleted_node_ids.add(node_id)
|
||||
|
||||
def get(self, node_id: str) -> Optional[SyncNode]:
|
||||
return self._nodes.get(node_id)
|
||||
|
||||
def get_by_node_id(self, node_id: str) -> Optional[SyncNode]:
|
||||
"""别名方法，与 get() 功能相同"""
|
||||
return self.get(node_id)
|
||||
|
||||
def generate_node_id(self) -> str:
|
||||
"""生成唯一的 node_id"""
|
||||
return str(uuid.uuid4())
|
||||
|
||||
def get_node_id_by_data_id(self, data_id: str) -> Optional[str]:
|
||||
"""按 data_id 反查 node_id（O(1)）。"""
|
||||
return self._data_id_to_node_id.get(data_id)
|
||||
|
||||
def get_by_data_id_any(self, data_id: str) -> Optional[SyncNode]:
|
||||
"""按 data_id 查找节点（不要求 node_type）（O(1)）。"""
|
||||
node_id = self._data_id_to_node_id.get(data_id)
|
||||
if not node_id:
|
||||
return None
|
||||
return self._nodes.get(node_id)
|
||||
|
||||
def get_by_data_id(self, node_type: str, data_id: str) -> Optional[SyncNode]:
|
||||
"""按 node_type 和 data_id 查找节点"""
|
||||
node = self.get_by_data_id_any(data_id)
|
||||
if not node:
|
||||
return None
|
||||
return node if node.node_type == node_type else None
|
||||
|
||||
def filter(
|
||||
self,
|
||||
node_type: Optional[str] = None,
|
||||
filter: Optional[Dict[str, Any]] = None,
|
||||
node_filter: Optional[
|
||||
Dict[
|
||||
Literal[
|
||||
"node_id",
|
||||
"data_id",
|
||||
"depend_ids",
|
||||
"origin_data",
|
||||
"action",
|
||||
"status",
|
||||
"binding_status",
|
||||
"error",
|
||||
"context",
|
||||
"sync_log",
|
||||
"node_type",
|
||||
],
|
||||
Any,
|
||||
]
|
||||
| Callable[[SyncNode], bool]
|
||||
] = None
|
||||
) -> List[SyncNode]:
|
||||
"""
|
||||
获取节点列表。
|
||||
- node_type: 按 node_type 过滤。
|
||||
- filter: 按 node.data 中的属性过滤 (业务数据)。
|
||||
- node_filter: 按 SyncNode 顶级属性过滤
|
||||
- 如果是字典：按属性匹配 (如 {"status": SyncStatus.PENDING})
|
||||
- 如果是函数：自定义过滤函数 (如 lambda n: n.status == SyncStatus.PENDING)
|
||||
"""
|
||||
results = self._nodes.values()
|
||||
|
||||
# 1. 按 node_type 过滤
|
||||
if node_type:
|
||||
results = [n for n in results if n.node_type == node_type]
|
||||
else:
|
||||
results = list(results)
|
||||
|
||||
# 2. 按 node_filter 过滤 (顶级属性)
|
||||
if node_filter:
|
||||
if callable(node_filter):
|
||||
# 函数过滤器
|
||||
results = [n for n in results if node_filter(n)]
|
||||
else:
|
||||
filtered = []
|
||||
for node in results:
|
||||
match = True
|
||||
for k, v in node_filter.items():
|
||||
if k == "node_id":
|
||||
actual = node.node_id
|
||||
elif k == "data_id":
|
||||
actual = node.data_id
|
||||
elif k == "depend_ids":
|
||||
actual = node.depend_ids
|
||||
elif k == "origin_data":
|
||||
actual = node.origin_data
|
||||
elif k == "action":
|
||||
actual = node.action
|
||||
elif k == "status":
|
||||
actual = node.status
|
||||
elif k == "binding_status":
|
||||
actual = node.binding_status
|
||||
elif k == "error":
|
||||
actual = node.error
|
||||
elif k == "context":
|
||||
actual = node.context
|
||||
elif k == "sync_log":
|
||||
actual = node.sync_log
|
||||
elif k == "node_type":
|
||||
actual = node.node_type
|
||||
else:
|
||||
match = False
|
||||
break
|
||||
if actual != v:
|
||||
match = False
|
||||
break
|
||||
if match:
|
||||
filtered.append(node)
|
||||
results = filtered
|
||||
|
||||
# 3. 按 filter 过滤 (data 内部)
|
||||
if filter:
|
||||
filtered = []
|
||||
for node in results:
|
||||
node_data = node.get_data()
|
||||
if not node_data:
|
||||
match = False
|
||||
else:
|
||||
match = True
|
||||
for k, v in filter.items():
|
||||
actual_val = node_data.get(k)
|
||||
if actual_val != v:
|
||||
match = False
|
||||
break
|
||||
if match:
|
||||
filtered.append(node)
|
||||
results = filtered
|
||||
|
||||
return results
|
||||
|
||||
@staticmethod
|
||||
def _expected_matches_actual(expected: Any, actual: Any) -> bool:
|
||||
if isinstance(expected, list):
|
||||
return any(DataCollection._expected_matches_actual(item, actual) for item in expected)
|
||||
|
||||
if isinstance(expected, str):
|
||||
actual_candidates = set()
|
||||
if isinstance(actual, Enum):
|
||||
actual_candidates.update({str(actual.name), str(actual.value)})
|
||||
else:
|
||||
actual_candidates.add(str(actual))
|
||||
expected_lower = expected.lower()
|
||||
return any(str(candidate).lower() == expected_lower for candidate in actual_candidates)
|
||||
|
||||
return expected == actual
|
||||
|
||||
@classmethod
|
||||
def _node_matches_state_id(
|
||||
cls,
|
||||
runtime,
|
||||
*,
|
||||
node: SyncNode,
|
||||
state_id: str,
|
||||
action_hint: Optional[str] = None,
|
||||
match_fields: Optional[Set[str]] = None,
|
||||
) -> bool:
|
||||
states = runtime._config.raw.get("states")
|
||||
pseudo_states = runtime._config.raw.get("pseudo_states")
|
||||
if not isinstance(states, dict):
|
||||
raise RuntimeError("invalid state-machine config: missing states")
|
||||
if pseudo_states is not None and not isinstance(pseudo_states, dict):
|
||||
raise RuntimeError("invalid state-machine config: pseudo_states must be mapping")
|
||||
|
||||
state_def = states.get(state_id)
|
||||
if state_def is None and isinstance(pseudo_states, dict):
|
||||
state_def = pseudo_states.get(state_id)
|
||||
if not isinstance(state_def, dict):
|
||||
return False
|
||||
|
||||
snapshot = {
|
||||
"binding_status": node.binding_status,
|
||||
"action": action_hint if action_hint is not None else node.action,
|
||||
"status": node.status,
|
||||
"data_id_exist": bool(node.data_id),
|
||||
}
|
||||
selected_fields = match_fields or set(snapshot.keys())
|
||||
|
||||
for field, actual in snapshot.items():
|
||||
if field not in selected_fields:
|
||||
continue
|
||||
if field not in state_def:
|
||||
continue
|
||||
expected = state_def[field]
|
||||
if not cls._expected_matches_actual(expected, actual):
|
||||
return False
|
||||
return True
|
||||
|
||||
def filter_by_state_ids(
|
||||
self,
|
||||
runtime: Optional[Any] = None,
|
||||
*,
|
||||
state_ids: List[str],
|
||||
node_type: Optional[str] = None,
|
||||
include_bootstrap_blocked: bool = False,
|
||||
action_hint: Optional[str] = None,
|
||||
match_fields: Optional[Set[str]] = None,
|
||||
) -> List[SyncNode]:
|
||||
"""
|
||||
按状态机状态名（Sxx）筛选节点。
|
||||
|
||||
说明：
|
||||
- 状态定义来源于 `runtime` 关联的 YAML 配置；
|
||||
- 默认排除 `bootstrap_blocked` 隔离节点；
|
||||
- `action_hint` 可用于匹配类似 E40 场景的动作分支。
|
||||
"""
|
||||
runtime_obj = runtime or self._sm_runtime
|
||||
if runtime_obj is None:
|
||||
raise RuntimeError("state-machine runtime is not set for collection")
|
||||
|
||||
candidates = self.filter(node_type=node_type)
|
||||
matched: List[SyncNode] = []
|
||||
for node in candidates:
|
||||
if not include_bootstrap_blocked and self.is_bootstrap_blocked(node):
|
||||
continue
|
||||
if any(
|
||||
self._node_matches_state_id(
|
||||
runtime_obj,
|
||||
node=node,
|
||||
state_id=state_id,
|
||||
action_hint=action_hint,
|
||||
match_fields=match_fields,
|
||||
)
|
||||
for state_id in state_ids
|
||||
):
|
||||
matched.append(node)
|
||||
return matched
|
||||
|
||||
def check_depends(self, node_ids: List[str]) -> bool:
|
||||
"""
|
||||
检查指定的 node_id 列表是否存在，且状态是否不是 FAILED。
|
||||
|
||||
Args:
|
||||
node_ids: 依赖的 node_id 列表
|
||||
|
||||
Returns:
|
||||
bool: 所有依赖均满足则返回 True
|
||||
"""
|
||||
for nid in node_ids:
|
||||
node = self.get(nid)
|
||||
if not node:
|
||||
return False
|
||||
if node.status == SyncStatus.FAILED:
|
||||
return False
|
||||
return True
|
||||
|
||||
async def load_from_persistence(self):
|
||||
"""
|
||||
从后端加载数据到内存
|
||||
|
||||
使用 DomainRegistry 恢复正确的 SyncNode 类型。
|
||||
如果 node_type 未注册，会抛出 ValueError。
|
||||
|
||||
**加载阶段**：
|
||||
- 原样恢复持久化状态（binding_status/action/status/error）
|
||||
- 清空 sync_log（避免跨轮次累积日志）
|
||||
- data/origin_data 保留（由 DataSource 覆盖）
|
||||
|
||||
**初始化归并（E01）**：
|
||||
- 由 Pipeline 在加载后调用 run_reset() 触发 E01
|
||||
- CREATE 僵尸节点进入 S15 并删除
|
||||
- 其余节点进入 S00（UNCHECKED/NONE/PENDING）
|
||||
"""
|
||||
if not self.persistence:
|
||||
return
|
||||
|
||||
self._nodes = {}
|
||||
self._data_id_to_node_id = {}
|
||||
self._deleted_node_ids.clear()
|
||||
|
||||
node_dicts = await self.persistence.load_nodes(self.collection_id)
|
||||
|
||||
from .types import SyncAction, SyncStatus, BindingStatus
|
||||
from .registry import DomainRegistry
|
||||
|
||||
for node_dict in node_dicts:
|
||||
node_type = node_dict["node_type"]
|
||||
|
||||
# 从注册中心获取正确的 SyncNode 类（未注册会抛出 ValueError）
|
||||
node_class = DomainRegistry.get_node_class(node_type)
|
||||
assert issubclass(node_class, SyncNode)
|
||||
|
||||
# 严格恢复 action/status/binding_status；非法持久化值进入隔离态（S17 语义）
|
||||
parse_errors: list[str] = []
|
||||
|
||||
# 原样恢复 action 字段（可能是字符串或枚举）
|
||||
raw_action = node_dict.get("action", SyncAction.NONE)
|
||||
if isinstance(raw_action, str):
|
||||
try:
|
||||
action_value = SyncAction(raw_action)
|
||||
except ValueError:
|
||||
parse_errors.append(f"invalid action={raw_action}")
|
||||
action_value = SyncAction.NONE
|
||||
elif isinstance(raw_action, SyncAction):
|
||||
action_value = raw_action
|
||||
else:
|
||||
if raw_action is None:
|
||||
action_value = SyncAction.NONE
|
||||
else:
|
||||
parse_errors.append(f"invalid action type={type(raw_action).__name__}")
|
||||
action_value = SyncAction.NONE
|
||||
|
||||
# 原样恢复 status 字段（可能是字符串或枚举）
|
||||
raw_status = node_dict.get("status", SyncStatus.PENDING)
|
||||
if isinstance(raw_status, str):
|
||||
try:
|
||||
status_value = SyncStatus(raw_status)
|
||||
except ValueError:
|
||||
parse_errors.append(f"invalid status={raw_status}")
|
||||
status_value = SyncStatus.FAILED
|
||||
elif isinstance(raw_status, SyncStatus):
|
||||
status_value = raw_status
|
||||
else:
|
||||
if raw_status is None:
|
||||
status_value = SyncStatus.PENDING
|
||||
else:
|
||||
parse_errors.append(f"invalid status type={type(raw_status).__name__}")
|
||||
status_value = SyncStatus.FAILED
|
||||
|
||||
# 原样恢复 binding_status 字段（可能是字符串或枚举）
|
||||
raw_binding_status = node_dict.get("binding_status", BindingStatus.UNCHECKED)
|
||||
if isinstance(raw_binding_status, str):
|
||||
try:
|
||||
binding_status_value = BindingStatus(raw_binding_status)
|
||||
except ValueError:
|
||||
parse_errors.append(f"invalid binding_status={raw_binding_status}")
|
||||
binding_status_value = BindingStatus.ABNORMAL
|
||||
elif isinstance(raw_binding_status, BindingStatus):
|
||||
binding_status_value = raw_binding_status
|
||||
else:
|
||||
if raw_binding_status is None:
|
||||
binding_status_value = BindingStatus.UNCHECKED
|
||||
else:
|
||||
parse_errors.append(
|
||||
f"invalid binding_status type={type(raw_binding_status).__name__}"
|
||||
)
|
||||
binding_status_value = BindingStatus.ABNORMAL
|
||||
|
||||
raw_context = node_dict.get("context", {})
|
||||
context_value = raw_context if isinstance(raw_context, dict) else {}
|
||||
error_value = node_dict.get("error")
|
||||
|
||||
if parse_errors:
|
||||
context_value = dict(context_value)
|
||||
context_value["bootstrap_blocked"] = True
|
||||
context_value["bootstrap_block_reason"] = "invalid_persisted_enum"
|
||||
context_value["bootstrap_block_errors"] = parse_errors
|
||||
binding_status_value = BindingStatus.ABNORMAL
|
||||
action_value = SyncAction.NONE
|
||||
status_value = SyncStatus.FAILED
|
||||
details = "; ".join(parse_errors)
|
||||
error_value = f"LOAD_INVALID_ENUM: {details}"
|
||||
|
||||
# 使用正确的类型创建节点（保留持久化状态，后续由 E01 统一归并）
|
||||
try:
|
||||
node = node_class(
|
||||
node_id=node_dict["node_id"],
|
||||
data_id=node_dict.get("data_id", ""),
|
||||
depend_ids=node_dict.get("depend_ids", []),
|
||||
data=node_dict.get("data"), # 保留，由 DataSource 覆盖
|
||||
origin_data=node_dict.get("origin_data"), # 保留，由 DataSource 覆盖
|
||||
action=action_value,
|
||||
status=status_value,
|
||||
binding_status=binding_status_value,
|
||||
error=error_value,
|
||||
sync_log=None, # 初始化加载时清空上一轮流转日志
|
||||
context=context_value,
|
||||
)
|
||||
node.context["_loaded_from_persistence"] = True
|
||||
except Exception:
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.error(
|
||||
f"Load failed for node_type={node_type}, node_id={node_dict.get('node_id')}",
|
||||
exc_info=True
|
||||
)
|
||||
logger.error(f"Offending data: {node_dict.get('data')}")
|
||||
raise
|
||||
|
||||
self._nodes[node.node_id] = node
|
||||
|
||||
if node.data_id and node.data_id != "":
|
||||
existing_node_id = self._data_id_to_node_id.get(node.data_id)
|
||||
if existing_node_id is not None and existing_node_id != node.node_id:
|
||||
raise ValueError(
|
||||
f"Duplicate data_id detected while loading: {node.data_id} already bound to node_id={existing_node_id}"
|
||||
)
|
||||
self._data_id_to_node_id[node.data_id] = node.node_id
|
||||
|
||||
node_types = {node.node_type for node in self._nodes.values()}
|
||||
for node_type in node_types:
|
||||
override_map = await self.persistence.load_bind_data_id_overrides(node_type)
|
||||
for node in self.filter(node_type=node_type):
|
||||
bind_data_id = override_map.get(node.node_id)
|
||||
if bind_data_id:
|
||||
node.context["bind_data_id"] = bind_data_id
|
||||
else:
|
||||
node.context.pop("bind_data_id", None)
|
||||
|
||||
async def persist(self) -> None:
|
||||
"""将当前 collection 全量持久化到后端"""
|
||||
if not self.persistence:
|
||||
return
|
||||
|
||||
# 先落库删除（用于 auto_persist=False 的场景）
|
||||
if self._deleted_node_ids:
|
||||
await self.persistence.delete_nodes_bulk(self.collection_id, list(self._deleted_node_ids))
|
||||
self._deleted_node_ids.clear()
|
||||
|
||||
if not self._nodes:
|
||||
return
|
||||
|
||||
await self.persistence.save_nodes_bulk(self.collection_id, list(self._nodes.values()))
|
||||
|
||||
async def filter_by_project_ids(self, target_project_ids: List[str]) -> int:
|
||||
"""按目标项目集合过滤当前 collection，返回删除数量。"""
|
||||
target_set = set(target_project_ids or [])
|
||||
if not target_set:
|
||||
return 0
|
||||
|
||||
nodes = self.filter()
|
||||
deleted_count = 0
|
||||
for node in nodes:
|
||||
should_delete = False
|
||||
if node.node_type == "project":
|
||||
if node.data_id not in target_set:
|
||||
should_delete = True
|
||||
else:
|
||||
data = node.get_data() or {}
|
||||
project_id = data.get("project_id")
|
||||
if project_id and project_id not in target_set:
|
||||
should_delete = True
|
||||
|
||||
if should_delete:
|
||||
await self.delete(node.node_id)
|
||||
deleted_count += 1
|
||||
|
||||
return deleted_count
|
||||
@@ -0,0 +1,411 @@
|
||||
import aiosqlite
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
from typing import Optional, Dict, List, Any, TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .sync_node import SyncNode
|
||||
|
||||
class PersistenceBackend:
|
||||
"""
|
||||
Persistence 负责同步系统运行时数据的持久化存储（aiosqlite 实现）。
|
||||
主要作为 DataCollection 和 BindingManager 的后端存储。
|
||||
"""
|
||||
def __init__(self, db_path: str = ":memory:", backend: str = "sqlite"):
|
||||
self.db_path = db_path
|
||||
self.backend = backend
|
||||
self.conn: Optional[aiosqlite.Connection] = None
|
||||
|
||||
@staticmethod
|
||||
def _is_read_only_sql(sql: str) -> bool:
|
||||
stmt = (sql or "").strip().lower()
|
||||
return stmt.startswith("select") or stmt.startswith("pragma") or stmt.startswith("with")
|
||||
|
||||
@classmethod
|
||||
def query_records(
|
||||
cls,
|
||||
*,
|
||||
backend: str,
|
||||
db_path: str,
|
||||
sql: str,
|
||||
limit: int = 200,
|
||||
) -> tuple[List[str], List[Dict[str, Any]]]:
|
||||
if backend != "sqlite":
|
||||
raise NotImplementedError(f"records query backend not supported yet: {backend}")
|
||||
if not cls._is_read_only_sql(sql):
|
||||
raise ValueError("Only read-only SQL is allowed")
|
||||
|
||||
path = db_path
|
||||
if not os.path.exists(path):
|
||||
raise FileNotFoundError(f"DB not found: {path}")
|
||||
|
||||
with sqlite3.connect(path) as conn:
|
||||
conn.row_factory = sqlite3.Row
|
||||
cursor = conn.execute(sql)
|
||||
rows = cursor.fetchmany(max(1, min(limit, 2000)))
|
||||
columns = [item[0] for item in (cursor.description or [])]
|
||||
data = [{k: row[k] for k in columns} for row in rows]
|
||||
return columns, data
|
||||
|
||||
@classmethod
|
||||
def records_summary(
|
||||
cls,
|
||||
*,
|
||||
backend: str,
|
||||
db_path: str,
|
||||
) -> Dict[str, Any]:
|
||||
summary: Dict[str, Any] = {
|
||||
"backend": backend,
|
||||
"db_path": db_path,
|
||||
"db_exists": os.path.exists(db_path),
|
||||
"tables": [],
|
||||
"sample_bindings": [],
|
||||
}
|
||||
if not os.path.exists(db_path):
|
||||
return summary
|
||||
if backend != "sqlite":
|
||||
summary["error"] = f"records summary backend not supported yet: {backend}"
|
||||
return summary
|
||||
|
||||
with sqlite3.connect(db_path) as conn:
|
||||
conn.row_factory = sqlite3.Row
|
||||
tables = conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name"
|
||||
).fetchall()
|
||||
table_names = [str(r["name"]) for r in tables]
|
||||
|
||||
for name in table_names:
|
||||
count = conn.execute(f"SELECT COUNT(1) AS c FROM {name}").fetchone()["c"]
|
||||
summary["tables"].append({"name": name, "count": int(count)})
|
||||
|
||||
if "bindings" in table_names:
|
||||
sample = conn.execute(
|
||||
"SELECT node_type, local_id, remote_id FROM bindings ORDER BY last_updated DESC LIMIT 50"
|
||||
).fetchall()
|
||||
summary["sample_bindings"] = [
|
||||
{
|
||||
"node_type": row["node_type"],
|
||||
"local_id": row["local_id"],
|
||||
"remote_id": row["remote_id"],
|
||||
}
|
||||
for row in sample
|
||||
]
|
||||
return summary
|
||||
|
||||
async def initialize(self):
|
||||
"""异步初始化数据库连接和表结构"""
|
||||
if self.backend != "sqlite":
|
||||
raise NotImplementedError(f"Persistence backend not supported yet: {self.backend}")
|
||||
|
||||
# 确保目录存在
|
||||
if self.db_path != ":memory:":
|
||||
dir_path = os.path.dirname(self.db_path)
|
||||
if dir_path:
|
||||
os.makedirs(dir_path, exist_ok=True)
|
||||
|
||||
self.conn = await aiosqlite.connect(self.db_path)
|
||||
self.conn.row_factory = aiosqlite.Row
|
||||
await self._init_db()
|
||||
|
||||
async def _init_db(self):
|
||||
# 1. 节点表: 仅用于持久化存储 DataCollection 中的 SyncNode
|
||||
# 注意: depend_ids不持久化，每次从data实时计算
|
||||
await self.conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS nodes (
|
||||
collection_id TEXT NOT NULL,
|
||||
node_id TEXT NOT NULL,
|
||||
node_type TEXT NOT NULL,
|
||||
data_id TEXT,
|
||||
bind_data_id TEXT,
|
||||
data TEXT,
|
||||
origin_data TEXT,
|
||||
action TEXT,
|
||||
status TEXT,
|
||||
binding_status TEXT,
|
||||
error TEXT,
|
||||
sync_log TEXT,
|
||||
context TEXT,
|
||||
PRIMARY KEY (collection_id, node_id)
|
||||
)
|
||||
""")
|
||||
await self._ensure_nodes_schema()
|
||||
|
||||
# 2. 绑定表: 存储本地到远程的映射关系
|
||||
# 只保留 local_id / remote_id 两字段（remote_id 可空）
|
||||
await self._ensure_bindings_schema()
|
||||
await self.conn.commit()
|
||||
|
||||
async def _ensure_nodes_schema(self) -> None:
|
||||
cursor = await self.conn.execute("PRAGMA table_info(nodes)")
|
||||
rows = await cursor.fetchall()
|
||||
columns = [r[1] for r in rows] if rows else []
|
||||
if "bind_data_id" not in columns:
|
||||
await self.conn.execute("ALTER TABLE nodes ADD COLUMN bind_data_id TEXT")
|
||||
|
||||
async def _ensure_bindings_schema(self) -> None:
|
||||
"""确保 bindings 表为 (node_type, local_id, remote_id) 结构。
|
||||
|
||||
若检测到旧表含 payload 列，则进行迁移。
|
||||
"""
|
||||
cursor = await self.conn.execute("PRAGMA table_info(bindings)")
|
||||
rows = await cursor.fetchall()
|
||||
columns = [r[1] for r in rows] if rows else []
|
||||
|
||||
if not columns:
|
||||
await self.conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS bindings (
|
||||
node_type TEXT NOT NULL,
|
||||
local_id TEXT NOT NULL,
|
||||
remote_id TEXT,
|
||||
last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (node_type, local_id)
|
||||
)
|
||||
""")
|
||||
return
|
||||
|
||||
if "payload" in columns and "remote_id" not in columns:
|
||||
# 迁移旧结构 payload -> remote_id
|
||||
await self.conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS bindings_new (
|
||||
node_type TEXT NOT NULL,
|
||||
local_id TEXT NOT NULL,
|
||||
remote_id TEXT,
|
||||
last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (node_type, local_id)
|
||||
)
|
||||
""")
|
||||
|
||||
async with self.conn.execute("SELECT node_type, local_id, payload FROM bindings") as cur:
|
||||
old_rows = await cur.fetchall()
|
||||
for row in old_rows:
|
||||
try:
|
||||
payload_dict = json.loads(row[2]) if row[2] else {}
|
||||
except json.JSONDecodeError:
|
||||
payload_dict = {}
|
||||
remote_id = payload_dict.get("remote_id")
|
||||
await self.conn.execute(
|
||||
"INSERT OR REPLACE INTO bindings_new (node_type, local_id, remote_id) VALUES (?, ?, ?)",
|
||||
(row[0], row[1], remote_id),
|
||||
)
|
||||
|
||||
await self.conn.execute("DROP TABLE bindings")
|
||||
await self.conn.execute("ALTER TABLE bindings_new RENAME TO bindings")
|
||||
|
||||
# --- DataCollection 支持 ---
|
||||
|
||||
async def save_node(self, collection_id: str, node: "SyncNode"):
|
||||
"""保存或更新单个节点（depend_ids不持久化，运行时从data实时计算）"""
|
||||
bind_data_id = None
|
||||
if isinstance(node.context, dict):
|
||||
value = node.context.get("bind_data_id")
|
||||
if value is not None and value != "":
|
||||
bind_data_id = str(value)
|
||||
await self.conn.execute("""
|
||||
INSERT OR REPLACE INTO nodes (
|
||||
collection_id, node_id, node_type, data_id, bind_data_id,
|
||||
data, origin_data, action, status, binding_status, error, sync_log, context
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
collection_id,
|
||||
node.node_id,
|
||||
node.node_type,
|
||||
node.data_id,
|
||||
bind_data_id,
|
||||
json.dumps(node.data) if node.data else None,
|
||||
json.dumps(node.origin_data) if node.origin_data else None,
|
||||
node.action.value,
|
||||
node.status.value,
|
||||
node.binding_status.value,
|
||||
node.error,
|
||||
node.sync_log,
|
||||
json.dumps(node.context) if node.context else None
|
||||
))
|
||||
await self.conn.commit()
|
||||
|
||||
async def save_nodes_bulk(self, collection_id: str, nodes: List["SyncNode"]):
|
||||
"""批量保存或更新节点 (单次提交，depend_ids不持久化)"""
|
||||
if not nodes:
|
||||
return
|
||||
rows = []
|
||||
for node in nodes:
|
||||
bind_data_id = None
|
||||
if isinstance(node.context, dict):
|
||||
value = node.context.get("bind_data_id")
|
||||
if value is not None and value != "":
|
||||
bind_data_id = str(value)
|
||||
rows.append((
|
||||
collection_id,
|
||||
node.node_id,
|
||||
node.node_type,
|
||||
node.data_id,
|
||||
bind_data_id,
|
||||
json.dumps(node.data) if node.data else None,
|
||||
json.dumps(node.origin_data) if node.origin_data else None,
|
||||
node.action.value,
|
||||
node.status.value,
|
||||
node.binding_status.value,
|
||||
node.error,
|
||||
node.sync_log,
|
||||
json.dumps(node.context) if node.context else None
|
||||
))
|
||||
|
||||
await self.conn.executemany("""
|
||||
INSERT OR REPLACE INTO nodes (
|
||||
collection_id, node_id, node_type, data_id, bind_data_id,
|
||||
data, origin_data, action, status, binding_status, error, sync_log, context
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", rows)
|
||||
await self.conn.commit()
|
||||
|
||||
async def delete_node(self, collection_id: str, node_id: str):
|
||||
"""删除单个节点"""
|
||||
await self.conn.execute("DELETE FROM nodes WHERE collection_id = ? AND node_id = ?", (collection_id, node_id))
|
||||
await self.conn.commit()
|
||||
|
||||
async def delete_nodes_bulk(self, collection_id: str, node_ids: List[str]):
|
||||
"""批量删除节点 (单次提交)"""
|
||||
if not node_ids:
|
||||
return
|
||||
rows = [(collection_id, node_id) for node_id in node_ids]
|
||||
await self.conn.executemany(
|
||||
"DELETE FROM nodes WHERE collection_id = ? AND node_id = ?",
|
||||
rows,
|
||||
)
|
||||
await self.conn.commit()
|
||||
|
||||
async def load_nodes(self, collection_id: str) -> List[Dict[str, Any]]:
|
||||
"""从特定 collection 中加载所有节点（depend_ids初始化为空，稍后从data实时计算）"""
|
||||
async with self.conn.execute("SELECT * FROM nodes WHERE collection_id = ?", (collection_id,)) as cursor:
|
||||
rows = await cursor.fetchall()
|
||||
results = []
|
||||
for row in rows:
|
||||
results.append({
|
||||
"node_id": row["node_id"],
|
||||
"node_type": row["node_type"],
|
||||
"data_id": row["data_id"],
|
||||
"bind_data_id": row["bind_data_id"],
|
||||
"depend_ids": [], # 不从数据库加载，运行时从data实时计算
|
||||
"data": json.loads(row["data"]) if row["data"] else None,
|
||||
"origin_data": json.loads(row["origin_data"]) if row["origin_data"] else None,
|
||||
"action": row["action"],
|
||||
"status": row["status"],
|
||||
"binding_status": row["binding_status"],
|
||||
"error": row["error"],
|
||||
"sync_log": row["sync_log"],
|
||||
"context": json.loads(row["context"]) if row["context"] else {}
|
||||
})
|
||||
if results[-1]["bind_data_id"] and isinstance(results[-1]["context"], dict):
|
||||
results[-1]["context"]["bind_data_id"] = results[-1]["bind_data_id"]
|
||||
return results
|
||||
|
||||
# --- BindingManager 支持 ---
|
||||
|
||||
async def save_binding(self, node_type: str, local_id: str, payload_dict: Dict[str, Any]):
|
||||
"""保存或更新绑定关系及其元数据"""
|
||||
remote_id = payload_dict.get("remote_id") if payload_dict else None
|
||||
await self.conn.execute("""
|
||||
INSERT OR REPLACE INTO bindings (node_type, local_id, remote_id)
|
||||
VALUES (?, ?, ?)
|
||||
""", (node_type, local_id, remote_id))
|
||||
await self.conn.commit()
|
||||
|
||||
async def save_bindings_bulk(self, node_type: str, bindings: Dict[str, Optional[str]]):
|
||||
"""批量保存或更新绑定关系 (单次提交)"""
|
||||
if not bindings:
|
||||
return
|
||||
rows = []
|
||||
for local_id, remote_id in bindings.items():
|
||||
rows.append((node_type, local_id, remote_id))
|
||||
await self.conn.executemany(
|
||||
"INSERT OR REPLACE INTO bindings (node_type, local_id, remote_id) VALUES (?, ?, ?)",
|
||||
rows,
|
||||
)
|
||||
await self.conn.commit()
|
||||
|
||||
async def delete_binding(self, node_type: str, local_id: str):
|
||||
"""删除绑定关系"""
|
||||
await self.conn.execute(
|
||||
"DELETE FROM bindings WHERE node_type = ? AND local_id = ?",
|
||||
(node_type, local_id)
|
||||
)
|
||||
await self.conn.commit()
|
||||
|
||||
async def delete_bindings_bulk(self, node_type: str, local_ids: List[str]):
|
||||
"""批量删除绑定关系 (单次提交)"""
|
||||
if not local_ids:
|
||||
return
|
||||
rows = [(node_type, local_id) for local_id in local_ids]
|
||||
await self.conn.executemany(
|
||||
"DELETE FROM bindings WHERE node_type = ? AND local_id = ?",
|
||||
rows,
|
||||
)
|
||||
await self.conn.commit()
|
||||
|
||||
async def load_bindings(self, node_type: str) -> List[Dict[str, Any]]:
|
||||
"""回写该类型下的所有绑定关系到内存"""
|
||||
async with self.conn.execute(
|
||||
"SELECT local_id, remote_id FROM bindings WHERE node_type = ?",
|
||||
(node_type,)
|
||||
) as cursor:
|
||||
rows = await cursor.fetchall()
|
||||
return [
|
||||
{"local_id": r["local_id"], "payload": {"remote_id": r["remote_id"]}}
|
||||
for r in rows
|
||||
]
|
||||
|
||||
async def load_bind_data_id_overrides(self, node_type: str) -> Dict[str, Optional[str]]:
|
||||
"""基于 bindings 表计算 node_id -> bind_data_id 覆盖映射。
|
||||
|
||||
规则：
|
||||
- local 节点的 bind_data_id 取其绑定 remote 节点的 data_id
|
||||
- remote 节点的 bind_data_id 取其绑定 local 节点的 data_id
|
||||
"""
|
||||
sql = """
|
||||
SELECT b.local_id AS src_node_id, rn.data_id AS bind_data_id
|
||||
FROM bindings b
|
||||
LEFT JOIN nodes rn
|
||||
ON rn.node_id = b.remote_id
|
||||
AND rn.node_type = b.node_type
|
||||
WHERE b.node_type = ?
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT b.remote_id AS src_node_id, ln.data_id AS bind_data_id
|
||||
FROM bindings b
|
||||
LEFT JOIN nodes ln
|
||||
ON ln.node_id = b.local_id
|
||||
AND ln.node_type = b.node_type
|
||||
WHERE b.node_type = ?
|
||||
AND b.remote_id IS NOT NULL
|
||||
"""
|
||||
async with self.conn.execute(sql, (node_type, node_type)) as cursor:
|
||||
rows = await cursor.fetchall()
|
||||
result: Dict[str, Optional[str]] = {}
|
||||
for row in rows:
|
||||
src_node_id = row["src_node_id"]
|
||||
if not src_node_id:
|
||||
continue
|
||||
result[str(src_node_id)] = row["bind_data_id"]
|
||||
return result
|
||||
|
||||
async def dump_to_runtime(self, filename: str):
|
||||
"""将当前状态转储到 _runtime 文件夹 (使用 VACUUM INTO)"""
|
||||
runtime_path = os.path.join("_runtime", filename)
|
||||
if os.path.exists(runtime_path):
|
||||
os.remove(runtime_path)
|
||||
|
||||
# VACUUM INTO 是备份 SQLite 的推荐方式 (SQLite 3.27+)
|
||||
await self.conn.execute("VACUUM INTO ?", (runtime_path,))
|
||||
|
||||
async def close(self):
|
||||
"""关闭数据库连接（包含异常处理）"""
|
||||
if self.conn:
|
||||
try:
|
||||
await self.conn.close()
|
||||
except Exception as e:
|
||||
print(f"⚠️ Error closing persistence: {e}")
|
||||
finally:
|
||||
self.conn = None
|
||||
@@ -0,0 +1,267 @@
|
||||
"""
|
||||
Domain 注册中心
|
||||
|
||||
提供统一的 domain 组件注册和查询接口。
|
||||
各模块（Collection/DataSource/Strategy）按需获取。
|
||||
|
||||
使用方式：
|
||||
1. 在 domain/__init__.py 中注册所有组件
|
||||
2. 各模块按需查询
|
||||
|
||||
示例：
|
||||
# 注册
|
||||
from ..common.registry import DomainRegistry
|
||||
from .contract import ContractSyncNode, ContractJsonlHandler
|
||||
|
||||
DomainRegistry.register(
|
||||
node_type="contract",
|
||||
schema=ContractResponse,
|
||||
node_class=ContractSyncNode,
|
||||
jsonl_handler_class=ContractJsonlHandler
|
||||
)
|
||||
|
||||
# Collection 查询 SyncNode 类
|
||||
node_class = DomainRegistry.get_node_class("contract")
|
||||
|
||||
# DataSource 查询 Handler
|
||||
handler_class = DomainRegistry.get_jsonl_handler("contract")
|
||||
"""
|
||||
from typing import Dict, Type, Optional, Any
|
||||
from dataclasses import dataclass, field
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
@dataclass
|
||||
class DomainRegistration:
|
||||
"""
|
||||
单个 domain 的注册信息
|
||||
|
||||
包含：
|
||||
- node_class: SyncNode 子类（用于 Collection 恢复）
|
||||
- strategy_class: Strategy 子类（用于 SyncSystem 初始化）
|
||||
- handlers: 各种 Handler（用于 DataSource 初始化）
|
||||
"""
|
||||
node_type: str
|
||||
schema: Type[BaseModel]
|
||||
node_class: Type[Any] # Type[SyncNode[T]]，避免循环导入
|
||||
strategy_class: Optional[Type[Any]] = None # 可选，部分 domain 可能只读
|
||||
jsonl_handler_class: Optional[Type[Any]] = None
|
||||
api_handler_class: Optional[Type[Any]] = None
|
||||
db_handler_class: Optional[Type[Any]] = None
|
||||
metadata: Dict[str, Any] = field(default_factory=dict) # 额外元数据
|
||||
|
||||
|
||||
class DomainRegistry:
|
||||
"""
|
||||
Domain 注册中心
|
||||
|
||||
职责分离：
|
||||
- Collection: 只关心 SyncNode 类（恢复类型）
|
||||
- DataSource: 只关心 Handler 类（执行层）
|
||||
- Strategy: 只关心 Strategy 类（业务逻辑）
|
||||
|
||||
每个 domain 只注册一次，各模块按需获取。
|
||||
"""
|
||||
|
||||
_registry: Dict[str, DomainRegistration] = {}
|
||||
|
||||
@classmethod
|
||||
def register(
|
||||
cls,
|
||||
node_type: str,
|
||||
schema: Type[BaseModel],
|
||||
node_class: Type[Any],
|
||||
strategy_class: Optional[Type[Any]] = None,
|
||||
jsonl_handler_class: Optional[Type[Any]] = None,
|
||||
api_handler_class: Optional[Type[Any]] = None,
|
||||
db_handler_class: Optional[Type[Any]] = None,
|
||||
**metadata
|
||||
) -> None:
|
||||
"""
|
||||
注册一个 domain
|
||||
|
||||
Args:
|
||||
node_type: 节点类型（如 "contract"）
|
||||
schema: Pydantic Schema 类
|
||||
node_class: SyncNode 子类
|
||||
strategy_class: Strategy 子类（可选）
|
||||
jsonl_handler_class: JSONL Handler 类（可选）
|
||||
api_handler_class: API Handler 类（可选）
|
||||
db_handler_class: DB Handler 类（可选）
|
||||
**metadata: 额外的元数据
|
||||
|
||||
Raises:
|
||||
ValueError: 如果 node_type 已注册
|
||||
"""
|
||||
if node_type in cls._registry:
|
||||
raise ValueError(f"Domain '{node_type}' already registered")
|
||||
|
||||
cls._registry[node_type] = DomainRegistration(
|
||||
node_type=node_type,
|
||||
schema=schema,
|
||||
node_class=node_class,
|
||||
strategy_class=strategy_class,
|
||||
jsonl_handler_class=jsonl_handler_class,
|
||||
api_handler_class=api_handler_class,
|
||||
db_handler_class=db_handler_class,
|
||||
metadata=metadata
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_node_class(cls, node_type: str) -> Type[Any]:
|
||||
"""
|
||||
获取 SyncNode 类（用于 Collection 恢复）
|
||||
|
||||
Args:
|
||||
node_type: 节点类型
|
||||
|
||||
Returns:
|
||||
SyncNode 子类
|
||||
|
||||
Raises:
|
||||
ValueError: 如果 node_type 未注册
|
||||
"""
|
||||
reg = cls._registry.get(node_type)
|
||||
if not reg:
|
||||
raise ValueError(f"Unknown node_type: '{node_type}'. Available: {list(cls._registry.keys())}")
|
||||
return reg.node_class
|
||||
|
||||
@classmethod
|
||||
def get_strategy_class(cls, node_type: str) -> Optional[Type[Any]]:
|
||||
"""
|
||||
获取 Strategy 类（用于 SyncSystem）
|
||||
|
||||
Args:
|
||||
node_type: 节点类型
|
||||
|
||||
Returns:
|
||||
Strategy 类，如果未注册则返回 None
|
||||
"""
|
||||
reg = cls._registry.get(node_type)
|
||||
if not reg:
|
||||
raise ValueError(f"Unknown node_type: '{node_type}'")
|
||||
return reg.strategy_class
|
||||
|
||||
@classmethod
|
||||
def get_jsonl_handler(cls, node_type: str) -> Optional[Type[Any]]:
|
||||
"""
|
||||
获取 JSONL Handler 类（用于 DataSource）
|
||||
|
||||
Args:
|
||||
node_type: 节点类型
|
||||
|
||||
Returns:
|
||||
Handler 类，如果未注册则返回 None
|
||||
"""
|
||||
reg = cls._registry.get(node_type)
|
||||
if not reg:
|
||||
raise ValueError(f"Unknown node_type: '{node_type}'")
|
||||
return reg.jsonl_handler_class
|
||||
|
||||
@classmethod
|
||||
def get_api_handler(cls, node_type: str) -> Optional[Type[Any]]:
|
||||
"""获取 API Handler 类"""
|
||||
reg = cls._registry.get(node_type)
|
||||
if not reg:
|
||||
raise ValueError(f"Unknown node_type: '{node_type}'")
|
||||
return reg.api_handler_class
|
||||
|
||||
@classmethod
|
||||
def get_db_handler(cls, node_type: str) -> Optional[Type[Any]]:
|
||||
"""获取 DB Handler 类"""
|
||||
reg = cls._registry.get(node_type)
|
||||
if not reg:
|
||||
raise ValueError(f"Unknown node_type: '{node_type}'")
|
||||
return reg.db_handler_class
|
||||
|
||||
@classmethod
|
||||
def get_schema(cls, node_type: str) -> Type[BaseModel]:
|
||||
"""获取 Schema 类"""
|
||||
reg = cls._registry.get(node_type)
|
||||
if not reg:
|
||||
raise ValueError(f"Unknown node_type: '{node_type}'")
|
||||
return reg.schema
|
||||
|
||||
@classmethod
|
||||
def get_registration(cls, node_type: str) -> DomainRegistration:
|
||||
"""获取完整的注册信息"""
|
||||
reg = cls._registry.get(node_type)
|
||||
if not reg:
|
||||
raise ValueError(f"Unknown node_type: '{node_type}'")
|
||||
return reg
|
||||
|
||||
@classmethod
|
||||
def is_registered(cls, node_type: str) -> bool:
|
||||
"""检查 node_type 是否已注册"""
|
||||
return node_type in cls._registry
|
||||
|
||||
@classmethod
|
||||
def list_registered(cls) -> list[str]:
|
||||
"""列出所有已注册的 node_type"""
|
||||
return list(cls._registry.keys())
|
||||
|
||||
@classmethod
|
||||
def get_registered_types(cls) -> list[str]:
|
||||
"""列出所有已注册的 node_type（list_registered 的别名）"""
|
||||
return cls.list_registered()
|
||||
|
||||
@classmethod
|
||||
def clear(cls) -> None:
|
||||
"""清空注册表（仅用于测试）"""
|
||||
cls._registry.clear()
|
||||
|
||||
@classmethod
|
||||
def print_registry(cls) -> None:
|
||||
"""打印所有注册的 domain 信息（用于调试）"""
|
||||
if not cls._registry:
|
||||
print("⚠️ DomainRegistry is empty - no domains registered")
|
||||
return
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print(f"📋 DomainRegistry - {len(cls._registry)} domain(s) registered")
|
||||
print(f"{'='*80}\n")
|
||||
|
||||
for node_type, reg in sorted(cls._registry.items()):
|
||||
print(f"🔹 {node_type}")
|
||||
print(f" Schema: {reg.schema.__name__}")
|
||||
print(f" Node Class: {reg.node_class.__name__}")
|
||||
print(f" Strategy Class: {reg.strategy_class.__name__ if reg.strategy_class else '❌ None'}")
|
||||
print(f" JSONL Handler: {reg.jsonl_handler_class.__name__ if reg.jsonl_handler_class else '❌ None'}")
|
||||
print(f" API Handler: {reg.api_handler_class.__name__ if reg.api_handler_class else '❌ None'}")
|
||||
print(f" DB Handler: {reg.db_handler_class.__name__ if reg.db_handler_class else '❌ None'}")
|
||||
if reg.metadata:
|
||||
print(f" Metadata: {reg.metadata}")
|
||||
print()
|
||||
|
||||
print(f"{'='*80}\n")
|
||||
|
||||
@classmethod
|
||||
def get_summary(cls) -> Dict[str, Dict[str, bool]]:
|
||||
"""
|
||||
获取注册摘要（用于自动化检查）
|
||||
|
||||
Returns:
|
||||
字典，key 为 node_type，value 为各组件的注册状态
|
||||
|
||||
Example:
|
||||
{
|
||||
"contract": {
|
||||
"has_node": True,
|
||||
"has_strategy": True,
|
||||
"has_jsonl_handler": True,
|
||||
"has_api_handler": True,
|
||||
"has_db_handler": False
|
||||
},
|
||||
...
|
||||
}
|
||||
"""
|
||||
summary = {}
|
||||
for node_type, reg in cls._registry.items():
|
||||
summary[node_type] = {
|
||||
"has_node": reg.node_class is not None,
|
||||
"has_strategy": reg.strategy_class is not None,
|
||||
"has_jsonl_handler": reg.jsonl_handler_class is not None,
|
||||
"has_api_handler": reg.api_handler_class is not None,
|
||||
"has_db_handler": reg.db_handler_class is not None,
|
||||
}
|
||||
return summary
|
||||
@@ -0,0 +1,435 @@
|
||||
"""
|
||||
SyncNode - 核心状态机容器
|
||||
|
||||
将data存储为Dict，但支持schema验证和Pydantic模型转换。
|
||||
提供统一的数据访问接口：get_data() / put_data()
|
||||
"""
|
||||
|
||||
import copy
|
||||
from contextlib import contextmanager
|
||||
from typing import TypeVar, Generic, Optional, Any, Dict, List, Type
|
||||
from pydantic import BaseModel, ValidationError
|
||||
|
||||
from .types import SyncAction, SyncStatus, BindingStatus
|
||||
|
||||
T = TypeVar("T", bound=BaseModel)
|
||||
|
||||
|
||||
class SyncNode(Generic[T]):
|
||||
"""
|
||||
核心状态机容器
|
||||
|
||||
数据存储策略：
|
||||
- origin_data/data 内部存储为 Dict
|
||||
- 通过 get_data() 获取数据副本（深拷贝）
|
||||
- 通过 set_data() 设置数据（自动验证+深拷贝）
|
||||
- 通过 to_model() 转换为 Pydantic 模型实例
|
||||
|
||||
基类 SyncNode 不应直接实例化，应通过子类使用。
|
||||
子类应设置类变量 node_type 和 schema：
|
||||
```python
|
||||
class ContractSyncNode(SyncNode[ContractResponse]):
|
||||
node_type = "contract"
|
||||
schema = ContractResponse
|
||||
```
|
||||
"""
|
||||
|
||||
# 类变量：子类必须设置
|
||||
node_type: str = None # type: ignore
|
||||
schema: Type[T] = None # type: ignore
|
||||
_CORE_FIELDS = {"binding_status", "action", "status", "data_id"}
|
||||
|
||||
def __setattr__(self, name: str, value: Any) -> None:
|
||||
if (
|
||||
name in self._CORE_FIELDS
|
||||
and self.__dict__.get("_core_guard_enabled", False)
|
||||
and self.__dict__.get("_core_write_depth", 0) <= 0
|
||||
):
|
||||
raise RuntimeError(
|
||||
f"Core field '{name}' can only be modified via event apply path for node={self.__dict__.get('node_id', '<uninitialized>')}"
|
||||
)
|
||||
object.__setattr__(self, name, value)
|
||||
|
||||
@contextmanager
|
||||
def allow_core_state_write(self, source: str = "unknown"):
|
||||
depth = self.__dict__.get("_core_write_depth", 0)
|
||||
prev_source = self.__dict__.get("_core_write_source", "")
|
||||
object.__setattr__(self, "_core_write_depth", depth + 1)
|
||||
object.__setattr__(self, "_core_write_source", source)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
object.__setattr__(self, "_core_write_depth", depth)
|
||||
object.__setattr__(self, "_core_write_source", prev_source)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
node_id: str,
|
||||
data_id: str = "",
|
||||
depend_ids: Optional[List[str]] = None,
|
||||
data: Optional[Dict[str, Any]] = None,
|
||||
origin_data: Optional[Dict[str, Any]] = None,
|
||||
action: SyncAction = SyncAction.NONE,
|
||||
status: SyncStatus = SyncStatus.PENDING,
|
||||
binding_status: BindingStatus = BindingStatus.UNCHECKED,
|
||||
error: Optional[str] = None,
|
||||
context: Optional[Dict[str, Any]] = None,
|
||||
sync_log: Optional[str] = None
|
||||
):
|
||||
object.__setattr__(self, "_core_guard_enabled", False)
|
||||
object.__setattr__(self, "_core_write_depth", 0)
|
||||
|
||||
# 检查类变量是否设置
|
||||
if self.node_type is None:
|
||||
raise ValueError(
|
||||
f"{self.__class__.__name__} must define class variable 'node_type'. "
|
||||
f"Example: node_type = 'contract'"
|
||||
)
|
||||
if self.schema is None:
|
||||
raise ValueError(
|
||||
f"{self.__class__.__name__} must define class variable 'schema'. "
|
||||
f"Example: schema = YourSchema"
|
||||
)
|
||||
|
||||
self.node_id = node_id
|
||||
self.data_id = data_id
|
||||
self.depend_ids = depend_ids if depend_ids is not None else []
|
||||
|
||||
if origin_data is not None:
|
||||
self.origin_data = copy.deepcopy(origin_data)
|
||||
else:
|
||||
self.origin_data = None
|
||||
self.action = action
|
||||
self.status = status
|
||||
self.binding_status = binding_status
|
||||
self.error = error
|
||||
self.sync_log = sync_log # Debug/info级别的日志信息
|
||||
|
||||
# Context 字典：存储额外的上下文信息（如 project_id, contract_id 等）
|
||||
self.context: Dict[str, Any] = context if context is not None else {}
|
||||
|
||||
# 在初始化时验证并设置 data
|
||||
if data is not None:
|
||||
self._validate_and_set_data(data)
|
||||
if self.origin_data is None:
|
||||
self.origin_data = copy.deepcopy(self.data)
|
||||
else:
|
||||
self.data = None
|
||||
|
||||
object.__setattr__(self, "_core_guard_enabled", True)
|
||||
|
||||
def _validate_and_set_data(self, data: Dict[str, Any]) -> None:
|
||||
"""验证并设置数据（内部方法）"""
|
||||
if not isinstance(data, dict):
|
||||
self.data = None
|
||||
raise ValueError(f"Data validation failed for {self.node_type}: data must be a dict")
|
||||
|
||||
try:
|
||||
# 基础兼容性：将 _id 转换为 id
|
||||
data_to_validate = data.copy()
|
||||
if "id" not in data_to_validate and "_id" in data_to_validate:
|
||||
data_to_validate["id"] = data_to_validate["_id"]
|
||||
|
||||
model = self.schema.model_validate(data_to_validate)
|
||||
# 使用 model_dump() 获得完整校验后的数据，去掉 schema 之外的字段
|
||||
self.data = model.model_dump()
|
||||
except Exception as e:
|
||||
# 如果出错就存 None, 然后 error 里加个错
|
||||
self.data = None
|
||||
self.error = f"Validation failed: {str(e)}"
|
||||
# 不要轻易降级错误。如果校验失败，应抛出 ValueError 供上层处理
|
||||
raise ValueError(f"Data validation failed for {self.node_type}: {e}")
|
||||
|
||||
# ===== 数据访问方法 =====
|
||||
|
||||
def get_data(self, exclude_unset: bool = False) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
获取数据的副本。
|
||||
|
||||
Args:
|
||||
exclude_unset: 是否排除未设置（默认值）的字段。
|
||||
|
||||
Returns:
|
||||
数据字典的副本，如果为None则返回None
|
||||
"""
|
||||
if self.data is None:
|
||||
return None
|
||||
|
||||
if not exclude_unset:
|
||||
return copy.deepcopy(self.data)
|
||||
|
||||
try:
|
||||
model = self.schema.model_validate(self.data)
|
||||
return model.model_dump(exclude_unset=True)
|
||||
except Exception:
|
||||
return copy.deepcopy(self.data)
|
||||
|
||||
def set_data(self, data: Optional[Dict[str, Any]], validate: bool = True) -> None:
|
||||
"""
|
||||
设置数据（自动深拷贝）。
|
||||
|
||||
Args:
|
||||
data: 要设置的数据字典
|
||||
validate: 是否进行schema验证（默认True）
|
||||
|
||||
Raises:
|
||||
ValueError: 如果数据验证失败
|
||||
"""
|
||||
if data is None:
|
||||
self.data = None
|
||||
return
|
||||
|
||||
if validate:
|
||||
self._validate_and_set_data(data)
|
||||
else:
|
||||
self.data = copy.deepcopy(data)
|
||||
|
||||
def get_origin_data(self) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
获取原始数据的深拷贝副本。
|
||||
|
||||
Returns:
|
||||
原始数据字典的副本，如果为None则返回None
|
||||
"""
|
||||
if self.origin_data is None:
|
||||
return None
|
||||
return copy.deepcopy(self.origin_data)
|
||||
|
||||
def set_origin_data(self, data: Optional[Dict[str, Any]]) -> None:
|
||||
"""
|
||||
设置原始数据（自动深拷贝）。
|
||||
|
||||
Args:
|
||||
data: 要设置的原始数据字典
|
||||
"""
|
||||
if data is None:
|
||||
self.origin_data = None
|
||||
return
|
||||
|
||||
if self.schema is None:
|
||||
raise ValueError(f"Cannot set origin_data without schema for {self.node_type}")
|
||||
|
||||
try:
|
||||
model = self.schema.model_validate(copy.deepcopy(data))
|
||||
# 使用 exclude_unset=True 确保 origin_data 也只记录实际提供/返回的字段
|
||||
self.origin_data = model.model_dump(exclude_unset=True)
|
||||
except AttributeError as e:
|
||||
raise ValueError(
|
||||
f"Origin data validation failed for {self.node_type}: model_dump not available"
|
||||
) from e
|
||||
except (ValidationError, TypeError, ValueError) as e:
|
||||
raise ValueError(f"Origin data validation failed for {self.node_type}: {e}") from e
|
||||
|
||||
# ===== Schema 相关方法 =====
|
||||
|
||||
def get_schema(self) -> Optional[Type[T]]:
|
||||
"""
|
||||
获取数据的 Schema 类型。
|
||||
|
||||
Returns:
|
||||
Pydantic Model 类型，如果未设置则返回 None
|
||||
"""
|
||||
return self.schema
|
||||
|
||||
def set_schema(self, schema: Type[T]) -> None:
|
||||
"""
|
||||
设置数据的 Schema 类型。
|
||||
|
||||
Args:
|
||||
schema: Pydantic Model 类型
|
||||
"""
|
||||
self.schema = schema
|
||||
|
||||
def validate(self) -> bool:
|
||||
"""
|
||||
验证当前数据是否符合 schema。
|
||||
|
||||
Returns:
|
||||
True 如果验证通过或无 schema，False 如果验证失败
|
||||
"""
|
||||
if self.schema is None or self.data is None:
|
||||
return True
|
||||
|
||||
try:
|
||||
self.schema.model_validate(self.data)
|
||||
return True
|
||||
except ValidationError:
|
||||
return False
|
||||
|
||||
def to_model(self) -> Optional[T]:
|
||||
"""
|
||||
将数据转换为 Pydantic 模型实例。
|
||||
|
||||
Returns:
|
||||
模型实例，如果无数据或无schema则返回None
|
||||
|
||||
Raises:
|
||||
ValidationError: 如果数据验证失败
|
||||
"""
|
||||
if self.schema is None or self.data is None:
|
||||
return None
|
||||
|
||||
return self.schema.model_validate(self.data)
|
||||
|
||||
def from_model(self, model: T) -> None:
|
||||
"""
|
||||
从 Pydantic 模型实例设置数据。
|
||||
|
||||
Args:
|
||||
model: Pydantic 模型实例
|
||||
"""
|
||||
try:
|
||||
self.data = model.model_dump()
|
||||
except AttributeError as e:
|
||||
raise ValueError(
|
||||
f"Cannot convert model to dict: model_dump not available ({type(model)})"
|
||||
) from e
|
||||
|
||||
# 自动设置schema
|
||||
if self.schema is None:
|
||||
self.schema = type(model)
|
||||
|
||||
# ===== 辅助方法 =====
|
||||
|
||||
def get_field(self, field_name: str, default: Any = None) -> Any:
|
||||
"""
|
||||
从data中获取指定字段的值。
|
||||
|
||||
Args:
|
||||
field_name: 字段名
|
||||
default: 默认值
|
||||
|
||||
Returns:
|
||||
字段值，如果不存在则返回default
|
||||
"""
|
||||
if self.data is None:
|
||||
return default
|
||||
return self.data.get(field_name, default)
|
||||
|
||||
def set_field(self, field_name: str, value: Any) -> None:
|
||||
"""
|
||||
设置data中的指定字段。
|
||||
|
||||
Args:
|
||||
field_name: 字段名
|
||||
value: 字段值
|
||||
"""
|
||||
if self.data is None:
|
||||
self.data = {}
|
||||
self.data[field_name] = value
|
||||
|
||||
def has_field(self, field_name: str) -> bool:
|
||||
"""
|
||||
检查data中是否存在指定字段。
|
||||
|
||||
Args:
|
||||
field_name: 字段名
|
||||
|
||||
Returns:
|
||||
True 如果字段存在
|
||||
"""
|
||||
if self.data is None:
|
||||
return False
|
||||
return field_name in self.data
|
||||
|
||||
# ===== 状态管理方法（带日志记录） =====
|
||||
|
||||
def append_log(self, message: str) -> None:
|
||||
"""追加日志到 sync_log（中文）"""
|
||||
import time
|
||||
from datetime import datetime
|
||||
timestamp = datetime.fromtimestamp(time.time()).strftime("%H:%M:%S")
|
||||
log_entry = f"[{timestamp}] {message}"
|
||||
|
||||
if self.sync_log:
|
||||
self.sync_log += "\n" + log_entry
|
||||
else:
|
||||
self.sync_log = log_entry
|
||||
|
||||
def set_status(self, new_status: SyncStatus, reason: str = "") -> None:
|
||||
"""
|
||||
设置status并更新时间戳和日志（如果状态改变）
|
||||
|
||||
Args:
|
||||
new_status: 新的执行状态
|
||||
reason: 状态变化原因（中文）
|
||||
"""
|
||||
if self.status != new_status:
|
||||
old_status = self.status
|
||||
self.status = new_status
|
||||
reason_text = f" - {reason}" if reason else ""
|
||||
|
||||
# DEBUG 日志
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.debug(
|
||||
f"[{self.node_type}] 节点 {self.node_id}: "
|
||||
f"执行状态 {old_status.value} → {new_status.value}{reason_text}"
|
||||
)
|
||||
|
||||
def set_data_id(self, new_data_id: str, reason: str = "") -> None:
|
||||
"""
|
||||
设置 data_id（仅允许在 event apply 路径内调用）
|
||||
"""
|
||||
if self.data_id != new_data_id:
|
||||
old_data_id = self.data_id
|
||||
self.data_id = new_data_id
|
||||
|
||||
reason_text = f" - {reason}" if reason else ""
|
||||
self.append_log(f"数据ID: {old_data_id or '<empty>'} → {new_data_id or '<empty>'}{reason_text}")
|
||||
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.debug(
|
||||
f"[{self.node_type}] 节点 {self.node_id}: "
|
||||
f"data_id {old_data_id or '<empty>'} → {new_data_id or '<empty>'}{reason_text}"
|
||||
)
|
||||
|
||||
def set_binding_status(self, new_status: BindingStatus, reason: str = "") -> None:
|
||||
"""
|
||||
设置binding_status并更新时间戳和日志（如果状态改变）
|
||||
|
||||
Args:
|
||||
new_status: 新的绑定状态
|
||||
reason: 状态变化原因（中文）
|
||||
"""
|
||||
if self.binding_status != new_status:
|
||||
if self.__dict__.get("_core_guard_enabled", False):
|
||||
write_source = self.__dict__.get("_core_write_source", "")
|
||||
if write_source != "state_machine":
|
||||
raise RuntimeError(
|
||||
f"binding_status update must go through state machine for node={self.node_id}, source={write_source or '<none>'}"
|
||||
)
|
||||
old_status = self.binding_status
|
||||
self.binding_status = new_status
|
||||
|
||||
reason_text = reason or "<no-reason>"
|
||||
|
||||
# DEBUG 日志
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.debug(
|
||||
f"[{self.node_type}] 节点 {self.node_id}: "
|
||||
f"绑定状态 {old_status.value} → {new_status.value} - {reason_text}"
|
||||
)
|
||||
|
||||
def set_action(self, new_action: SyncAction, reason: str = "") -> None:
|
||||
"""
|
||||
设置action并更新时间戳和日志（如果动作改变）
|
||||
|
||||
Args:
|
||||
new_action: 新的同步动作
|
||||
reason: 动作变化原因（中文）
|
||||
"""
|
||||
if self.action != new_action:
|
||||
old_action = self.action
|
||||
self.action = new_action
|
||||
reason_text = f" - {reason}" if reason else ""
|
||||
|
||||
# DEBUG 日志
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.debug(
|
||||
f"[{self.node_type}] 节点 {self.node_id}: "
|
||||
f"同步动作 {old_action.value} → {new_action.value}{reason_text}"
|
||||
)
|
||||
@@ -0,0 +1,15 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Mapping, TypeGuard
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
def is_pydantic_model_class(value: Any) -> TypeGuard[type[BaseModel]]:
|
||||
return isinstance(value, type) and issubclass(value, BaseModel)
|
||||
|
||||
|
||||
def get_pydantic_model_fields(value: Any) -> Mapping[str, Any]:
|
||||
if is_pydantic_model_class(value):
|
||||
return value.model_fields
|
||||
return {}
|
||||
@@ -0,0 +1,48 @@
|
||||
"""
|
||||
Common type definitions for the sync system.
|
||||
|
||||
This module contains only enum definitions. SyncNode is defined in sync_node.py.
|
||||
"""
|
||||
|
||||
from enum import Enum
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class SyncAction(str, Enum):
|
||||
"""同步动作枚举"""
|
||||
NONE = "NONE" # 保持现状 (SKIP)
|
||||
CREATE = "CREATE" # 创建新对象
|
||||
UPDATE = "UPDATE" # 更新现有对象
|
||||
DELETE = "DELETE" # 删除对象
|
||||
|
||||
|
||||
class SyncStatus(str, Enum):
|
||||
"""同步执行状态枚举"""
|
||||
PENDING = "PENDING"
|
||||
IN_PROGRESS = "IN_PROGRESS"
|
||||
SUCCESS = "SUCCESS"
|
||||
FAILED = "FAILED"
|
||||
SKIPPED = "SKIPPED"
|
||||
PRECONDITION_FAILED = "PRECONDITION_FAILED"
|
||||
|
||||
|
||||
class BindingStatus(str, Enum):
|
||||
"""绑定状态枚举"""
|
||||
UNCHECKED = "unchecked" # 初始瞬态 (Init)
|
||||
|
||||
# --- 终态 (Final States) ---
|
||||
NORMAL = "normal" # 稳态: 绑定存在，数据健康
|
||||
MISSING = "missing" # 孤儿: 未找到绑定对象
|
||||
|
||||
# --- 阻断态 (Blocking States) ---
|
||||
ABNORMAL = "abnormal" # 只有ID没有数据 (Corrupted)
|
||||
WARNING = "warning" # 匹配成功但数据不可用 (Data Invalid) 或 对方损坏
|
||||
DEPENDENCY_ERROR = "dep_err" # 依赖项缺失、父级ID解析失败或异常
|
||||
PEER_CREATING = "peer_creating" # 对侧创建中，本侧等待提交
|
||||
|
||||
|
||||
class BindingRecord(BaseModel):
|
||||
"""持久化绑定记录的结构化对象 (纯 ID 映射)"""
|
||||
local_id: str
|
||||
remote_id: Optional[str] = None
|
||||
@@ -0,0 +1,51 @@
|
||||
from .datasource_config import ApiDataSourceConfig, JsonlDataSourceConfig, DataSourceConfig
|
||||
from .persist_config import PersistConfig
|
||||
from .logging_config import LoggingConfig, resolve_log_file_path
|
||||
from .run_presets import (
|
||||
list_preset_names,
|
||||
get_preset_file_candidates,
|
||||
resolve_preset_file,
|
||||
load_overrides_from_file,
|
||||
load_named_preset_overrides,
|
||||
)
|
||||
from .strategy_config import (
|
||||
OrphanAction,
|
||||
UpdateDirection,
|
||||
StrategyConfig,
|
||||
StrategyRuntimeConfig,
|
||||
ConfigPresets,
|
||||
)
|
||||
from .pipeline_config import PipelineRunConfig, DEFAULT_NODE_TYPES
|
||||
from .builder import (
|
||||
build_default_config,
|
||||
build_config,
|
||||
apply_config_overrides,
|
||||
apply_env_overrides,
|
||||
config_snapshot,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"ApiDataSourceConfig",
|
||||
"JsonlDataSourceConfig",
|
||||
"PipelineRunConfig",
|
||||
"DataSourceConfig",
|
||||
"PersistConfig",
|
||||
"LoggingConfig",
|
||||
"resolve_log_file_path",
|
||||
"list_preset_names",
|
||||
"get_preset_file_candidates",
|
||||
"resolve_preset_file",
|
||||
"load_overrides_from_file",
|
||||
"load_named_preset_overrides",
|
||||
"OrphanAction",
|
||||
"UpdateDirection",
|
||||
"StrategyConfig",
|
||||
"StrategyRuntimeConfig",
|
||||
"ConfigPresets",
|
||||
"DEFAULT_NODE_TYPES",
|
||||
"build_default_config",
|
||||
"build_config",
|
||||
"apply_config_overrides",
|
||||
"apply_env_overrides",
|
||||
"config_snapshot",
|
||||
]
|
||||
@@ -0,0 +1,189 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Mapping, Optional, Union
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ..common.registry import DomainRegistry
|
||||
from .datasource_config import DataSourceConfig
|
||||
from .pipeline_config import PipelineRunConfig
|
||||
from .strategy_config import ConfigPresets, StrategyRuntimeConfig
|
||||
|
||||
|
||||
def _deep_merge(base: Mapping[str, Any], override: Mapping[str, Any]) -> Dict[str, Any]:
|
||||
merged: Dict[str, Any] = dict(base)
|
||||
for key, value in override.items():
|
||||
if isinstance(value, dict) and isinstance(merged.get(key), dict):
|
||||
if key in {"local_datasource", "remote_datasource"}:
|
||||
current_type = merged[key].get("type")
|
||||
next_type = value.get("type", current_type)
|
||||
if current_type != next_type:
|
||||
merged[key] = dict(value)
|
||||
continue
|
||||
merged[key] = _deep_merge(merged[key], value)
|
||||
else:
|
||||
merged[key] = value
|
||||
return merged
|
||||
|
||||
|
||||
def _default_strategy_map(node_types: list[str]) -> Dict[str, StrategyRuntimeConfig]:
|
||||
strategy_map: Dict[str, StrategyRuntimeConfig] = {}
|
||||
for node_type in node_types:
|
||||
strategy_class = DomainRegistry.get_strategy_class(node_type)
|
||||
if strategy_class is None:
|
||||
continue
|
||||
strategy_map[node_type] = StrategyRuntimeConfig(
|
||||
node_type=node_type,
|
||||
config=strategy_class.default_config.model_copy(deep=True),
|
||||
)
|
||||
return strategy_map
|
||||
|
||||
|
||||
def _apply_strategy_overrides(
|
||||
node_types: list[str],
|
||||
strategy_overrides: Any,
|
||||
) -> list[StrategyRuntimeConfig]:
|
||||
strategy_map = _default_strategy_map(node_types)
|
||||
|
||||
if isinstance(strategy_overrides, list):
|
||||
for item in strategy_overrides:
|
||||
runtime_cfg = StrategyRuntimeConfig.model_validate(item)
|
||||
strategy_map[runtime_cfg.node_type] = runtime_cfg
|
||||
return [strategy_map[n] for n in node_types if n in strategy_map]
|
||||
|
||||
if not isinstance(strategy_overrides, dict):
|
||||
return [strategy_map[n] for n in node_types if n in strategy_map]
|
||||
|
||||
for node_type, raw_override in strategy_overrides.items():
|
||||
if node_type not in strategy_map:
|
||||
strategy_map[node_type] = StrategyRuntimeConfig(node_type=node_type)
|
||||
|
||||
runtime_cfg = strategy_map[node_type]
|
||||
override = dict(raw_override or {})
|
||||
|
||||
preset = override.pop("preset", None)
|
||||
if preset:
|
||||
runtime_cfg.config = ConfigPresets.get_preset(str(preset))
|
||||
|
||||
override.pop("skip_sync", None)
|
||||
override.pop("force_sync", None)
|
||||
override.pop("load_mode", None)
|
||||
|
||||
if "config" in override and isinstance(override["config"], dict):
|
||||
runtime_cfg.config = runtime_cfg.config.model_validate(
|
||||
_deep_merge(runtime_cfg.config.model_dump(), override.pop("config"))
|
||||
)
|
||||
|
||||
if override:
|
||||
runtime_cfg.config = runtime_cfg.config.model_validate(
|
||||
_deep_merge(runtime_cfg.config.model_dump(), override)
|
||||
)
|
||||
|
||||
return [strategy_map[n] for n in node_types if n in strategy_map]
|
||||
|
||||
|
||||
def config_snapshot(config: PipelineRunConfig) -> Dict[str, Any]:
|
||||
snap = config.model_dump(mode="json")
|
||||
for key in ("local_datasource", "remote_datasource"):
|
||||
ds = snap.get(key) or {}
|
||||
if ds.get("api_secret"):
|
||||
ds["api_secret"] = "***"
|
||||
return snap
|
||||
|
||||
|
||||
def build_default_config(project_root: Path) -> PipelineRunConfig:
|
||||
defaults = {
|
||||
"local_datasource": {
|
||||
"type": "jsonl",
|
||||
"jsonl_dir": str(project_root / "filtered_datasource" / "datasource" / "ecm-2025-12-02"),
|
||||
"read_only": False,
|
||||
"handler_configs": {},
|
||||
},
|
||||
"remote_datasource": {
|
||||
"type": "jsonl",
|
||||
"jsonl_dir": str(project_root / "remote_datasource" / "datasource"),
|
||||
"read_only": False,
|
||||
"handler_configs": {},
|
||||
},
|
||||
"target_project_ids": [],
|
||||
"persist": {
|
||||
"db_path": str(project_root / "test_results" / "sync_state_machine" / "pipeline_default.db"),
|
||||
"enable": True,
|
||||
"wipe_on_start": False,
|
||||
},
|
||||
"logging": {
|
||||
"initialize": True,
|
||||
"file_path": None,
|
||||
"level": 20,
|
||||
"suppress_node_logs": True,
|
||||
},
|
||||
}
|
||||
|
||||
config = PipelineRunConfig.model_validate(defaults)
|
||||
config.strategies = _apply_strategy_overrides(config.node_types, {})
|
||||
return config
|
||||
|
||||
|
||||
def apply_config_overrides(config: PipelineRunConfig, overrides: Dict[str, Any]) -> PipelineRunConfig:
|
||||
merged = _deep_merge(config.model_dump(), overrides)
|
||||
raw_strategies = merged.pop("strategies", None)
|
||||
legacy_strategy_overrides = merged.pop("strategy_overrides", None)
|
||||
|
||||
resolved = PipelineRunConfig.model_validate(merged)
|
||||
|
||||
if raw_strategies is not None:
|
||||
resolved.strategies = _apply_strategy_overrides(resolved.node_types, raw_strategies)
|
||||
else:
|
||||
resolved.strategies = _apply_strategy_overrides(resolved.node_types, legacy_strategy_overrides or {})
|
||||
|
||||
return resolved
|
||||
|
||||
|
||||
def apply_env_overrides(
|
||||
config: PipelineRunConfig,
|
||||
project_root: Path,
|
||||
*,
|
||||
dotenv_file: Optional[Union[str, Path]] = None,
|
||||
) -> PipelineRunConfig:
|
||||
raise NotImplementedError(
|
||||
"Environment-based overrides are removed. Use JSON config files via overrides_file instead."
|
||||
)
|
||||
|
||||
|
||||
def build_config(
|
||||
*,
|
||||
project_root: Path,
|
||||
local_datasource: Optional[Union[DataSourceConfig, Dict[str, Any]]] = None,
|
||||
remote_datasource: Optional[Union[DataSourceConfig, Dict[str, Any]]] = None,
|
||||
overrides: Optional[Dict[str, Any]] = None,
|
||||
overrides_file: Optional[Union[str, Path]] = None,
|
||||
dotenv_file: Optional[Union[str, Path]] = None,
|
||||
) -> PipelineRunConfig:
|
||||
if dotenv_file is not None:
|
||||
raise ValueError("dotenv_file is no longer supported; use overrides_file JSON instead")
|
||||
|
||||
config = build_default_config(project_root)
|
||||
|
||||
if local_datasource is not None:
|
||||
local_payload = (
|
||||
local_datasource.model_dump() if isinstance(local_datasource, BaseModel) else dict(local_datasource)
|
||||
)
|
||||
config = apply_config_overrides(config, {"local_datasource": local_payload})
|
||||
|
||||
if remote_datasource is not None:
|
||||
remote_payload = (
|
||||
remote_datasource.model_dump() if isinstance(remote_datasource, BaseModel) else dict(remote_datasource)
|
||||
)
|
||||
config = apply_config_overrides(config, {"remote_datasource": remote_payload})
|
||||
|
||||
if overrides_file:
|
||||
override_path = Path(overrides_file)
|
||||
file_data = json.loads(override_path.read_text(encoding="utf-8"))
|
||||
config = apply_config_overrides(config, file_data)
|
||||
|
||||
if overrides:
|
||||
config = apply_config_overrides(config, overrides)
|
||||
|
||||
return config
|
||||
@@ -0,0 +1,34 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Annotated, Any, Dict, List, Literal, Union
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
|
||||
class JsonlDataSourceConfig(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", validate_assignment=True)
|
||||
|
||||
type: Literal["jsonl"] = "jsonl"
|
||||
jsonl_dir: str
|
||||
read_only: bool = False
|
||||
target_project_ids: List[str] = Field(default_factory=list)
|
||||
handler_configs: Dict[str, Dict[str, Any]] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class ApiDataSourceConfig(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", validate_assignment=True)
|
||||
|
||||
type: Literal["api"] = "api"
|
||||
api_base_url: str = ""
|
||||
api_uid: str = ""
|
||||
api_secret: str = ""
|
||||
api_debug: bool = False
|
||||
poll_max_retries: int = 10
|
||||
poll_interval: float = 0.5
|
||||
api_rate_limit_max_requests: int = 10
|
||||
api_rate_limit_window_seconds: float = 10.0
|
||||
target_project_ids: List[str] = Field(min_length=1)
|
||||
handler_configs: Dict[str, Dict[str, Any]] = Field(default_factory=dict)
|
||||
|
||||
|
||||
DataSourceConfig = Annotated[Union[JsonlDataSourceConfig, ApiDataSourceConfig], Field(discriminator="type")]
|
||||
@@ -0,0 +1,30 @@
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
|
||||
class LoggingConfig(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", validate_assignment=True)
|
||||
|
||||
initialize: bool = True
|
||||
file_path: Optional[str] = None
|
||||
file_dir: Optional[str] = None
|
||||
file_name_pattern: Optional[str] = None
|
||||
level: int = 20
|
||||
suppress_node_logs: bool = True
|
||||
|
||||
|
||||
def resolve_log_file_path(cfg: LoggingConfig) -> Optional[str]:
|
||||
if cfg.file_path:
|
||||
return cfg.file_path
|
||||
if not cfg.file_dir:
|
||||
return None
|
||||
|
||||
pattern = cfg.file_name_pattern or "pipeline_{timestamp}.log"
|
||||
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
filename = pattern.replace("{timestamp}", ts)
|
||||
log_dir = Path(cfg.file_dir)
|
||||
log_dir.mkdir(parents=True, exist_ok=True)
|
||||
return str(log_dir / filename)
|
||||
@@ -0,0 +1,600 @@
|
||||
meta:
|
||||
name: node_state_machine
|
||||
version: 0.1
|
||||
language: zh-CN
|
||||
desc: "节点状态机配置（用于驱动/校验未来的状态机组件）"
|
||||
|
||||
# 命名约定（配置内唯一真名；图上不再有 display/别名）：
|
||||
# - Sxx: State / PseudoState（节点）
|
||||
# - Exx: Event（事件）
|
||||
# - 事件分支由渲染器在边上追加 a/b/c...（对应 outcomes 顺序）
|
||||
id_convention:
|
||||
state_prefix: "S"
|
||||
event_prefix: "E"
|
||||
|
||||
# Variables are computed by the caller and passed into the state machine.
|
||||
# Rule: the YAML never directly mutates "peer" nodes. Cross-node changes must be
|
||||
# represented as separate invocations with different inputs.
|
||||
context:
|
||||
is_create_zombie: { type: bool, desc: "初始化时判定为 CREATE 僵尸：action=CREATE 且 (status=FAILED 或 data_id 为空)" }
|
||||
|
||||
has_binding_record: { type: bool, desc: "binding_manager 里存在绑定记录 (remote_id/local_id 非空)" }
|
||||
core_binding_result: { type: enum, values: [NORMAL, WARNING, ABNORMAL], desc: "SyncDecisionEngine.determine_core_status() 给本节点的结果（仅在 has_binding_record=true 时有效）" }
|
||||
peer_core_binding_result: { type: enum, values: [NORMAL, WARNING, ABNORMAL], desc: "SyncDecisionEngine.determine_core_status() 给对方节点的结果（用于静态约束 core matrix 对称性）" }
|
||||
|
||||
data_present: { type: bool, desc: "node.get_data() 非空" }
|
||||
has_depend_fields: { type: bool, desc: "config.depend_fields 非空" }
|
||||
dep_satisfied: { type: bool, desc: "依赖满足：空依赖值跳过；非空依赖值必须能在 collection 中按 data_id 找到且节点状态可用" }
|
||||
|
||||
auto_bind_enabled: { type: bool, desc: "config.auto_bind && auto_bind_fields 非空" }
|
||||
auto_bind_outcome:
|
||||
type: enum
|
||||
values: [ONE_TO_ONE, AMBIGUOUS, ZERO, KEY_MISSING, KEY_ID_RESOLVE_FAIL, DATA_MISSING]
|
||||
desc: "自动绑定结果（由业务代码完成全集搜索+去噪后 1:1 判定）"
|
||||
|
||||
create_enabled: { type: bool, desc: "create() 是否对当前方向启用 (local_orphan_action/remote_orphan_action)" }
|
||||
create_id_resolve_ok: { type: bool, desc: "IDResolver.resolve_and_validate() 是否成功" }
|
||||
spawn_success: { type: bool, desc: "创建副作用（spawn + bind）是否成功" }
|
||||
|
||||
update_enabled: { type: bool, desc: "update() 是否启用 (update_direction != NONE)" }
|
||||
target_has_data_id: { type: bool, desc: "UPDATE 目标节点 data_id 非空" }
|
||||
update_id_resolve_ok: { type: bool, desc: "UPDATE 前 IDResolver.resolve_and_validate() 是否成功" }
|
||||
has_diff: { type: bool, desc: "needs_update_check 判定有差异" }
|
||||
|
||||
handler_result:
|
||||
type: enum
|
||||
values: [SUCCESS, FAILED, SKIPPED, IN_PROGRESS]
|
||||
desc: "TaskResult.status"
|
||||
poll_timeout: { type: bool, desc: "异步轮询超过 max_retries" }
|
||||
execute_action:
|
||||
type: enum
|
||||
values: [CREATE, UPDATE, DELETE, NONE]
|
||||
desc: "E40 执行时的动作类型（用于区分 SKIPPED 的落点）"
|
||||
|
||||
# State IDs are for visualization and static validation.
|
||||
# NOTE: 本状态机按“加载时清空执行状态，再开始本轮”建模：
|
||||
# - 加载/初始化后，稳态节点的 status 统一回到 PENDING
|
||||
# - 上一轮遗留的 FAILED/SUCCESS 等不作为本轮的入口状态
|
||||
states:
|
||||
# --- Stable entry to bind (after init reset + zombie cleanup) ---
|
||||
S00:
|
||||
binding_status: UNCHECKED
|
||||
action: NONE
|
||||
status: PENDING
|
||||
data_id_exist: true
|
||||
desc: |
|
||||
进入 bind 前的唯一起点
|
||||
binding_status: UNCHECKED
|
||||
action: NONE
|
||||
status: PENDING
|
||||
data_id_exist: true
|
||||
|
||||
# --- Bind results ---
|
||||
S01:
|
||||
binding_status: NORMAL
|
||||
action: NONE
|
||||
status: PENDING
|
||||
data_id_exist: true
|
||||
desc: |
|
||||
绑定正常（待后续 update 检查）
|
||||
binding_status: NORMAL
|
||||
action: NONE
|
||||
status: PENDING
|
||||
data_id_exist: true
|
||||
|
||||
S02:
|
||||
binding_status: MISSING
|
||||
action: NONE
|
||||
status: PENDING
|
||||
data_id_exist: true
|
||||
desc: |
|
||||
孤儿节点（可能进入 create / auto_bind / delete）
|
||||
binding_status: MISSING
|
||||
action: NONE
|
||||
status: PENDING
|
||||
data_id_exist: true
|
||||
|
||||
S03:
|
||||
binding_status: ABNORMAL
|
||||
action: NONE
|
||||
status: PENDING
|
||||
data_id_exist: true
|
||||
desc: |
|
||||
数据损坏/缺失（阻断态，需人工）
|
||||
binding_status: ABNORMAL
|
||||
action: NONE
|
||||
status: PENDING
|
||||
data_id_exist: true
|
||||
S04:
|
||||
binding_status: WARNING
|
||||
action: NONE
|
||||
status: PENDING
|
||||
data_id_exist: true
|
||||
desc: |
|
||||
不可自动处理（阻断态，需人工）
|
||||
binding_status: WARNING
|
||||
action: NONE
|
||||
status: PENDING
|
||||
data_id_exist: true
|
||||
S05:
|
||||
binding_status: DEPENDENCY_ERROR
|
||||
action: NONE
|
||||
status: PENDING
|
||||
data_id_exist: true
|
||||
desc: |
|
||||
依赖项不满足/ID解析失败（阻断态）
|
||||
binding_status: DEPENDENCY_ERROR
|
||||
action: NONE
|
||||
status: PENDING
|
||||
data_id_exist: true
|
||||
|
||||
# --- Create & Update prepared actions (target-side nodes) ---
|
||||
S06:
|
||||
binding_status: NORMAL
|
||||
action: CREATE
|
||||
status: PENDING
|
||||
data_id_exist: false
|
||||
shape: pseudo
|
||||
desc: |
|
||||
待执行 CREATE（执行入口）
|
||||
binding_status: NORMAL
|
||||
action: CREATE
|
||||
status: PENDING
|
||||
data_id_exist: false
|
||||
S07:
|
||||
binding_status: NORMAL
|
||||
action: UPDATE
|
||||
status: PENDING
|
||||
data_id_exist: true
|
||||
desc: |
|
||||
准备更新
|
||||
binding_status: NORMAL
|
||||
action: UPDATE
|
||||
status: PENDING
|
||||
data_id_exist: true
|
||||
|
||||
S08:
|
||||
binding_status: NORMAL
|
||||
action: UPDATE
|
||||
status: PRECONDITION_FAILED
|
||||
data_id_exist: [false, true]
|
||||
desc: |
|
||||
更新前置失败（缺 data_id / 依赖ID未解析等）
|
||||
binding_status: NORMAL
|
||||
action: UPDATE
|
||||
status: PRECONDITION_FAILED
|
||||
data_id_exist: false | true
|
||||
|
||||
S09:
|
||||
binding_status: NORMAL
|
||||
action: DELETE
|
||||
status: PENDING
|
||||
data_id_exist: true
|
||||
desc: |
|
||||
待执行 DELETE（执行入口）
|
||||
binding_status: NORMAL
|
||||
action: DELETE
|
||||
status: PENDING
|
||||
data_id_exist: true
|
||||
|
||||
# --- Sync results ---
|
||||
S10C:
|
||||
binding_status: NORMAL
|
||||
action: CREATE
|
||||
status: IN_PROGRESS
|
||||
data_id_exist: false
|
||||
desc: |
|
||||
CREATE 异步执行中
|
||||
binding_status: NORMAL
|
||||
action: CREATE
|
||||
status: IN_PROGRESS
|
||||
data_id_exist: false
|
||||
S10U:
|
||||
binding_status: NORMAL
|
||||
action: UPDATE
|
||||
status: IN_PROGRESS
|
||||
data_id_exist: true
|
||||
desc: |
|
||||
UPDATE 异步执行中
|
||||
binding_status: NORMAL
|
||||
action: UPDATE
|
||||
status: IN_PROGRESS
|
||||
data_id_exist: true
|
||||
S10D:
|
||||
binding_status: NORMAL
|
||||
action: DELETE
|
||||
status: IN_PROGRESS
|
||||
data_id_exist: true
|
||||
desc: |
|
||||
DELETE 异步执行中
|
||||
binding_status: NORMAL
|
||||
action: DELETE
|
||||
status: IN_PROGRESS
|
||||
data_id_exist: true
|
||||
S11C:
|
||||
binding_status: NORMAL
|
||||
action: CREATE
|
||||
status: SUCCESS
|
||||
data_id_exist: true
|
||||
desc: |
|
||||
CREATE 执行成功
|
||||
binding_status: NORMAL
|
||||
action: CREATE
|
||||
status: SUCCESS
|
||||
data_id_exist: true
|
||||
S11U:
|
||||
binding_status: NORMAL
|
||||
action: UPDATE
|
||||
status: SUCCESS
|
||||
data_id_exist: true
|
||||
desc: |
|
||||
UPDATE 执行成功
|
||||
binding_status: NORMAL
|
||||
action: UPDATE
|
||||
status: SUCCESS
|
||||
data_id_exist: true
|
||||
S11D:
|
||||
binding_status: NORMAL
|
||||
action: DELETE
|
||||
status: SUCCESS
|
||||
data_id_exist: true
|
||||
desc: |
|
||||
DELETE 执行成功（随后删除节点）
|
||||
binding_status: NORMAL
|
||||
action: DELETE
|
||||
status: SUCCESS
|
||||
data_id_exist: true
|
||||
S12C:
|
||||
binding_status: NORMAL
|
||||
action: CREATE
|
||||
status: FAILED
|
||||
data_id_exist: [false, true]
|
||||
desc: |
|
||||
CREATE 执行失败
|
||||
binding_status: NORMAL
|
||||
action: CREATE
|
||||
status: FAILED
|
||||
data_id_exist: false | true
|
||||
S12U:
|
||||
binding_status: NORMAL
|
||||
action: UPDATE
|
||||
status: FAILED
|
||||
data_id_exist: [false, true]
|
||||
desc: |
|
||||
UPDATE 执行失败
|
||||
binding_status: NORMAL
|
||||
action: UPDATE
|
||||
status: FAILED
|
||||
data_id_exist: false | true
|
||||
S12D:
|
||||
binding_status: NORMAL
|
||||
action: DELETE
|
||||
status: FAILED
|
||||
data_id_exist: [false, true]
|
||||
desc: |
|
||||
DELETE 执行失败
|
||||
binding_status: NORMAL
|
||||
action: DELETE
|
||||
status: FAILED
|
||||
data_id_exist: false | true
|
||||
|
||||
S13:
|
||||
binding_status: PEER_CREATING
|
||||
action: NONE
|
||||
status: PENDING
|
||||
data_id_exist: true
|
||||
desc: |
|
||||
对侧创建进行中（本侧等待提交）
|
||||
binding_status: PEER_CREATING
|
||||
action: NONE
|
||||
status: PENDING
|
||||
data_id_exist: true
|
||||
|
||||
# Handler 在 update 路径中可能把 action 降级为 NONE 并返回 SKIPPED。
|
||||
# NOTE: 这会产生 (NORMAL, NONE, SKIPPED)，并且该节点会被统计到 none 列。
|
||||
S14:
|
||||
binding_status: NORMAL
|
||||
action: NONE
|
||||
status: SKIPPED
|
||||
data_id_exist: true
|
||||
desc: |
|
||||
更新被跳过（执行时降级为 NONE）
|
||||
binding_status: NORMAL
|
||||
action: NONE
|
||||
status: SKIPPED
|
||||
data_id_exist: true
|
||||
|
||||
pseudo_states:
|
||||
S15:
|
||||
desc: "节点被清理删除（zombie cleanup）"
|
||||
S16:
|
||||
desc: "持久化加载输入态：本轮 E01 的输入节点集合（可包含上轮遗留与已稳定节点）"
|
||||
S17:
|
||||
desc: "加载异常隔离态：持久化枚举非法，保留错误供人工处理，不进入自动流程"
|
||||
|
||||
stages:
|
||||
bootstrap:
|
||||
desc: "启动/初始化（仅针对持久化加载出的节点）：僵尸清理与归并到 S00"
|
||||
entry_states: [S16]
|
||||
exit_states: [S00, S15]
|
||||
|
||||
bind:
|
||||
desc: "绑定判定：core matrix +（可选）依赖检查 +（可选）自动绑定；stage 仅约束边界，不要求内部顺序"
|
||||
entry_states: [S00]
|
||||
exit_states: [S01, S02, S03, S04, S05, S13]
|
||||
|
||||
create_prepare:
|
||||
desc: "Create: 从 MISSING 源节点发起创建请求并在副作用提交后归并"
|
||||
entry_states: [S02, S13]
|
||||
exit_states: [S01, S02, S04, S05, S09, S13]
|
||||
|
||||
update_prepare:
|
||||
desc: "Update: 检测差异并在目标节点上设置 UPDATE/PRECONDITION_FAILED"
|
||||
entry_states: [S01]
|
||||
exit_states: [S01, S07, S08]
|
||||
|
||||
post_create_ready:
|
||||
desc: "创建成功后归并：CREATE 成功节点回到 NORMAL/NONE/PENDING，作为 update 起点"
|
||||
entry_states: [S11C]
|
||||
exit_states: [S01]
|
||||
|
||||
sync_execute:
|
||||
desc: "Sync: DataSource 调用 handler，根据 TaskResult 更新 status；部分 handler 可能降级 action"
|
||||
entry_states: [S06, S07, S09, S10C, S10U, S10D]
|
||||
exit_states: [S10C, S10U, S10D, S11C, S11U, S11D, S12C, S12U, S12D, S14]
|
||||
|
||||
transitions:
|
||||
E01:
|
||||
stage: bootstrap
|
||||
desc: "初始化：删CREATE僵尸；其余重置后进入起点"
|
||||
from: [S16]
|
||||
outcomes:
|
||||
- when: { is_create_zombie: true }
|
||||
to: S15
|
||||
- when: { is_create_zombie: false }
|
||||
to: S00
|
||||
|
||||
# --- Bind phase1 core ---
|
||||
E10:
|
||||
stage: bind
|
||||
desc: "核心绑定判定：有绑定按core结果分流；无绑定→MISSING"
|
||||
from: [S00]
|
||||
outcomes:
|
||||
- when: { has_binding_record: true, core_binding_result: NORMAL, peer_core_binding_result: NORMAL }
|
||||
to: S01
|
||||
emit:
|
||||
note: "绑定关系一致，进入正常态"
|
||||
- when: { has_binding_record: true, core_binding_result: WARNING, peer_core_binding_result: ABNORMAL }
|
||||
to: S04
|
||||
emit:
|
||||
note: "对端异常，本端标记WARNING"
|
||||
- when: { has_binding_record: true, core_binding_result: ABNORMAL, peer_core_binding_result: WARNING }
|
||||
to: S03
|
||||
emit:
|
||||
note: "本端异常，进入ABNORMAL"
|
||||
- when: { has_binding_record: true, core_binding_result: ABNORMAL, peer_core_binding_result: ABNORMAL }
|
||||
to: S03
|
||||
emit:
|
||||
note: "双方异常，进入ABNORMAL"
|
||||
- when: { has_binding_record: false }
|
||||
to: S02
|
||||
emit:
|
||||
note: "无绑定记录，标记为MISSING"
|
||||
|
||||
# --- Bind phase2 dependency ---
|
||||
E15:
|
||||
stage: bind
|
||||
desc: "依赖检查：失败→阻断；缺少依赖配置→WARNING；通过→不变（不在图上画自环）"
|
||||
from: [S02]
|
||||
outcomes:
|
||||
- when: { has_depend_fields: false, data_present: true }
|
||||
to: S04
|
||||
emit:
|
||||
note: "缺少依赖字段配置，无法自动处理"
|
||||
- when: { data_present: false }
|
||||
to: S03
|
||||
emit:
|
||||
note: "节点数据为空或不存在"
|
||||
- when: { data_present: true, has_depend_fields: true, dep_satisfied: false }
|
||||
to: S05
|
||||
emit:
|
||||
note: "依赖项不满足"
|
||||
|
||||
# --- Bind phase3 auto bind ---
|
||||
E16:
|
||||
stage: bind
|
||||
desc: "自动绑定：1:1→NORMAL；歧义/缺字段/数据为空→阻断；N:0 且允许创建→S13 等待创建提交"
|
||||
from: [S02]
|
||||
outcomes:
|
||||
- when: { auto_bind_enabled: false }
|
||||
to: S04
|
||||
emit:
|
||||
note: "自动绑定未启用"
|
||||
- when: { auto_bind_enabled: true, auto_bind_outcome: DATA_MISSING }
|
||||
to: S04
|
||||
emit:
|
||||
note: "节点数据为空，无法自动绑定"
|
||||
- when: { auto_bind_enabled: true, auto_bind_outcome: ONE_TO_ONE }
|
||||
to: S01
|
||||
emit:
|
||||
note: "自动绑定成功（1:1）"
|
||||
- when: { auto_bind_enabled: true, auto_bind_outcome: AMBIGUOUS }
|
||||
to: S04
|
||||
emit:
|
||||
note: "存在多个候选节点，无法自动绑定"
|
||||
- when: { auto_bind_enabled: true, auto_bind_outcome: KEY_MISSING }
|
||||
to: S04
|
||||
emit:
|
||||
note: "缺少业务键字段，无法自动绑定"
|
||||
- when: { auto_bind_enabled: true, auto_bind_outcome: KEY_ID_RESOLVE_FAIL }
|
||||
to: S05
|
||||
emit:
|
||||
note: "业务键中的ID字段解析失败"
|
||||
- when: { auto_bind_enabled: true, auto_bind_outcome: ZERO, create_enabled: true }
|
||||
to: S13
|
||||
emit:
|
||||
spawn_request: true
|
||||
spawn_target_node_state: S06
|
||||
note: "自动绑定识别为需创建，进入创建提交中间态"
|
||||
- when: { auto_bind_enabled: true, auto_bind_outcome: ZERO, create_enabled: false }
|
||||
to: S04
|
||||
emit:
|
||||
note: "未匹配到候选节点且不允许创建"
|
||||
|
||||
# --- Create prepare ---
|
||||
E20:
|
||||
stage: create_prepare
|
||||
desc: "创建准备阶段2：spawn 成功则提交到 NORMAL；失败保持等待态（不迁移）"
|
||||
from: [S13]
|
||||
outcomes:
|
||||
- when: { spawn_success: true }
|
||||
to: S01
|
||||
emit:
|
||||
note: "创建副作用提交成功"
|
||||
|
||||
E21:
|
||||
stage: create_prepare
|
||||
desc: "创建依赖解析失败→DEPENDENCY_ERROR"
|
||||
from: [S02]
|
||||
outcomes:
|
||||
- when: { create_enabled: true, data_present: true, create_id_resolve_ok: false }
|
||||
to: S05
|
||||
emit:
|
||||
note: "创建前依赖ID解析失败"
|
||||
|
||||
E22:
|
||||
stage: create_prepare
|
||||
desc: "孤儿自删准备：从 MISSING 进入 DELETE 执行入口"
|
||||
from: [S02]
|
||||
outcomes:
|
||||
- when: {}
|
||||
to: S09
|
||||
emit:
|
||||
note: "孤儿节点按策略进入自删"
|
||||
|
||||
# --- Update prepare (target-side) ---
|
||||
E30:
|
||||
stage: update_prepare
|
||||
desc: "更新准备：缺data_id/依赖未解析→PRECONDITION_FAILED；有差异→UPDATE"
|
||||
from: [S01]
|
||||
outcomes:
|
||||
- when: { update_enabled: true, target_has_data_id: false }
|
||||
to: S08
|
||||
emit:
|
||||
note: "目标节点缺少data_id"
|
||||
- when: { update_enabled: true, target_has_data_id: true, update_id_resolve_ok: false }
|
||||
to: S08
|
||||
emit:
|
||||
note: "更新前依赖ID未解析"
|
||||
- when: { update_enabled: true, target_has_data_id: true, update_id_resolve_ok: true, has_diff: true }
|
||||
to: S07
|
||||
emit:
|
||||
note: "检测到数据差异，准备更新"
|
||||
|
||||
# --- Post-create normalization ---
|
||||
E41:
|
||||
stage: post_create_ready
|
||||
desc: "创建成功后归并到 update 起点"
|
||||
from: [S11C]
|
||||
outcomes:
|
||||
- when: { execute_action: CREATE }
|
||||
to: S01
|
||||
emit:
|
||||
note: "创建成功后转入更新起点"
|
||||
|
||||
# --- Sync execute (split by action) ---
|
||||
E40C_START:
|
||||
stage: sync_execute
|
||||
desc: "CREATE 执行入口：进入异步执行中"
|
||||
from: [S06]
|
||||
outcomes:
|
||||
- when: { handler_result: IN_PROGRESS }
|
||||
to: S10C
|
||||
emit:
|
||||
note: "CREATE 进入异步执行中"
|
||||
|
||||
E40C:
|
||||
stage: sync_execute
|
||||
desc: "CREATE 执行结果回写：进行中/成功/失败"
|
||||
from: [S10C]
|
||||
outcomes:
|
||||
- when: { handler_result: SUCCESS }
|
||||
to: S11C
|
||||
emit:
|
||||
note: "CREATE 执行成功"
|
||||
- when: { handler_result: FAILED }
|
||||
to: S12C
|
||||
emit:
|
||||
note: "CREATE 执行失败"
|
||||
|
||||
E40U_START:
|
||||
stage: sync_execute
|
||||
desc: "UPDATE 执行入口：进入异步执行中"
|
||||
from: [S07]
|
||||
outcomes:
|
||||
- when: { handler_result: IN_PROGRESS }
|
||||
to: S10U
|
||||
emit:
|
||||
note: "UPDATE 进入异步执行中"
|
||||
|
||||
E40U:
|
||||
stage: sync_execute
|
||||
desc: "UPDATE 执行结果回写：进行中/成功/失败/跳过"
|
||||
from: [S10U]
|
||||
outcomes:
|
||||
- when: { handler_result: SUCCESS }
|
||||
to: S11U
|
||||
emit:
|
||||
note: "UPDATE 执行成功"
|
||||
- when: { handler_result: FAILED }
|
||||
to: S12U
|
||||
emit:
|
||||
note: "UPDATE 执行失败"
|
||||
- when: { handler_result: SKIPPED }
|
||||
to: S14
|
||||
emit:
|
||||
note: "更新降级并跳过"
|
||||
|
||||
E40D_START:
|
||||
stage: sync_execute
|
||||
desc: "DELETE 执行入口：进入异步执行中"
|
||||
from: [S09]
|
||||
outcomes:
|
||||
- when: { handler_result: IN_PROGRESS }
|
||||
to: S10D
|
||||
emit:
|
||||
note: "DELETE 进入异步执行中"
|
||||
|
||||
E40D:
|
||||
stage: sync_execute
|
||||
desc: "DELETE 执行结果回写：进行中/成功/失败"
|
||||
from: [S10D]
|
||||
outcomes:
|
||||
- when: { handler_result: SUCCESS }
|
||||
to: S11D
|
||||
emit:
|
||||
note: "DELETE 执行成功"
|
||||
delete_node: true
|
||||
- when: { handler_result: FAILED }
|
||||
to: S12D
|
||||
emit:
|
||||
note: "DELETE 执行失败"
|
||||
|
||||
invariants:
|
||||
INV-1_UNCHECKED_ACTION_NONE:
|
||||
desc: "UNCHECKED 不允许有动作"
|
||||
when: { binding_status: UNCHECKED }
|
||||
require: { action: NONE }
|
||||
|
||||
INV-2_BLOCKING_NO_ACTION:
|
||||
desc: "阻断态不允许执行动作"
|
||||
when_any:
|
||||
- { binding_status: ABNORMAL }
|
||||
- { binding_status: WARNING }
|
||||
- { binding_status: DEPENDENCY_ERROR }
|
||||
require: { action: NONE }
|
||||
|
||||
INV-3_PRECOND_IMPLIES_UPDATE:
|
||||
desc: "PRECONDITION_FAILED 必须伴随 action=UPDATE"
|
||||
when: { status: PRECONDITION_FAILED }
|
||||
require: { action: UPDATE }
|
||||
@@ -0,0 +1,13 @@
|
||||
from typing import Any, Dict, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
|
||||
class PersistConfig(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", validate_assignment=True)
|
||||
|
||||
backend: Literal["sqlite", "sqlalchemy", "custom"] = "sqlite"
|
||||
db_path: str
|
||||
backend_options: Dict[str, Any] = Field(default_factory=dict)
|
||||
enable: bool = True
|
||||
wipe_on_start: bool = False
|
||||
@@ -0,0 +1,48 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from .datasource_config import DataSourceConfig
|
||||
from .logging_config import LoggingConfig
|
||||
from .persist_config import PersistConfig
|
||||
from .strategy_config import StrategyRuntimeConfig
|
||||
|
||||
|
||||
DEFAULT_NODE_TYPES: List[str] = [
|
||||
"company",
|
||||
"supplier",
|
||||
"user",
|
||||
"project",
|
||||
"contract",
|
||||
"contract_change",
|
||||
"construction_task",
|
||||
"construction_task_detail",
|
||||
"material",
|
||||
"material_detail",
|
||||
"contract_settlement",
|
||||
"contract_settlement_detail",
|
||||
"personnel",
|
||||
"personnel_detail",
|
||||
"preparation",
|
||||
"production",
|
||||
"lar",
|
||||
"lar_change",
|
||||
"project_detail_data",
|
||||
"units",
|
||||
]
|
||||
|
||||
|
||||
class PipelineRunConfig(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", validate_assignment=True)
|
||||
|
||||
node_types: List[str] = Field(default_factory=lambda: list(DEFAULT_NODE_TYPES))
|
||||
target_project_ids: List[str] = Field(default_factory=list)
|
||||
|
||||
local_datasource: DataSourceConfig
|
||||
remote_datasource: DataSourceConfig
|
||||
|
||||
strategies: List[StrategyRuntimeConfig] = Field(default_factory=list)
|
||||
persist: PersistConfig
|
||||
logging: LoggingConfig = Field(default_factory=LoggingConfig)
|
||||
@@ -0,0 +1,86 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
PRESET_FILES: Dict[str, str] = {
|
||||
"jsonl_sm": "jsonl_to_jsonl",
|
||||
"simple_sm": "jsonl_to_api",
|
||||
}
|
||||
|
||||
|
||||
def _run_profiles_dir() -> Path:
|
||||
return Path(__file__).resolve().parents[2] / "run_profiles"
|
||||
|
||||
|
||||
def _preset_templates_dir() -> Path:
|
||||
return _run_profiles_dir() / "preset"
|
||||
|
||||
|
||||
def _replace_placeholders(value: Any, project_root: Path) -> Any:
|
||||
if isinstance(value, str):
|
||||
return value.replace("${PROJECT_ROOT}", str(project_root))
|
||||
if isinstance(value, dict):
|
||||
return {k: _replace_placeholders(v, project_root) for k, v in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [_replace_placeholders(v, project_root) for v in value]
|
||||
return value
|
||||
|
||||
|
||||
def list_preset_names() -> List[str]:
|
||||
return sorted(PRESET_FILES.keys())
|
||||
|
||||
|
||||
def get_preset_file_candidates(preset_name: str) -> List[Path]:
|
||||
if preset_name not in PRESET_FILES:
|
||||
raise ValueError(f"Unknown preset: {preset_name}. Available: {list_preset_names()}")
|
||||
|
||||
base = PRESET_FILES[preset_name]
|
||||
run_profiles_root = _run_profiles_dir()
|
||||
preset_templates_root = _preset_templates_dir()
|
||||
return [
|
||||
run_profiles_root / f"{base}.local.yaml",
|
||||
run_profiles_root / f"{base}.local.yml",
|
||||
run_profiles_root / f"{base}.local.json",
|
||||
preset_templates_root / f"{base}.default.yaml",
|
||||
preset_templates_root / f"{base}.default.yml",
|
||||
preset_templates_root / f"{base}.default.json",
|
||||
]
|
||||
|
||||
|
||||
def resolve_preset_file(preset_name: str) -> Tuple[Path, bool]:
|
||||
candidates = get_preset_file_candidates(preset_name)
|
||||
for idx, path in enumerate(candidates):
|
||||
if path.exists() and path.is_file():
|
||||
return path, idx > 0
|
||||
raise FileNotFoundError(
|
||||
f"No preset file found for {preset_name}. Tried: {[str(p) for p in candidates]}"
|
||||
)
|
||||
|
||||
|
||||
def load_overrides_from_file(file_path: Path, project_root: Path) -> Dict[str, Any]:
|
||||
suffix = file_path.suffix.lower()
|
||||
text = file_path.read_text(encoding="utf-8")
|
||||
|
||||
if suffix in {".yaml", ".yml"}:
|
||||
try:
|
||||
import yaml # type: ignore
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
"YAML config requires PyYAML. Install with: pip install pyyaml"
|
||||
) from exc
|
||||
payload = yaml.safe_load(text)
|
||||
else:
|
||||
payload = json.loads(text)
|
||||
|
||||
resolved = _replace_placeholders(payload, project_root)
|
||||
if not isinstance(resolved, dict):
|
||||
raise ValueError(f"Preset file must contain object mapping: {file_path}")
|
||||
return resolved
|
||||
|
||||
|
||||
def load_named_preset_overrides(project_root: Path, preset_name: str) -> Tuple[Dict[str, Any], Path, bool]:
|
||||
file_path, used_default = resolve_preset_file(preset_name)
|
||||
overrides = load_overrides_from_file(file_path, project_root)
|
||||
return overrides, file_path, used_default
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Compatibility layer for legacy imports.
|
||||
|
||||
Prefer importing from `sync_state_machine.config` directly.
|
||||
"""
|
||||
|
||||
from .datasource_config import ApiDataSourceConfig, JsonlDataSourceConfig, DataSourceConfig
|
||||
from .persist_config import PersistConfig
|
||||
from .logging_config import LoggingConfig
|
||||
from .strategy_config import (
|
||||
OrphanAction,
|
||||
UpdateDirection,
|
||||
StrategyConfig,
|
||||
StrategyRuntimeConfig,
|
||||
ConfigPresets,
|
||||
)
|
||||
from .pipeline_config import PipelineRunConfig, DEFAULT_NODE_TYPES
|
||||
from .builder import (
|
||||
build_default_config,
|
||||
build_config,
|
||||
apply_config_overrides,
|
||||
apply_env_overrides,
|
||||
config_snapshot,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"ApiDataSourceConfig",
|
||||
"JsonlDataSourceConfig",
|
||||
"DataSourceConfig",
|
||||
"PersistConfig",
|
||||
"LoggingConfig",
|
||||
"OrphanAction",
|
||||
"UpdateDirection",
|
||||
"StrategyConfig",
|
||||
"StrategyRuntimeConfig",
|
||||
"ConfigPresets",
|
||||
"PipelineRunConfig",
|
||||
"DEFAULT_NODE_TYPES",
|
||||
"build_default_config",
|
||||
"build_config",
|
||||
"apply_config_overrides",
|
||||
"apply_env_overrides",
|
||||
"config_snapshot",
|
||||
]
|
||||
@@ -0,0 +1,113 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import Enum
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
|
||||
class OrphanAction(str, Enum):
|
||||
CREATE_REMOTE = "create_remote"
|
||||
CREATE_LOCAL = "create_local"
|
||||
DELETE_LOCAL = "delete_local"
|
||||
DELETE_REMOTE = "delete_remote"
|
||||
NONE = "none"
|
||||
|
||||
|
||||
class UpdateDirection(str, Enum):
|
||||
PUSH = "push"
|
||||
PULL = "pull"
|
||||
BIDIRECTIONAL = "bidirectional"
|
||||
NONE = "none"
|
||||
|
||||
|
||||
class PreBindDataIdPair(BaseModel):
|
||||
model_config = ConfigDict(validate_assignment=True, extra="forbid")
|
||||
|
||||
local_data_id: str
|
||||
remote_data_id: str
|
||||
|
||||
|
||||
class StrategyConfig(BaseModel):
|
||||
model_config = ConfigDict(validate_assignment=True, extra="forbid")
|
||||
|
||||
auto_bind: bool = True
|
||||
auto_bind_fields: List[str] = Field(default_factory=list)
|
||||
pre_bind_data_id: List[PreBindDataIdPair] = Field(default_factory=list)
|
||||
depend_fields: Dict[str, str] = Field(default_factory=dict)
|
||||
|
||||
local_orphan_action: OrphanAction = OrphanAction.NONE
|
||||
remote_orphan_action: OrphanAction = OrphanAction.NONE
|
||||
update_direction: UpdateDirection = UpdateDirection.PUSH
|
||||
|
||||
# 字段级同步方向覆盖。
|
||||
# field_direction_key: 节点 data 中用作查表键的字段名，如 "key"。
|
||||
# field_direction_overrides: 字段值 → UpdateDirection 的映射。
|
||||
# 未命中映射的节点沿用 update_direction 作为默认方向。
|
||||
field_direction_key: Optional[str] = None
|
||||
field_direction_overrides: Dict[str, UpdateDirection] = Field(default_factory=dict)
|
||||
|
||||
# post-check 一致性比较时，忽略列表型字段内各元素的特定子键。
|
||||
# 格式: {字段名: [子键, ...]}，如 {"plan_node": ["is_audit"]}。
|
||||
# 用于排除后端硬写死、永远与本地不一致的字段，避免误报。
|
||||
post_check_ignore_list_item_fields: Dict[str, List[str]] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class StrategyRuntimeConfig(BaseModel):
|
||||
"""Pipeline 层策略运行配置（按 node_type）。"""
|
||||
|
||||
model_config = ConfigDict(validate_assignment=True, extra="forbid")
|
||||
|
||||
node_type: str
|
||||
config: StrategyConfig = Field(default_factory=StrategyConfig)
|
||||
|
||||
|
||||
class ConfigPresets:
|
||||
@staticmethod
|
||||
def first_sync() -> StrategyConfig:
|
||||
return StrategyConfig(
|
||||
local_orphan_action=OrphanAction.CREATE_REMOTE,
|
||||
remote_orphan_action=OrphanAction.CREATE_LOCAL,
|
||||
update_direction=UpdateDirection.PUSH,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def daily_sync() -> StrategyConfig:
|
||||
return ConfigPresets.push_only()
|
||||
|
||||
@staticmethod
|
||||
def repair_sync() -> StrategyConfig:
|
||||
return StrategyConfig(
|
||||
local_orphan_action=OrphanAction.NONE,
|
||||
remote_orphan_action=OrphanAction.NONE,
|
||||
update_direction=UpdateDirection.NONE,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def pull_only() -> StrategyConfig:
|
||||
return StrategyConfig(
|
||||
local_orphan_action=OrphanAction.NONE,
|
||||
remote_orphan_action=OrphanAction.CREATE_LOCAL,
|
||||
update_direction=UpdateDirection.PULL,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def push_only() -> StrategyConfig:
|
||||
return StrategyConfig(
|
||||
local_orphan_action=OrphanAction.CREATE_REMOTE,
|
||||
remote_orphan_action=OrphanAction.NONE,
|
||||
update_direction=UpdateDirection.PUSH,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_preset(name: str) -> StrategyConfig:
|
||||
presets = {
|
||||
"first_sync": ConfigPresets.first_sync,
|
||||
"daily_sync": ConfigPresets.daily_sync,
|
||||
"repair_sync": ConfigPresets.repair_sync,
|
||||
"pull_only": ConfigPresets.pull_only,
|
||||
"push_only": ConfigPresets.push_only,
|
||||
}
|
||||
if name not in presets:
|
||||
raise ValueError(f"Unknown preset: {name}. Available: {list(presets.keys())}")
|
||||
return presets[name]()
|
||||
@@ -0,0 +1,252 @@
|
||||
# DataSource Layer Implementation
|
||||
|
||||
## Overview
|
||||
|
||||
This document describes the new datasource layer for `sync_system_new`, which is completely decoupled from the sync strategy and state machine logic.
|
||||
|
||||
## Architecture
|
||||
|
||||
The datasource layer follows a pure execution model:
|
||||
|
||||
1. **Load**: Reads data from backend and creates `SyncNode` instances
|
||||
2. **Save**: Executes actions (CREATE/UPDATE/DELETE) based on `node.action`
|
||||
3. **Update Status**: Sets `node.status` to SUCCESS or FAILED after execution
|
||||
|
||||
### Key Principles
|
||||
|
||||
- **No Strategy Logic**: Datasource doesn't make decisions about what to sync
|
||||
- **State-Driven**: Only reads `node.action` and `node.status` to determine behavior
|
||||
- **No ID Resolution**: IDs are already resolved by the strategy layer
|
||||
- **Backend Agnostic**: Abstract base class supports multiple backends
|
||||
|
||||
## Components
|
||||
|
||||
### 1. Base Protocol (`base.py`)
|
||||
|
||||
#### `DataSourceProtocol`
|
||||
|
||||
Protocol defining the interface for all datasource implementations:
|
||||
|
||||
```python
|
||||
async def load_nodes(
|
||||
node_type: str,
|
||||
collection: DataCollection,
|
||||
**filters: Any
|
||||
) -> None:
|
||||
"""Load data from backend and create SyncNodes"""
|
||||
|
||||
async def save_nodes(
|
||||
node_type: str,
|
||||
collection: DataCollection
|
||||
) -> None:
|
||||
"""Execute pending actions for nodes"""
|
||||
```
|
||||
|
||||
#### `BaseDataSource`
|
||||
|
||||
Abstract base class providing common functionality:
|
||||
|
||||
- `load_nodes`: Creates SyncNodes from raw data
|
||||
- `save_nodes`: Executes actions and updates status
|
||||
- `_handle_create/update/delete`: Action-specific handlers
|
||||
|
||||
Subclasses implement:
|
||||
- `_load_raw_data`: Load from backend
|
||||
- `_create_item`: Execute CREATE
|
||||
- `_update_item`: Execute UPDATE
|
||||
- `_delete_item`: Execute DELETE
|
||||
- `_generate_id`: Generate new IDs
|
||||
|
||||
### 2. JSONL Implementation (`jsonl_datasource.py`)
|
||||
|
||||
#### `JsonlDataSource`
|
||||
|
||||
JSONL-based datasource for local testing:
|
||||
|
||||
**Features:**
|
||||
- Loads `.jsonl` files into memory
|
||||
- Generates UUIDs for new items
|
||||
- Supports filtering during load
|
||||
- Supports read-only mode
|
||||
- Persists changes back to files
|
||||
|
||||
**File Structure:**
|
||||
```
|
||||
data/
|
||||
├── contract_1.jsonl
|
||||
├── project_1.jsonl
|
||||
└── ...
|
||||
```
|
||||
|
||||
**Usage:**
|
||||
```python
|
||||
datasource = JsonlDataSource(Path("data/"))
|
||||
|
||||
# Load nodes
|
||||
await datasource.load_nodes("contract", collection, project_id="P1")
|
||||
|
||||
# Execute actions
|
||||
await datasource.save_nodes("contract", collection)
|
||||
|
||||
# Persist to disk
|
||||
await datasource.save_all()
|
||||
```
|
||||
|
||||
### 3. Domain Handlers
|
||||
|
||||
Domain-specific handlers provide business logic (optional):
|
||||
|
||||
**Example: `domain/contract/jsonl_handler.py`**
|
||||
|
||||
```python
|
||||
class ContractJsonlHandler:
|
||||
@staticmethod
|
||||
def validate_create(data: Any) -> None:
|
||||
"""Validate before create"""
|
||||
|
||||
@staticmethod
|
||||
def transform_for_create(data: Any) -> Dict[str, Any]:
|
||||
"""Transform data before create"""
|
||||
```
|
||||
|
||||
## Node Lifecycle
|
||||
|
||||
### Loading Phase
|
||||
|
||||
1. Datasource reads raw data from backend
|
||||
2. Creates `SyncNode` for each item:
|
||||
- `node_id`: Unique session ID (e.g., "contract_C1")
|
||||
- `data_id`: Backend ID (e.g., "C1")
|
||||
- `origin_data`: Raw data from backend
|
||||
- `data`: Copy of origin_data (may be modified by strategy)
|
||||
- `action`: NONE (initial)
|
||||
- `status`: PENDING (initial)
|
||||
|
||||
### Execution Phase
|
||||
|
||||
For each node with `status == PENDING` and `action != NONE`:
|
||||
|
||||
1. **CREATE**:
|
||||
- Generate new ID via `_generate_id`
|
||||
- Call `_create_item` with new ID and data
|
||||
- Set `node.data_id` to new ID
|
||||
- Set `node.status` to SUCCESS
|
||||
|
||||
2. **UPDATE**:
|
||||
- Call `_update_item` with existing `data_id` and data
|
||||
- Set `node.status` to SUCCESS
|
||||
|
||||
3. **DELETE**:
|
||||
- Call `_delete_item` with `data_id`
|
||||
- Set `node.status` to SUCCESS
|
||||
|
||||
4. **Error Handling**:
|
||||
- On exception: Set `node.status` to FAILED
|
||||
- Set `node.error` to exception message
|
||||
|
||||
## Integration with Sync System
|
||||
|
||||
The datasource is used by the pipeline layer:
|
||||
|
||||
```python
|
||||
# 1. Load data
|
||||
local_ds = JsonlDataSource(Path("local_data/"))
|
||||
remote_ds = JsonlDataSource(Path("remote_data/"))
|
||||
|
||||
local_collection = DataCollection("local")
|
||||
remote_collection = DataCollection("remote")
|
||||
|
||||
await local_ds.load_nodes("contract", local_collection)
|
||||
await remote_ds.load_nodes("contract", remote_collection)
|
||||
|
||||
# 2. Strategy layer determines actions
|
||||
strategy = ContractSyncStrategy(...)
|
||||
await strategy.bind(...) # Sets node.action
|
||||
|
||||
# 3. Execute actions
|
||||
await remote_ds.save_nodes("contract", remote_collection)
|
||||
await remote_ds.save_all()
|
||||
```
|
||||
|
||||
## Testing
|
||||
|
||||
Comprehensive test suite in `tests/test_datasource.py`:
|
||||
|
||||
- Basic loading and node creation
|
||||
- Filtering during load
|
||||
- CREATE/UPDATE/DELETE actions
|
||||
- Error handling and status updates
|
||||
- Read-only mode
|
||||
- Multiple node types
|
||||
|
||||
Run tests:
|
||||
```bash
|
||||
python -m pytest sync_system_new/tests/test_datasource.py -v
|
||||
```
|
||||
|
||||
## Future Extensions
|
||||
|
||||
### API DataSource
|
||||
|
||||
For production use, implement `ApiDataSource`:
|
||||
|
||||
```python
|
||||
class ApiDataSource(BaseDataSource):
|
||||
async def _load_raw_data(self, node_type: str, **filters):
|
||||
# Call REST API
|
||||
|
||||
async def _create_item(self, node_type: str, item_id: str, data):
|
||||
# POST request
|
||||
|
||||
async def _generate_id(self, node_type: str, data):
|
||||
# Return None - server generates ID
|
||||
```
|
||||
|
||||
### Database DataSource
|
||||
|
||||
For SQL databases:
|
||||
|
||||
```python
|
||||
class DatabaseDataSource(BaseDataSource):
|
||||
async def _load_raw_data(self, node_type: str, **filters):
|
||||
# SELECT query
|
||||
|
||||
async def _create_item(self, node_type: str, item_id: str, data):
|
||||
# INSERT query
|
||||
```
|
||||
|
||||
## Comparison with Legacy System
|
||||
|
||||
### Legacy System (`sync_system/datasource/`)
|
||||
|
||||
- Tightly coupled with RepositoryProtocol
|
||||
- Mixed concerns (data access + business logic)
|
||||
- Schema validation in datasource layer
|
||||
- Different interface for each entity type
|
||||
|
||||
### New System (`sync_system_new/datasource/`)
|
||||
|
||||
- Decoupled from strategy and state machine
|
||||
- Pure data access (business logic in domain handlers)
|
||||
- Generic interface for all entity types
|
||||
- Direct integration with SyncNode and DataCollection
|
||||
|
||||
## Key Differences
|
||||
|
||||
| Aspect | Legacy | New |
|
||||
|--------|--------|-----|
|
||||
| Coupling | Tight with sync policies | Decoupled from sync logic |
|
||||
| Interface | Repository per entity | Generic load/save |
|
||||
| ID Handling | Mixed responsibilities | IDs already resolved |
|
||||
| State Management | Mixed with data access | SyncNode-based |
|
||||
| Testing | Complex setup | Simple, focused tests |
|
||||
|
||||
## Conclusion
|
||||
|
||||
The new datasource layer provides:
|
||||
|
||||
1. **Clear Separation**: Data access is isolated from sync logic
|
||||
2. **Flexibility**: Easy to add new backend types
|
||||
3. **Testability**: Simple, focused tests
|
||||
4. **Type Safety**: Full integration with Pydantic models
|
||||
5. **Maintainability**: Single responsibility per component
|
||||
@@ -0,0 +1,41 @@
|
||||
"""
|
||||
DataSource Layer for sync_state_machine
|
||||
|
||||
DataSource 只负责底层 I/O，不关心节点类型。
|
||||
Handler 负责类型转换和业务逻辑。
|
||||
"""
|
||||
|
||||
from .datasource import BaseDataSource
|
||||
from .handler import (
|
||||
NodeHandler,
|
||||
BaseNodeHandler,
|
||||
NodeHandlerRegistry,
|
||||
ValidationResult
|
||||
)
|
||||
from .task_result import TaskResult, TaskStatus
|
||||
|
||||
# API DataSource
|
||||
from .api import ApiClient, ApiDataSource, BaseApiHandler
|
||||
|
||||
# JSONL DataSource
|
||||
from .jsonl import JsonlDataSource, BaseJsonlHandler
|
||||
|
||||
__all__ = [
|
||||
# 基础类
|
||||
"BaseDataSource",
|
||||
"NodeHandler",
|
||||
"BaseNodeHandler",
|
||||
"NodeHandlerRegistry",
|
||||
"TaskResult",
|
||||
"TaskStatus",
|
||||
"ValidationResult",
|
||||
|
||||
# API DataSource
|
||||
"ApiClient",
|
||||
"ApiDataSource",
|
||||
"BaseApiHandler",
|
||||
|
||||
# JSONL DataSource
|
||||
"JsonlDataSource",
|
||||
"BaseJsonlHandler",
|
||||
]
|
||||
@@ -0,0 +1,15 @@
|
||||
"""
|
||||
API DataSource 模块
|
||||
|
||||
提供基于 HTTP API 的数据源实现。
|
||||
"""
|
||||
|
||||
from .client import ApiClient
|
||||
from .datasource import ApiDataSource
|
||||
from .handler import BaseApiHandler
|
||||
|
||||
__all__ = [
|
||||
"ApiClient",
|
||||
"ApiDataSource",
|
||||
"BaseApiHandler",
|
||||
]
|
||||
@@ -0,0 +1,627 @@
|
||||
"""
|
||||
ApiClient - HTTP 通信层
|
||||
|
||||
职责:
|
||||
- 封装所有 HTTP 通信逻辑
|
||||
- 管理 httpx.AsyncClient，维护连接池和 session
|
||||
- 生成请求签名（timestamp, nonce, sign）
|
||||
- 统一请求入口，自动注入 uid
|
||||
- 批量 Push Log 查询
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import time
|
||||
import uuid
|
||||
import json as json_lib
|
||||
import logging
|
||||
import asyncio
|
||||
from typing import Dict, List, Any, Optional
|
||||
from collections import defaultdict, deque
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
import httpx
|
||||
|
||||
from schemas.common.push_system import PushIdsSchema
|
||||
|
||||
|
||||
def _build_default_push_steps() -> List[Dict[str, str]]:
|
||||
"""构造最小审批步骤字段，满足第三方 v2 push schema。"""
|
||||
return []
|
||||
|
||||
|
||||
class ApiClient:
|
||||
"""API HTTP 客户端"""
|
||||
_TRACE_TEXT_MAX_LEN = 1000
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
base_url: str,
|
||||
uid: str,
|
||||
secret: str,
|
||||
debug: bool = False,
|
||||
log_file: Optional[Path] = None,
|
||||
rate_limit_max_requests: int = 30,
|
||||
rate_limit_window_seconds: float = 10.0,
|
||||
):
|
||||
"""
|
||||
初始化 API 客户端
|
||||
|
||||
Args:
|
||||
base_url: API 基础 URL（如 https://api.example.com）
|
||||
uid: 用户标识
|
||||
secret: 签名密钥
|
||||
debug: 是否启用调试模式（打印请求详情）
|
||||
log_file: 日志文件路径（可选）
|
||||
rate_limit_max_requests: 限流窗口内最大请求数（<=0 表示禁用）
|
||||
rate_limit_window_seconds: 限流窗口秒数（<=0 表示禁用）
|
||||
"""
|
||||
self._base_url = base_url.rstrip('/')
|
||||
self._uid = uid
|
||||
self._secret = secret
|
||||
self._client: Optional[httpx.AsyncClient] = None
|
||||
self._debug = debug
|
||||
self._log_file = log_file
|
||||
self._trace_seq = 0
|
||||
self._trace_by_push_id: Dict[str, deque[Dict[str, Any]]] = defaultdict(deque)
|
||||
self._trace_by_biz_id: Dict[str, deque[Dict[str, Any]]] = defaultdict(deque)
|
||||
self._trace_by_op: Dict[str, deque[Dict[str, Any]]] = defaultdict(deque)
|
||||
self._load_trace_by_item: Dict[tuple[str, str], deque[Dict[str, Any]]] = defaultdict(deque)
|
||||
self._logger = logging.getLogger(__name__)
|
||||
self._rate_limit_max_requests = int(rate_limit_max_requests)
|
||||
self._rate_limit_window_seconds = float(rate_limit_window_seconds)
|
||||
self._rate_limit_lock = asyncio.Lock()
|
||||
self._rate_limit_timestamps: deque[float] = deque()
|
||||
|
||||
def _is_rate_limit_enabled(self) -> bool:
|
||||
return self._rate_limit_max_requests > 0 and self._rate_limit_window_seconds > 0
|
||||
|
||||
async def _acquire_rate_limit_slot(self) -> None:
|
||||
if not self._is_rate_limit_enabled():
|
||||
return
|
||||
|
||||
while True:
|
||||
wait_seconds = 0.0
|
||||
async with self._rate_limit_lock:
|
||||
now = time.monotonic()
|
||||
window_start = now - self._rate_limit_window_seconds
|
||||
while self._rate_limit_timestamps and self._rate_limit_timestamps[0] <= window_start:
|
||||
self._rate_limit_timestamps.popleft()
|
||||
|
||||
if len(self._rate_limit_timestamps) < self._rate_limit_max_requests:
|
||||
self._rate_limit_timestamps.append(now)
|
||||
return
|
||||
|
||||
oldest = self._rate_limit_timestamps[0]
|
||||
wait_seconds = max(0.001, self._rate_limit_window_seconds - (now - oldest))
|
||||
|
||||
await asyncio.sleep(wait_seconds)
|
||||
|
||||
async def initialize(self):
|
||||
"""初始化 HTTP 客户端"""
|
||||
if self._client is None:
|
||||
# trust_env=False 阻止从环境变量读取代理设置（避免 localhost 走代理导致超时）
|
||||
self._client = httpx.AsyncClient(timeout=300.0, trust_env=False)
|
||||
|
||||
async def close(self):
|
||||
"""关闭 HTTP 客户端"""
|
||||
if self._client:
|
||||
await self._client.aclose()
|
||||
self._client = None
|
||||
|
||||
def _log(self, message: str, level: str = "DEBUG"):
|
||||
"""
|
||||
统一日志输出
|
||||
|
||||
Args:
|
||||
message: 日志消息
|
||||
level: 日志级别（INFO/ERROR）
|
||||
"""
|
||||
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
log_line = f"[{timestamp}] [{level}] {message}"
|
||||
|
||||
level_num = logging.getLevelNamesMapping().get(level.upper(), logging.DEBUG)
|
||||
self._logger.log(level_num, log_line)
|
||||
|
||||
# 文件输出
|
||||
if self._log_file:
|
||||
self._log_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(self._log_file, 'a', encoding='utf-8') as f:
|
||||
f.write(log_line + '\n')
|
||||
|
||||
def _get_client(self) -> httpx.AsyncClient:
|
||||
"""获取 HTTP 客户端实例"""
|
||||
if self._client is None:
|
||||
raise RuntimeError("ApiClient not initialized. Call 'await client.initialize()' first.")
|
||||
return self._client
|
||||
|
||||
def _derive_op_kind(self, method: str, endpoint: str) -> str:
|
||||
endpoint_norm = endpoint.lower().strip()
|
||||
if endpoint_norm == "/push/result":
|
||||
return "poll"
|
||||
method_upper = method.upper()
|
||||
if method_upper == "GET":
|
||||
if endpoint_norm.endswith("/list"):
|
||||
return "load"
|
||||
return "get"
|
||||
if endpoint_norm.endswith("/create"):
|
||||
return "create"
|
||||
if endpoint_norm.endswith("/delete"):
|
||||
return "delete"
|
||||
if endpoint_norm.endswith("/update") or endpoint_norm.endswith("/plan"):
|
||||
return "update"
|
||||
if method_upper == "POST":
|
||||
return "create"
|
||||
if method_upper == "PUT":
|
||||
return "update"
|
||||
if method_upper == "DELETE":
|
||||
return "delete"
|
||||
return method.lower()
|
||||
|
||||
def _derive_node_type_from_list_endpoint(self, endpoint: str) -> Optional[str]:
|
||||
endpoint_norm = endpoint.strip("/")
|
||||
if not endpoint_norm.endswith("/list"):
|
||||
return None
|
||||
prefix = endpoint_norm[:-5]
|
||||
if not prefix:
|
||||
return None
|
||||
return prefix.replace("/", "_")
|
||||
|
||||
def _record_trace(
|
||||
self,
|
||||
*,
|
||||
method: str,
|
||||
endpoint: str,
|
||||
url: str,
|
||||
request_params: Optional[Dict[str, Any]],
|
||||
request_payload: Optional[Dict[str, Any]],
|
||||
response: Optional[Dict[str, Any]],
|
||||
elapsed_ms: int,
|
||||
) -> None:
|
||||
self._trace_seq += 1
|
||||
op_kind = self._derive_op_kind(method, endpoint)
|
||||
payload = request_payload or {}
|
||||
response_view = self._build_trace_response_view(op_kind=op_kind, response=response)
|
||||
params_text = self._to_trace_text(request_params or {}, max_len=self._TRACE_TEXT_MAX_LEN)
|
||||
payload_text = self._to_trace_text(payload, max_len=self._TRACE_TEXT_MAX_LEN)
|
||||
response_text = self._to_trace_text(response_view, max_len=self._TRACE_TEXT_MAX_LEN)
|
||||
|
||||
trace = {
|
||||
"trace_id": self._trace_seq,
|
||||
"op": op_kind,
|
||||
"method": method.upper(),
|
||||
"endpoint": endpoint,
|
||||
"url": url,
|
||||
"params": request_params or {},
|
||||
"payload": payload,
|
||||
"response": response_view,
|
||||
"params_text": params_text,
|
||||
"payload_text": payload_text,
|
||||
"response_text": response_text,
|
||||
"elapsed_ms": elapsed_ms,
|
||||
"ts": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
}
|
||||
|
||||
push_id = payload.get("push_id")
|
||||
biz_id = payload.get("biz_id")
|
||||
if isinstance(push_id, str) and push_id:
|
||||
self._trace_by_push_id[push_id].append(trace)
|
||||
if isinstance(biz_id, str) and biz_id:
|
||||
self._trace_by_biz_id[biz_id].append(trace)
|
||||
self._trace_by_op[op_kind].append(trace)
|
||||
|
||||
if op_kind == "load":
|
||||
node_type = self._derive_node_type_from_list_endpoint(endpoint)
|
||||
data = (response or {}).get("data") if isinstance(response, dict) else None
|
||||
if isinstance(data, list):
|
||||
items = data
|
||||
elif isinstance(data, dict):
|
||||
items = data.get("list", [])
|
||||
else:
|
||||
items = []
|
||||
if node_type and isinstance(items, list):
|
||||
for item in items:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
item_id = item.get("id") or item.get("_id")
|
||||
if not item_id:
|
||||
continue
|
||||
self._load_trace_by_item[(node_type, str(item_id))].append(trace)
|
||||
|
||||
def _to_trace_text(self, value: Any, *, max_len: int) -> str:
|
||||
try:
|
||||
text = json_lib.dumps(value, ensure_ascii=False, separators=(",", ":"))
|
||||
except Exception:
|
||||
text = str(value)
|
||||
if len(text) > max_len:
|
||||
return f"{text[:max_len]}...(已截断)"
|
||||
return text
|
||||
|
||||
def _build_trace_response_view(self, *, op_kind: str, response: Optional[Dict[str, Any]]) -> Any:
|
||||
return response or {}
|
||||
|
||||
def consume_traces_by_push_id(self, push_id: str) -> List[Dict[str, Any]]:
|
||||
if not push_id:
|
||||
return []
|
||||
bucket = self._trace_by_push_id.get(push_id)
|
||||
if not bucket:
|
||||
return []
|
||||
traces = list(bucket)
|
||||
bucket.clear()
|
||||
return traces
|
||||
|
||||
def consume_latest_trace_by_biz_id(self, biz_id: str, op_kind: Optional[str] = None) -> Optional[Dict[str, Any]]:
|
||||
if not biz_id:
|
||||
return None
|
||||
bucket = self._trace_by_biz_id.get(biz_id)
|
||||
if not bucket:
|
||||
return None
|
||||
if op_kind is None:
|
||||
return bucket.pop() if bucket else None
|
||||
for index in range(len(bucket) - 1, -1, -1):
|
||||
trace = bucket[index]
|
||||
if trace.get("op") == op_kind:
|
||||
del bucket[index]
|
||||
return trace
|
||||
return None
|
||||
|
||||
def peek_latest_trace_by_biz_id(self, biz_id: str, op_kind: Optional[str] = None) -> Optional[Dict[str, Any]]:
|
||||
if not biz_id:
|
||||
return None
|
||||
bucket = self._trace_by_biz_id.get(biz_id)
|
||||
if not bucket:
|
||||
return None
|
||||
if op_kind is None:
|
||||
return bucket[-1] if bucket else None
|
||||
for index in range(len(bucket) - 1, -1, -1):
|
||||
trace = bucket[index]
|
||||
if trace.get("op") == op_kind:
|
||||
return trace
|
||||
return None
|
||||
|
||||
def consume_latest_trace_by_op(self, op_kind: str) -> Optional[Dict[str, Any]]:
|
||||
bucket = self._trace_by_op.get(op_kind)
|
||||
if not bucket:
|
||||
return None
|
||||
return bucket.pop() if bucket else None
|
||||
|
||||
def peek_latest_trace_by_op(self, op_kind: str) -> Optional[Dict[str, Any]]:
|
||||
bucket = self._trace_by_op.get(op_kind)
|
||||
if not bucket:
|
||||
return None
|
||||
return bucket[-1] if bucket else None
|
||||
|
||||
def consume_load_traces(self, node_type: str, data_id: str) -> List[Dict[str, Any]]:
|
||||
key = (node_type, data_id)
|
||||
bucket = self._load_trace_by_item.get(key)
|
||||
if not bucket:
|
||||
return []
|
||||
traces = list(bucket)
|
||||
bucket.clear()
|
||||
return traces
|
||||
|
||||
def _generate_signature(self, params: Dict[str, Any]) -> Dict[str, str]:
|
||||
"""
|
||||
生成签名参数（兼容旧版签名算法）
|
||||
|
||||
Args:
|
||||
params: 请求参数
|
||||
|
||||
Returns:
|
||||
包含 timestamp, nonce, sign 的字典
|
||||
"""
|
||||
timestamp = int(time.time() * 1000) # 毫秒时间戳
|
||||
nonce = uuid.uuid4().hex[:16] # 16位随机字符串
|
||||
|
||||
# 构建签名参数
|
||||
sign_params = params.copy()
|
||||
sign_params['timestamp'] = timestamp
|
||||
sign_params['nonce'] = nonce
|
||||
|
||||
# 按key排序
|
||||
sorted_items = sorted(sign_params.items())
|
||||
|
||||
# 构建签名字符串
|
||||
items = []
|
||||
for k, v in sorted_items:
|
||||
if v is None:
|
||||
v_str = "null"
|
||||
elif isinstance(v, bool):
|
||||
v_str = "true" if v else "false"
|
||||
elif isinstance(v, str):
|
||||
v_str = v
|
||||
elif isinstance(v, (int, float)):
|
||||
v_str = str(v)
|
||||
elif isinstance(v, (list, dict)):
|
||||
v_str = json_lib.dumps(v, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
|
||||
else:
|
||||
v_str = str(v)
|
||||
items.append(f"{k}={v_str}")
|
||||
|
||||
sign_str = "&".join(items)
|
||||
|
||||
# 使用 HMAC-SHA256 生成签名
|
||||
sign = hmac.new(self._secret.encode("utf-8"), sign_str.encode("utf-8"), hashlib.sha256).hexdigest()
|
||||
|
||||
return {
|
||||
'timestamp': str(timestamp),
|
||||
'nonce': nonce,
|
||||
'sign': sign
|
||||
}
|
||||
|
||||
async def request(
|
||||
self,
|
||||
method: str,
|
||||
endpoint: str,
|
||||
params: Optional[Dict[str, Any]] = None,
|
||||
data: Optional[Dict[str, Any]] = None,
|
||||
json_data: Optional[Dict[str, Any]] = None,
|
||||
json: Optional[Dict[str, Any]] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
统一请求入口，自动签名
|
||||
|
||||
Args:
|
||||
method: HTTP 方法（GET/POST/PUT/DELETE）
|
||||
endpoint: API 端点（如 /api/v2/project）
|
||||
params: URL 查询参数
|
||||
data: 请求体数据（form data）
|
||||
json_data: JSON 请求体数据
|
||||
json: JSON 请求体数据（兼容参数）
|
||||
|
||||
Returns:
|
||||
响应 JSON 数据
|
||||
|
||||
Raises:
|
||||
httpx.HTTPStatusError: HTTP 错误
|
||||
"""
|
||||
# 兼容json参数
|
||||
if json is not None and json_data is None:
|
||||
json_data = json
|
||||
|
||||
client = self._get_client()
|
||||
url = f"{self._base_url}{endpoint}"
|
||||
await self._acquire_rate_limit_slot()
|
||||
|
||||
# 记录请求信息
|
||||
self._log(f"\n{'='*60}", level="DEBUG")
|
||||
self._log(f"REQUEST: {method} {endpoint}", level="DEBUG")
|
||||
self._log(f"Full URL: {url}", level="DEBUG")
|
||||
if params:
|
||||
self._log(f"Params: {json_lib.dumps(params, ensure_ascii=False, indent=2)}", level="DEBUG")
|
||||
if data:
|
||||
self._log(f"Data: {json_lib.dumps(data, ensure_ascii=False, indent=2)}", level="DEBUG")
|
||||
if json_data:
|
||||
self._log(f"JSON: {json_lib.dumps(json_data, ensure_ascii=False, indent=2)}", level="DEBUG")
|
||||
|
||||
# 准备签名参数
|
||||
if method.upper() == 'GET':
|
||||
# GET 请求：uid 放在 params 中
|
||||
if params is None:
|
||||
params = {}
|
||||
params['uid'] = self._uid
|
||||
|
||||
# 转换所有参数值为字符串（除了 list/dict）
|
||||
params = {k: str(v) if not isinstance(v, (list, dict)) else v for k, v in params.items()}
|
||||
|
||||
sign_params = params.copy()
|
||||
# 生成签名并附加到 params
|
||||
signature = self._generate_signature(sign_params)
|
||||
params.update(signature)
|
||||
else:
|
||||
# POST/PUT/DELETE：uid 和签名放在 body 中（符合 v2 schema）
|
||||
if data is None:
|
||||
data = {}
|
||||
if 'steps' not in data:
|
||||
data['steps'] = _build_default_push_steps()
|
||||
data['uid'] = self._uid
|
||||
sign_params = data.copy()
|
||||
if params:
|
||||
sign_params.update(params)
|
||||
signature = self._generate_signature(sign_params)
|
||||
data.update(signature)
|
||||
|
||||
# 发送请求并增强异常处理
|
||||
start_time = time.time()
|
||||
try:
|
||||
response = await client.request(
|
||||
method=method,
|
||||
url=url,
|
||||
params=params,
|
||||
json=data if method.upper() in ['POST', 'PUT'] else None
|
||||
)
|
||||
|
||||
# 记录响应时间
|
||||
elapsed = time.time() - start_time
|
||||
self._log(f"Response Time: {elapsed:.3f}s", level="DEBUG")
|
||||
|
||||
# 检查 HTTP 状态
|
||||
response.raise_for_status()
|
||||
|
||||
# 记录响应
|
||||
result = response.json()
|
||||
self._log(f"Status Code: {response.status_code}", level="DEBUG")
|
||||
self._log(f"Response: {json_lib.dumps(result, ensure_ascii=False, indent=2)}", level="DEBUG")
|
||||
self._log(f"{'='*60}\n", level="DEBUG")
|
||||
|
||||
self._record_trace(
|
||||
method=method,
|
||||
endpoint=endpoint,
|
||||
url=url,
|
||||
request_params=params,
|
||||
request_payload=data if method.upper() in ['POST', 'PUT', 'DELETE'] else None,
|
||||
response=result,
|
||||
elapsed_ms=int(elapsed * 1000),
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
import traceback
|
||||
|
||||
# 记录错误
|
||||
elapsed = time.time() - start_time
|
||||
error_msg = "\n========== [ApiClient HTTP Error] ==========\n"
|
||||
error_msg += f"Request: {method} {url}\n"
|
||||
error_msg += f"Params: {params}\n"
|
||||
if method.upper() in ['POST', 'PUT']:
|
||||
error_msg += f"JSON: {data}\n"
|
||||
error_msg += f"Exception: {type(e).__name__}: {e}\n"
|
||||
error_msg += f"Elapsed: {elapsed:.3f}s\n"
|
||||
|
||||
response_obj = e.response if isinstance(e, httpx.HTTPStatusError) else None
|
||||
if response_obj is not None:
|
||||
error_msg += f"Status Code: {response_obj.status_code}\n"
|
||||
error_msg += f"Response Headers: {response_obj.headers}\n"
|
||||
try:
|
||||
error_msg += f"Response Text: {response_obj.text}\n"
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
error_msg += traceback.format_exc()
|
||||
error_msg += "============================================\n"
|
||||
|
||||
# 记录到日志文件
|
||||
self._log(error_msg, level="ERROR")
|
||||
|
||||
raise
|
||||
|
||||
async def get(self, endpoint: str, params: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
"""
|
||||
GET 请求
|
||||
|
||||
Args:
|
||||
endpoint: API 端点
|
||||
params: URL 查询参数
|
||||
|
||||
Returns:
|
||||
响应 JSON 数据
|
||||
"""
|
||||
return await self.request('GET', endpoint, params=params)
|
||||
|
||||
async def post(self, endpoint: str, data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
POST 请求，data 中已包含 push_id/biz_id（由 Handler 生成）
|
||||
|
||||
Args:
|
||||
endpoint: API 端点
|
||||
data: 请求体数据（包含业务数据和 push_id/biz_id）
|
||||
|
||||
Returns:
|
||||
响应 JSON 数据
|
||||
"""
|
||||
return await self.request('POST', endpoint, data=data)
|
||||
|
||||
async def put(self, endpoint: str, data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
PUT 请求，data 中已包含 push_id/biz_id
|
||||
|
||||
Args:
|
||||
endpoint: API 端点
|
||||
data: 请求体数据
|
||||
|
||||
Returns:
|
||||
响应 JSON 数据
|
||||
"""
|
||||
return await self.request('PUT', endpoint, data=data)
|
||||
|
||||
async def delete(self, endpoint: str, data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
DELETE 请求，data 中已包含 push_id/biz_id
|
||||
|
||||
Args:
|
||||
endpoint: API 端点
|
||||
data: 请求体数据
|
||||
|
||||
Returns:
|
||||
响应 JSON 数据
|
||||
"""
|
||||
return await self.request('DELETE', endpoint, data=data)
|
||||
|
||||
async def get_push_logs(self, push_ids: List[str]) -> Dict[str, Dict[str, Any]]:
|
||||
"""
|
||||
批量查询 Push Log
|
||||
|
||||
注意：单个查询也传单元素列表
|
||||
当前实现：用 for 循环模拟，后续等后端实现批量接口
|
||||
|
||||
Args:
|
||||
push_ids: Push ID 列表
|
||||
|
||||
Returns:
|
||||
{push_id: push_log_data} 的映射
|
||||
"""
|
||||
result: Dict[str, Dict[str, Any]] = {}
|
||||
|
||||
if not push_ids:
|
||||
return result
|
||||
|
||||
try:
|
||||
payload = PushIdsSchema(push_ids=','.join(push_ids)).model_dump(exclude_none=True)
|
||||
response = await self.request(
|
||||
method='POST',
|
||||
endpoint='/push/result',
|
||||
data=payload
|
||||
)
|
||||
except httpx.HTTPStatusError as e:
|
||||
return {push_id: {'error': str(e)} for push_id in push_ids}
|
||||
|
||||
data_list: List[Dict[str, Any]] = []
|
||||
if isinstance(response, dict):
|
||||
raw_data = response.get('data')
|
||||
if isinstance(raw_data, list):
|
||||
data_list = [item for item in raw_data if isinstance(item, dict)]
|
||||
elif isinstance(raw_data, dict):
|
||||
data_list = [raw_data]
|
||||
elif 'push_id' in response and 'status' in response:
|
||||
data_list = [response]
|
||||
|
||||
for item in data_list:
|
||||
push_id = item.get('push_id')
|
||||
if not push_id:
|
||||
continue
|
||||
|
||||
poll_trace = {
|
||||
"trace_id": f"poll-{push_id}-{int(time.time() * 1000)}",
|
||||
"op": "poll",
|
||||
"method": "POST",
|
||||
"endpoint": "/push/result",
|
||||
"url": f"{self._base_url}/push/result",
|
||||
"params": {},
|
||||
"payload": {"push_ids": push_id},
|
||||
"response": item,
|
||||
"elapsed_ms": 0,
|
||||
"ts": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
}
|
||||
self._trace_by_push_id[push_id].append(poll_trace)
|
||||
|
||||
status = (item.get('status') or '').lower()
|
||||
error_msg = item.get('error') or ''
|
||||
result_data = item.get('result') or {}
|
||||
|
||||
if status == 'success':
|
||||
# result_data 的结构是 {"code": 200, "message": "OK", "data": {"biz_id": "..."}}
|
||||
# 需要提取 result_data["data"] 作为最终的 data
|
||||
inner_data = result_data.get('data', {}) if isinstance(result_data, dict) else {}
|
||||
result[push_id] = {
|
||||
'code': 200,
|
||||
'status': 'success',
|
||||
'data': inner_data
|
||||
}
|
||||
elif status in {'failed', 'not_find'}:
|
||||
result[push_id] = {
|
||||
'code': 500,
|
||||
'status': 'failed',
|
||||
'message': error_msg or status or 'Unknown error',
|
||||
'error': error_msg or status or 'Unknown error'
|
||||
}
|
||||
else:
|
||||
result[push_id] = {
|
||||
'status': status or 'processing'
|
||||
}
|
||||
|
||||
for push_id in push_ids:
|
||||
if push_id not in result:
|
||||
result[push_id] = {'status': 'processing'}
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,264 @@
|
||||
"""
|
||||
ApiDataSource - API 数据源协调层
|
||||
|
||||
职责：
|
||||
- 持有 ApiClient 实例，提供给 Handler 使用
|
||||
- 实现 DataSource 接口（load_all, sync_all）
|
||||
- 协调多个 Handler 的执行顺序
|
||||
- 统一异常处理和日志记录
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Optional
|
||||
from pathlib import Path
|
||||
from ..datasource import BaseDataSource
|
||||
from ..handler import NodeHandler, ensure_handler_compat
|
||||
from ..task_result import TaskResult
|
||||
from .client import ApiClient
|
||||
|
||||
|
||||
class ApiDataSource(BaseDataSource):
|
||||
"""基于 API 的数据源实现"""
|
||||
_TRACE_MAX_PARAMS_LEN = 1000
|
||||
_TRACE_MAX_RESPONSE_LEN = 1000
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
base_url: str,
|
||||
uid: str,
|
||||
secret: str,
|
||||
poll_mode: str = "async",
|
||||
debug: bool = False,
|
||||
log_file: Optional[Path] = None,
|
||||
poll_max_retries: int = 1,
|
||||
poll_interval: float = 0.5,
|
||||
rate_limit_max_requests: int = 30,
|
||||
rate_limit_window_seconds: float = 10.0,
|
||||
):
|
||||
"""
|
||||
初始化 API 数据源
|
||||
|
||||
Args:
|
||||
base_url: API 基础 URL
|
||||
uid: 用户标识
|
||||
secret: 签名密钥
|
||||
debug: 是否启用调试模式
|
||||
log_file: 日志文件路径（可选）
|
||||
"""
|
||||
super().__init__(poll_max_retries=poll_max_retries, poll_interval=poll_interval)
|
||||
self.client = ApiClient(
|
||||
base_url,
|
||||
uid,
|
||||
secret,
|
||||
debug=debug,
|
||||
log_file=log_file,
|
||||
rate_limit_max_requests=rate_limit_max_requests,
|
||||
rate_limit_window_seconds=rate_limit_window_seconds,
|
||||
)
|
||||
self.poll_mode = poll_mode
|
||||
|
||||
async def initialize(self):
|
||||
"""初始化数据源（初始化 HTTP 客户端）"""
|
||||
await self.client.initialize()
|
||||
|
||||
async def close(self):
|
||||
"""关闭数据源（关闭 HTTP 客户端）"""
|
||||
await self.client.close()
|
||||
|
||||
def register_handler(self, handler: NodeHandler) -> None:
|
||||
"""
|
||||
注册业务 Handler
|
||||
|
||||
Args:
|
||||
handler: API Handler 实例
|
||||
"""
|
||||
compatible_handler = ensure_handler_compat(handler)
|
||||
compatible_handler.set_api_client(self.client)
|
||||
compatible_handler.set_poll_mode(self.poll_mode)
|
||||
|
||||
# 调用基类注册方法
|
||||
super().register_handler(compatible_handler)
|
||||
|
||||
# 如果 collection 已设置，确保新 handler 拿到引用
|
||||
if self._collection is not None:
|
||||
compatible_handler.set_collection(self._collection) # type: ignore[arg-type]
|
||||
|
||||
@staticmethod
|
||||
def _fmt_trace_value(value, max_len: int = 600) -> str:
|
||||
try:
|
||||
text = json.dumps(value, ensure_ascii=False, separators=(",", ":"))
|
||||
except Exception:
|
||||
text = str(value)
|
||||
if len(text) > max_len:
|
||||
return text[:max_len] + "...<truncated>"
|
||||
return text
|
||||
|
||||
def _append_trace_log(self, node, trace: dict, *, stage: str) -> None:
|
||||
method = str(trace.get("method", "")).upper()
|
||||
endpoint = str(trace.get("endpoint", ""))
|
||||
url = str(trace.get("url", ""))
|
||||
params = trace.get("params_text")
|
||||
if not isinstance(params, str):
|
||||
params = self._fmt_trace_value(trace.get("params", {}), max_len=self._TRACE_MAX_PARAMS_LEN)
|
||||
|
||||
payload = trace.get("payload_text")
|
||||
if not isinstance(payload, str):
|
||||
payload = self._fmt_trace_value(trace.get("payload", {}), max_len=self._TRACE_MAX_PARAMS_LEN)
|
||||
|
||||
response = trace.get("response_text")
|
||||
if not isinstance(response, str):
|
||||
response = self._fmt_trace_value(trace.get("response", {}), max_len=self._TRACE_MAX_RESPONSE_LEN)
|
||||
elapsed_ms = trace.get("elapsed_ms", 0)
|
||||
|
||||
node.append_log(f"API链路[{stage}] {method} {endpoint}")
|
||||
node.append_log(f" url={url}")
|
||||
if params not in {"{}", "null"}:
|
||||
node.append_log(f" params={params}")
|
||||
if payload not in {"{}", "null"}:
|
||||
node.append_log(f" payload={payload}")
|
||||
node.append_log(f" response={response}")
|
||||
node.append_log(f" elapsed_ms={elapsed_ms}")
|
||||
|
||||
def _consume_and_append_push_traces(self, node, push_id: str, *, stage: str) -> int:
|
||||
if not push_id:
|
||||
return 0
|
||||
traces = self.client.consume_traces_by_push_id(push_id)
|
||||
for trace in traces:
|
||||
self._append_trace_log(node, trace, stage=stage)
|
||||
return len(traces)
|
||||
|
||||
@staticmethod
|
||||
def _extract_push_id_from_trace(trace: dict | None) -> Optional[str]:
|
||||
if trace is None:
|
||||
return None
|
||||
payload = trace.get("payload")
|
||||
if not isinstance(payload, dict):
|
||||
return None
|
||||
push_id = payload.get("push_id")
|
||||
return push_id if isinstance(push_id, str) and push_id else None
|
||||
|
||||
def _resolve_push_id_for_success(self, node, result: TaskResult) -> Optional[str]:
|
||||
if result.task_id:
|
||||
return result.task_id
|
||||
|
||||
if result.push_id:
|
||||
return result.push_id
|
||||
|
||||
action_name = node.action.value.lower()
|
||||
biz_id = node.data_id or ""
|
||||
|
||||
if biz_id:
|
||||
by_biz_and_op = self.client.peek_latest_trace_by_biz_id(biz_id, op_kind=action_name)
|
||||
if by_biz_and_op is not None:
|
||||
push_id = self._extract_push_id_from_trace(by_biz_and_op)
|
||||
if push_id:
|
||||
self.client.consume_latest_trace_by_biz_id(biz_id, op_kind=action_name)
|
||||
return push_id
|
||||
|
||||
by_biz_any = self.client.peek_latest_trace_by_biz_id(biz_id)
|
||||
if by_biz_any is not None:
|
||||
push_id = self._extract_push_id_from_trace(by_biz_any)
|
||||
if push_id:
|
||||
self.client.consume_latest_trace_by_biz_id(biz_id)
|
||||
return push_id
|
||||
|
||||
return None
|
||||
|
||||
def _append_missing_push_id_debug_trace(self, node) -> None:
|
||||
action_name = node.action.value.lower()
|
||||
biz_id = node.data_id or ""
|
||||
|
||||
if biz_id:
|
||||
trace = self.client.peek_latest_trace_by_biz_id(biz_id, op_kind=action_name)
|
||||
if trace is None:
|
||||
trace = self.client.peek_latest_trace_by_biz_id(biz_id)
|
||||
if trace is not None:
|
||||
self._append_trace_log(node, trace, stage="submit-missing-push-id")
|
||||
return
|
||||
|
||||
node.append_log(
|
||||
f"API链路[submit-missing-push-id] 未找到匹配 biz_id 的请求记录，action={node.action.value}, data_id={node.data_id or '<empty>'}"
|
||||
)
|
||||
return
|
||||
|
||||
node.append_log(
|
||||
f"API链路[submit-missing-push-id] 未找到可用请求记录（已禁用按操作类型兜底），action={node.action.value}, data_id={node.data_id or '<empty>'}"
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
def _on_node_loaded(self, handler: NodeHandler, node) -> None:
|
||||
data_id = node.data_id or ""
|
||||
if not data_id:
|
||||
return
|
||||
traces = self.client.consume_load_traces(node.node_type, data_id)
|
||||
for trace in traces:
|
||||
self._append_trace_log(node, trace, stage="load")
|
||||
|
||||
async def _intercept_success_result(self, handler: NodeHandler, node, result: TaskResult, async_tasks: dict) -> bool:
|
||||
if node.action.value not in {"CREATE", "UPDATE", "DELETE"}:
|
||||
return False
|
||||
|
||||
push_id = self._resolve_push_id_for_success(node, result)
|
||||
if not push_id:
|
||||
self._append_missing_push_id_debug_trace(node)
|
||||
raise RuntimeError(
|
||||
f"[{node.node_type}] unified polling requires push_id for {node.action.value}, "
|
||||
f"but none resolved for node={node.node_id}, data_id={node.data_id}"
|
||||
)
|
||||
|
||||
synthetic = TaskResult.in_progress(node_id=node.node_id, task_id=push_id)
|
||||
appended = self._consume_and_append_push_traces(node, push_id, stage="submit")
|
||||
if appended == 0:
|
||||
node.append_log(f"API链路[submit] 未捕获请求记录，push_id={push_id}")
|
||||
await self._handle_in_progress_result(node, synthetic, async_tasks)
|
||||
return True
|
||||
|
||||
def _on_in_progress_result(self, handler: NodeHandler, node, result: TaskResult) -> None:
|
||||
if result.task_id:
|
||||
appended = self._consume_and_append_push_traces(node, result.task_id, stage="submit")
|
||||
if appended == 0:
|
||||
node.append_log(f"API链路[submit] 未捕获请求记录，push_id={result.task_id}")
|
||||
|
||||
def _on_poll_result(self, handler: NodeHandler, node, task_id: str, result: TaskResult) -> None:
|
||||
appended = self._consume_and_append_push_traces(node, task_id, stage="poll")
|
||||
if appended == 0:
|
||||
node.append_log(f"API链路[poll] 未捕获请求记录，push_id={task_id}")
|
||||
|
||||
def _on_failed_result(self, handler: NodeHandler, node, result: TaskResult) -> None:
|
||||
if node.action.value not in {"CREATE", "UPDATE", "DELETE"}:
|
||||
return
|
||||
|
||||
if result.push_id:
|
||||
traces = self.client.consume_traces_by_push_id(result.push_id)
|
||||
if traces:
|
||||
for trace in traces:
|
||||
self._append_trace_log(node, trace, stage="submit-failed")
|
||||
return
|
||||
node.append_log(f"API链路[submit-failed] 未捕获请求记录，push_id={result.push_id}")
|
||||
|
||||
action_name = node.action.value.lower()
|
||||
biz_id = node.data_id or ""
|
||||
|
||||
trace = None
|
||||
if biz_id:
|
||||
trace = self.client.consume_latest_trace_by_biz_id(biz_id, op_kind=action_name)
|
||||
if trace is None:
|
||||
trace = self.client.consume_latest_trace_by_biz_id(biz_id)
|
||||
|
||||
if trace is not None:
|
||||
self._append_trace_log(node, trace, stage="submit-failed")
|
||||
return
|
||||
|
||||
if biz_id:
|
||||
node.append_log(
|
||||
f"API链路[submit-failed] 未找到匹配 biz_id 的请求记录，action={node.action.value}, data_id={biz_id}"
|
||||
)
|
||||
return
|
||||
|
||||
node.append_log(
|
||||
f"API链路[submit-failed] 未找到可用请求记录（已禁用按操作类型兜底），action={node.action.value}, data_id=<empty>"
|
||||
)
|
||||
|
||||
# load_all 和 sync_all 继承自 BaseDataSource
|
||||
# BaseDataSource 会按顺序调用 Handler 的方法
|
||||
@@ -0,0 +1,25 @@
|
||||
"""
|
||||
API Handler 通用错误常量
|
||||
"""
|
||||
|
||||
# API Client 相关错误
|
||||
ERROR_API_CLIENT_NOT_INITIALIZED = "api_client not initialized"
|
||||
ERROR_API_CLIENT_NOT_INJECTED = "api_client not injected"
|
||||
|
||||
# Collection 相关错误
|
||||
ERROR_COLLECTION_NOT_SET = "collection not set"
|
||||
|
||||
# 数据验证错误
|
||||
ERROR_NODE_DATA_NONE = "Node data is None or validation failed"
|
||||
ERROR_VALIDATION_FAILED = "Validation failed"
|
||||
|
||||
# 业务数据错误
|
||||
ERROR_PROJECT_ID_NOT_FOUND = "project_id not found"
|
||||
ERROR_CONTRACT_ID_NOT_FOUND = "contract_id not found"
|
||||
ERROR_ID_NOT_FOUND = "id not found"
|
||||
|
||||
# 操作不支持错误
|
||||
ERROR_OPERATION_NOT_SUPPORTED = "Operation not supported"
|
||||
ERROR_CREATE_NOT_SUPPORTED = "Create operation not supported"
|
||||
ERROR_UPDATE_NOT_SUPPORTED = "Update operation not supported"
|
||||
ERROR_DELETE_NOT_SUPPORTED = "Delete operation not supported"
|
||||
@@ -0,0 +1,538 @@
|
||||
"""
|
||||
BaseApiHandler - API Handler 基类
|
||||
|
||||
职责：
|
||||
- 实现 Handler 接口（load, create, update, delete, poll）
|
||||
- 使用 self.api_client 发送 HTTP 请求
|
||||
- 生成 push_id 和 biz_id（业务标识）
|
||||
- 从 collection 获取依赖节点数据
|
||||
- 管理节点的 context（存储 project_id, contract_id 等）
|
||||
- 实现 poll 方法，调用 api_client.get_push_logs 批量查询
|
||||
- 提取 created_id（因为不同业务的 id 字段名和位置可能不同）
|
||||
- 返回 TaskResult（SUCCESS/FAILED/IN_PROGRESS）
|
||||
"""
|
||||
|
||||
import uuid
|
||||
import asyncio
|
||||
from abc import abstractmethod
|
||||
from typing import ClassVar, Dict, List, Any, Optional, TYPE_CHECKING, TypeVar
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ...common.type_safety import get_pydantic_model_fields
|
||||
from ..handler import NodeHandler
|
||||
from ..task_result import TaskResult
|
||||
from ..handler import ValidationResult # Import ValidationResult
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ...common.sync_node import SyncNode
|
||||
from ...common.collection import DataCollection
|
||||
from .client import ApiClient
|
||||
|
||||
T = TypeVar("T", bound=BaseModel)
|
||||
|
||||
|
||||
class BaseApiHandler(NodeHandler[T]):
|
||||
"""API Handler 基类"""
|
||||
|
||||
# 子类需要设置这些属性
|
||||
_node_type: str
|
||||
_node_class: type['SyncNode']
|
||||
_schema: Any
|
||||
|
||||
def __init__(self):
|
||||
# 子类应该在调用 super().__init__() 之前或之后设置 _node_type, _node_class, _schema
|
||||
self.api_client: 'ApiClient' = None # type: ignore # 由 DataSource 注入
|
||||
self.poll_mode: str = "async" # async | sync
|
||||
self._collection: 'DataCollection' = None # type: ignore # 由 DataSource 注入
|
||||
# 子类在 __init__ 中设置此列表以声明 create/update 中涉及的 Pydantic schema 类；
|
||||
# post-check 将只比较这些 schema 覆盖的字段，过滤后端只读的派生字段。
|
||||
self.update_schemas: List[type] = []
|
||||
|
||||
def get_update_fields(self, *, exclude: frozenset = frozenset({'id'})) -> List[str]:
|
||||
"""从 update_schemas 中提取所有可写字段名，排除标识键（默认排除 id）。"""
|
||||
fields: set = set()
|
||||
for schema in self.update_schemas:
|
||||
for field_name in get_pydantic_model_fields(schema):
|
||||
if field_name not in exclude:
|
||||
fields.add(field_name)
|
||||
return sorted(fields)
|
||||
|
||||
def set_api_client(self, client: 'ApiClient') -> None:
|
||||
self.api_client = client
|
||||
|
||||
def set_collection(self, collection: 'DataCollection') -> None:
|
||||
"""设置 Collection（由 DataSource 调用）"""
|
||||
self._collection = collection
|
||||
|
||||
@property
|
||||
def node_type(self) -> str:
|
||||
"""返回节点类型"""
|
||||
return self._node_type
|
||||
|
||||
@property
|
||||
def node_class(self) -> type['SyncNode']:
|
||||
"""返回节点类"""
|
||||
return self._node_class
|
||||
|
||||
@property
|
||||
def schema(self) -> Any:
|
||||
"""返回 schema（子类可重写）"""
|
||||
return self._schema
|
||||
|
||||
def _generate_push_id(self) -> str:
|
||||
"""
|
||||
生成 push_id（推送唯一标识）
|
||||
|
||||
Returns:
|
||||
UUID 格式的 push_id
|
||||
"""
|
||||
return str(uuid.uuid4())
|
||||
|
||||
def _generate_biz_id(self, node: 'SyncNode') -> str:
|
||||
"""
|
||||
生成 biz_id（业务标识）
|
||||
|
||||
子类可重写此方法实现自定义业务 ID 生成逻辑。
|
||||
默认使用 node_id 作为 biz_id。
|
||||
|
||||
Args:
|
||||
node: 同步节点
|
||||
|
||||
Returns:
|
||||
业务标识符
|
||||
"""
|
||||
return node.node_id
|
||||
|
||||
def _create_basic_node(self, data: Dict[str, Any], depend_ids: Optional[List[str]] = None) -> "SyncNode":
|
||||
"""创建或复用基础节点（默认使用 data['id'] 或 data['_id']）"""
|
||||
data_id = str(data.get("id") or data.get("_id") or "")
|
||||
if self._collection is None:
|
||||
raise RuntimeError("Collection not set. Call set_collection() before creating nodes.")
|
||||
|
||||
if data_id:
|
||||
existing_node = self._collection.get_by_data_id(self.node_type, data_id)
|
||||
if existing_node is not None:
|
||||
existing_node.depend_ids = list(depend_ids or [])
|
||||
existing_node.set_data(data.copy())
|
||||
existing_node.set_origin_data(data.copy())
|
||||
return existing_node
|
||||
|
||||
node_id = self._collection.generate_node_id()
|
||||
node = self.node_class(
|
||||
node_id=node_id,
|
||||
data_id=data_id,
|
||||
data=data.copy(),
|
||||
depend_ids=depend_ids or [],
|
||||
origin_data=data.copy(),
|
||||
)
|
||||
return node
|
||||
|
||||
def _create_node(self, data: Dict[str, Any]) -> "SyncNode":
|
||||
"""默认节点创建（子类可覆盖并补充 context）"""
|
||||
return self._create_basic_node(data)
|
||||
|
||||
def extract_id(self, data: Dict[str, Any]) -> str:
|
||||
"""从原始数据提取 ID（默认优先 id，其次 _id）"""
|
||||
return str(data.get("id") or data.get("_id") or "")
|
||||
|
||||
def extract_created_id(self, push_log: Dict[str, Any]) -> Optional[str]:
|
||||
"""
|
||||
从 push_log 提取 created_id
|
||||
|
||||
子类可重写此方法以适配不同的响应结构。
|
||||
默认从 push_log['data']['biz_id'] 提取。
|
||||
|
||||
Args:
|
||||
push_log: Push Log 响应数据
|
||||
|
||||
Returns:
|
||||
创建的对象 ID，未找到返回 None
|
||||
"""
|
||||
try:
|
||||
data = push_log.get('data', {})
|
||||
if isinstance(data, dict):
|
||||
return data.get('biz_id')
|
||||
return None
|
||||
except (KeyError, AttributeError):
|
||||
return None
|
||||
|
||||
def set_poll_mode(self, mode: str) -> None:
|
||||
"""设置轮询模式（sync/async）"""
|
||||
if mode not in {"sync", "async"}:
|
||||
raise ValueError(f"Unsupported poll mode: {mode}")
|
||||
self.poll_mode = mode
|
||||
|
||||
def _extract_created_id_from_response(self, response: Dict[str, Any]) -> Optional[str]:
|
||||
"""从同步创建响应提取创建的 ID（默认 data.biz_id）"""
|
||||
data = response.get("data") if isinstance(response, dict) else None
|
||||
if isinstance(data, dict):
|
||||
return data.get("biz_id")
|
||||
return None
|
||||
|
||||
def _build_create_task_result(
|
||||
self,
|
||||
node_id: str,
|
||||
response: Optional[Dict[str, Any]] = None,
|
||||
task_id: Optional[str] = None,
|
||||
fallback_data_id: Optional[str] = None,
|
||||
) -> TaskResult:
|
||||
"""根据轮询模式构建 create 的 TaskResult"""
|
||||
if self.poll_mode == "sync":
|
||||
created_id = None
|
||||
if response is not None:
|
||||
created_id = self._extract_created_id_from_response(response)
|
||||
if not created_id:
|
||||
created_id = fallback_data_id
|
||||
return TaskResult.success(node_id=node_id, data_id=created_id)
|
||||
|
||||
if not task_id:
|
||||
return TaskResult.failed(node_id=node_id, error="Missing task_id for async create")
|
||||
return TaskResult.in_progress(node_id=node_id, task_id=task_id)
|
||||
|
||||
# ========== 通用辅助方法 ==========
|
||||
|
||||
def _validate_schema(self, data: Dict[str, Any], schema: Any) -> Optional[str]:
|
||||
"""
|
||||
验证数据是否符合 schema
|
||||
|
||||
Args:
|
||||
data: 待验证数据
|
||||
schema: Pydantic schema
|
||||
|
||||
Returns:
|
||||
错误信息，无错误返回 None
|
||||
"""
|
||||
try:
|
||||
schema.model_validate(data)
|
||||
return None
|
||||
except Exception as e:
|
||||
return str(e)
|
||||
|
||||
def _get_schema_diff(
|
||||
self,
|
||||
schema,
|
||||
data: Dict[str, Any],
|
||||
origin_data: Dict[str, Any],
|
||||
node_id: Optional[str] = None
|
||||
) -> Dict[str, Any] | None:
|
||||
"""
|
||||
获取数据在指定 schema 下的差异字段
|
||||
|
||||
Args:
|
||||
schema: Pydantic schema 类
|
||||
data: 新数据
|
||||
origin_data: 原始数据
|
||||
node_id: 节点ID（用于日志）
|
||||
|
||||
Returns:
|
||||
Dict[str, Any] | None: 有差异返回差异字段字典，无差异或字段不全返回 None
|
||||
"""
|
||||
from pydantic import ValidationError
|
||||
|
||||
try:
|
||||
new_model = schema.model_validate(data)
|
||||
origin_model = schema.model_validate(origin_data)
|
||||
|
||||
new_dict = new_model.model_dump(exclude_unset=True)
|
||||
origin_dict = origin_model.model_dump(exclude_unset=True)
|
||||
|
||||
if new_dict == origin_dict:
|
||||
return None
|
||||
|
||||
# 返回有差异的字段
|
||||
diff_fields = {k: v for k, v in new_dict.items() if origin_dict.get(k) != v}
|
||||
|
||||
if diff_fields:
|
||||
# 打印差异详情
|
||||
node_info = f"[{node_id}]" if node_id else ""
|
||||
print(f" [DIFF] {self.node_type}{node_info} schema={schema.__name__}")
|
||||
for k, v in diff_fields.items():
|
||||
orig_v = origin_dict.get(k)
|
||||
print(f" • {k}: {orig_v!r} → {v!r}")
|
||||
|
||||
return diff_fields if diff_fields else None
|
||||
|
||||
except ValidationError:
|
||||
# schema 字段不全，视为无差异
|
||||
return None
|
||||
|
||||
def _filter_update_data(
|
||||
self,
|
||||
new_data: Dict[str, Any],
|
||||
original_data: Dict[str, Any],
|
||||
schema: Optional[Any] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
过滤出需要更新的字段
|
||||
|
||||
规则：
|
||||
1. 提取 schema 定义的所有字段（包括必填）
|
||||
2. 检查是否有任何字段发生变化
|
||||
3. 如果有变化，返回所有 schema 字段；否则返回空
|
||||
|
||||
Args:
|
||||
new_data: 新数据
|
||||
original_data: 原始数据
|
||||
schema: 更新 schema（可选）
|
||||
|
||||
Returns:
|
||||
包含 schema 所有字段的数据（如果有变化），否则返回空字典
|
||||
"""
|
||||
schema_fields = get_pydantic_model_fields(schema)
|
||||
if not schema_fields:
|
||||
# 无 schema，按原逻辑处理
|
||||
update_data = {}
|
||||
for key, new_value in new_data.items():
|
||||
if new_value == "" or new_value is None:
|
||||
continue
|
||||
original_value = original_data.get(key)
|
||||
if new_value != original_value:
|
||||
update_data[key] = new_value
|
||||
return update_data
|
||||
|
||||
# 1. 提取 schema 定义的所有字段
|
||||
schema_fields = set(schema_fields.keys())
|
||||
schema_data = {}
|
||||
has_change = False
|
||||
|
||||
for key in schema_fields:
|
||||
new_value = new_data.get(key)
|
||||
|
||||
# 排除空值
|
||||
if new_value == "" or new_value is None:
|
||||
continue
|
||||
|
||||
schema_data[key] = new_value
|
||||
|
||||
# 检查是否变化
|
||||
original_value = original_data.get(key)
|
||||
if new_value != original_value:
|
||||
# 记录具体差异以便排查
|
||||
# print(f" [DEBUG] Field '{key}' changed: '{original_value}' -> '{new_value}'")
|
||||
has_change = True
|
||||
|
||||
# 2. 只有存在变化时才返回数据
|
||||
return schema_data if has_change else {}
|
||||
|
||||
def _schema_changed_fields(
|
||||
self,
|
||||
schema: Any,
|
||||
new_data: Dict[str, Any],
|
||||
origin_data: Dict[str, Any],
|
||||
) -> List[str]:
|
||||
schema_fields = get_pydantic_model_fields(schema)
|
||||
if not schema_fields:
|
||||
return []
|
||||
changed: List[str] = []
|
||||
for field_name in schema_fields.keys():
|
||||
if new_data.get(field_name) != origin_data.get(field_name):
|
||||
changed.append(field_name)
|
||||
return changed
|
||||
|
||||
def _build_update_schema_status(
|
||||
self,
|
||||
*,
|
||||
schema_name: str,
|
||||
schema: Any,
|
||||
new_data: Dict[str, Any],
|
||||
origin_data: Dict[str, Any],
|
||||
will_update: bool,
|
||||
update_error: Optional[str] = None,
|
||||
) -> str:
|
||||
new_error = self._validate_schema(new_data, schema)
|
||||
origin_error = self._validate_schema(origin_data, schema)
|
||||
changed_fields = self._schema_changed_fields(schema, new_data, origin_data)
|
||||
|
||||
if new_error:
|
||||
compact = new_error.replace("\n", " ").strip()
|
||||
if len(compact) > 220:
|
||||
compact = compact[:220] + "..."
|
||||
return f"{schema_name}: NEW_INVALID({compact})"
|
||||
|
||||
if not changed_fields:
|
||||
return f"{schema_name}: CONSISTENT"
|
||||
|
||||
changed_text = ",".join(changed_fields)
|
||||
if update_error:
|
||||
compact = update_error.replace("\n", " ").strip()
|
||||
if len(compact) > 220:
|
||||
compact = compact[:220] + "..."
|
||||
return f"{schema_name}: DIFF[{changed_text}] -> UPDATE_FAILED({compact})"
|
||||
|
||||
if will_update:
|
||||
if origin_error:
|
||||
compact = origin_error.replace("\n", " ").strip()
|
||||
if len(compact) > 160:
|
||||
compact = compact[:160] + "..."
|
||||
return f"{schema_name}: DIFF[{changed_text}] -> UPDATE_TRIGGERED (origin_invalid={compact})"
|
||||
return f"{schema_name}: DIFF[{changed_text}] -> UPDATE_TRIGGERED"
|
||||
|
||||
if origin_error:
|
||||
compact = origin_error.replace("\n", " ").strip()
|
||||
if len(compact) > 160:
|
||||
compact = compact[:160] + "..."
|
||||
return f"{schema_name}: DIFF[{changed_text}] -> NOT_TRIGGERED (origin_invalid={compact})"
|
||||
return f"{schema_name}: DIFF[{changed_text}] -> NOT_TRIGGERED"
|
||||
|
||||
async def _execute_with_retry(
|
||||
self,
|
||||
operation,
|
||||
max_retries: int = 3,
|
||||
retry_delay: float = 1.0
|
||||
) -> Any:
|
||||
"""
|
||||
通用重试逻辑
|
||||
|
||||
子类可重写此方法实现自定义重试策略。
|
||||
默认策略：网络错误/5xx 重试，4xx 不重试。
|
||||
|
||||
Args:
|
||||
operation: 异步操作（协程函数）
|
||||
max_retries: 最大重试次数
|
||||
retry_delay: 重试延迟（秒）
|
||||
|
||||
Returns:
|
||||
操作执行结果
|
||||
|
||||
Raises:
|
||||
最后一次重试的异常
|
||||
"""
|
||||
import httpx
|
||||
|
||||
last_error = None
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
return await operation()
|
||||
except httpx.HTTPStatusError as e:
|
||||
# 4xx 客户端错误不重试
|
||||
if 400 <= e.response.status_code < 500:
|
||||
raise
|
||||
last_error = e
|
||||
if attempt < max_retries - 1:
|
||||
await asyncio.sleep(retry_delay * (attempt + 1))
|
||||
except (httpx.NetworkError, httpx.TimeoutException) as e:
|
||||
# 网络错误重试
|
||||
last_error = e
|
||||
if attempt < max_retries - 1:
|
||||
await asyncio.sleep(retry_delay * (attempt + 1))
|
||||
|
||||
# 重试次数用尽
|
||||
if last_error:
|
||||
raise last_error
|
||||
else:
|
||||
raise RuntimeError("Retry failed with unknown error")
|
||||
|
||||
# ========== 批量接口（子类必须实现） ==========
|
||||
|
||||
@abstractmethod
|
||||
async def create_all(self, nodes: List['SyncNode']) -> List[TaskResult]:
|
||||
"""
|
||||
批量创建节点
|
||||
|
||||
Args:
|
||||
nodes: 需要创建的节点列表（DataSource 已筛选为 CREATE + PENDING）
|
||||
|
||||
Returns:
|
||||
List[TaskResult]，每个结果关联 node_id
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def update_all(self, nodes: List['SyncNode']) -> List[TaskResult]:
|
||||
"""
|
||||
批量更新节点
|
||||
|
||||
Args:
|
||||
nodes: 需要更新的节点列表（DataSource 已筛选为 UPDATE + PENDING）
|
||||
|
||||
Returns:
|
||||
List[TaskResult]，每个结果关联 node_id
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def delete_all(self, nodes: List['SyncNode']) -> List[TaskResult]:
|
||||
"""
|
||||
批量删除节点
|
||||
|
||||
Args:
|
||||
nodes: 需要删除的节点列表（DataSource 已筛选为 DELETE + PENDING）
|
||||
|
||||
Returns:
|
||||
List[TaskResult]，每个结果关联 node_id
|
||||
"""
|
||||
pass
|
||||
|
||||
async def poll_tasks(self, task_ids: List[str]) -> Dict[str, TaskResult]:
|
||||
"""
|
||||
批量轮询异步任务状态
|
||||
|
||||
Args:
|
||||
task_ids: 任务 ID 列表（push_id 列表）
|
||||
|
||||
Returns:
|
||||
{task_id: TaskResult} 映射
|
||||
"""
|
||||
if not task_ids:
|
||||
return {}
|
||||
if not self.api_client:
|
||||
raise RuntimeError("api_client not injected")
|
||||
|
||||
# 批量查询 push_logs
|
||||
try:
|
||||
push_logs = await self.api_client.get_push_logs(task_ids)
|
||||
except Exception as e:
|
||||
# 查询失败，所有任务标记为失败
|
||||
return {
|
||||
task_id: TaskResult.failed(error=f"Poll failed: {str(e)}")
|
||||
for task_id in task_ids
|
||||
}
|
||||
|
||||
results = {}
|
||||
for task_id in task_ids:
|
||||
push_log = push_logs.get(task_id, {})
|
||||
|
||||
if 'error' in push_log:
|
||||
results[task_id] = TaskResult.failed(error=push_log['error'])
|
||||
elif push_log.get('code') == 200: # 通过code==200判断成功
|
||||
created_id = self.extract_created_id(push_log)
|
||||
results[task_id] = TaskResult.success(data_id=created_id)
|
||||
elif push_log.get('code') and push_log.get('code') != 200: # code存在但不是200表示失败
|
||||
results[task_id] = TaskResult.failed(error=push_log.get('message', 'Unknown error'))
|
||||
else:
|
||||
# 仍在进行中
|
||||
results[task_id] = TaskResult.in_progress(task_id=task_id)
|
||||
|
||||
return results
|
||||
|
||||
async def validate(self, data: Dict[str, Any]) -> ValidationResult:
|
||||
"""
|
||||
数据质量检查（使用 schema 验证）
|
||||
|
||||
Args:
|
||||
data: 原始数据
|
||||
|
||||
Returns:
|
||||
ValidationResult（是否有效 + 错误列表）
|
||||
"""
|
||||
error = self._validate_schema(data, self.schema)
|
||||
if error:
|
||||
return ValidationResult(is_valid=False, errors=[error])
|
||||
return ValidationResult(is_valid=True)
|
||||
|
||||
# ========== 抽象方法：子类必须实现 ==========
|
||||
|
||||
@abstractmethod
|
||||
async def load(self) -> List['SyncNode']:
|
||||
"""
|
||||
从 API 加载数据并创建节点
|
||||
|
||||
Handler 负责：
|
||||
1. 加载原始数据（可使用 self._collection 查询依赖）
|
||||
2. 创建 SyncNode（调用 self.create_node）
|
||||
3. 设置 context（如 project_id, contract_id 等）
|
||||
|
||||
Returns:
|
||||
SyncNode 列表
|
||||
"""
|
||||
pass
|
||||
@@ -0,0 +1,962 @@
|
||||
"""
|
||||
DataSource V2 - 状态管理 + Handler 编排层
|
||||
|
||||
DataSource 职责：
|
||||
1. 注册和管理 Handler
|
||||
2. 按依赖顺序加载数据
|
||||
3. 执行同步（调用 Handler）
|
||||
4. 管理节点状态流转（PENDING → IN_PROGRESS → SUCCESS/FAILED）
|
||||
5. 处理异步任务轮询
|
||||
6. 统一异常处理
|
||||
|
||||
DataSource 不负责：
|
||||
- ❌ 后端 API 调用（由 Handler 负责）
|
||||
- ❌ ID 映射（由 Strategy 负责）
|
||||
- ❌ 业务逻辑（由 Strategy 负责）
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Any, List, Dict, Optional, TYPE_CHECKING
|
||||
from abc import ABC
|
||||
from pathlib import Path
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..common.collection import DataCollection
|
||||
from .handler import NodeHandler
|
||||
from ..common.sync_node import SyncNode
|
||||
from ..common.types import SyncAction
|
||||
from .task_result import TaskStatus
|
||||
|
||||
from ..common.types import SyncStatus
|
||||
from ..common.type_safety import get_pydantic_model_fields
|
||||
from .handler import ensure_handler_compat
|
||||
from .task_result import TaskResult
|
||||
from ..engine import (
|
||||
StateMachineConfig,
|
||||
StateMachineRuntime,
|
||||
e40_sync_execute,
|
||||
)
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BaseDataSource(ABC):
|
||||
"""
|
||||
数据源基类 V2（状态管理 + 编排层）
|
||||
|
||||
核心流程：
|
||||
1. 注册 Handler：每个节点类型对应一个 Handler
|
||||
2. 加载数据：按依赖顺序调用 Handler.load()
|
||||
3. 执行同步：根据 node.action 调用 Handler 的 create/update/delete
|
||||
4. 状态管理：根据 TaskResult 更新 node.status
|
||||
5. 异步轮询：处理 IN_PROGRESS 节点
|
||||
|
||||
示例：
|
||||
# 1. 初始化 DataSource
|
||||
datasource = MyDataSource()
|
||||
|
||||
# 2. 注册 Handler
|
||||
datasource.register_handler(project_handler)
|
||||
datasource.register_handler(contract_handler)
|
||||
|
||||
# 3. 设置 Collection
|
||||
datasource.set_collection(collection)
|
||||
|
||||
# 4. 加载数据（按顺序）
|
||||
await datasource.load_all(order=["project", "contract"])
|
||||
|
||||
# 5. 执行同步（按顺序）
|
||||
await datasource.sync_all(order=["project", "contract"])
|
||||
"""
|
||||
|
||||
def __init__(self, poll_max_retries: int = 1, poll_interval: float = 0.5):
|
||||
self._handlers: Dict[str, "NodeHandler"] = {}
|
||||
self._collection: Optional["DataCollection"] = None
|
||||
self._target_project_ids: List[str] = []
|
||||
|
||||
# 按节点类型统计信息
|
||||
# {"company": {"loaded": 10, "synced": 10, "success": 10, "failed": 0}, ...}
|
||||
self._stats: Dict[str, Dict[str, int]] = {}
|
||||
# 记录本次 sync 的动作统计（CREATE/UPDATE/DELETE/NONE）
|
||||
self._action_summary: Dict[str, Dict[str, Dict[str, int]]] = {}
|
||||
# 记录本次 sync 中的节点动作（用于异步完成后的统计）
|
||||
self._action_by_node_id: Dict[str, "SyncAction"] = {}
|
||||
self._poll_max_retries = max(1, poll_max_retries)
|
||||
self._poll_interval = max(0, poll_interval)
|
||||
self._sm_runtime: Optional[StateMachineRuntime] = None
|
||||
|
||||
def _ensure_sm_runtime(self) -> StateMachineRuntime:
|
||||
if self._sm_runtime is None:
|
||||
cfg_path = Path(__file__).resolve().parents[1] / "config" / "node_state_machine.yaml"
|
||||
self._sm_runtime = StateMachineRuntime(StateMachineConfig.load(cfg_path))
|
||||
return self._sm_runtime
|
||||
|
||||
def set_state_machine_runtime(self, runtime: StateMachineRuntime) -> None:
|
||||
"""注入统一状态机 runtime（通常由 pipeline 负责）。"""
|
||||
self._sm_runtime = runtime
|
||||
|
||||
def _apply_sync_execute_state(
|
||||
self,
|
||||
node: "SyncNode",
|
||||
*,
|
||||
handler_status: "TaskStatus",
|
||||
poll_timeout: bool = False,
|
||||
detail: str = "",
|
||||
action_override: Optional["SyncAction"] = None,
|
||||
result_data_id: Optional[str] = None,
|
||||
) -> bool:
|
||||
action_for_event = action_override or node.action
|
||||
decision = e40_sync_execute(
|
||||
self._ensure_sm_runtime(),
|
||||
node=node,
|
||||
handler_result=handler_status.name,
|
||||
action=action_for_event.value,
|
||||
poll_timeout=poll_timeout,
|
||||
result_data_id=result_data_id,
|
||||
)
|
||||
if decision is None:
|
||||
if detail:
|
||||
node.append_log(
|
||||
f"执行状态未变更: action={action_for_event.value}, result={handler_status.name}, detail={detail}"
|
||||
)
|
||||
return False
|
||||
if detail:
|
||||
node.append_log(
|
||||
f"执行状态写回: action={action_for_event.value}, result={handler_status.name}, detail={detail}"
|
||||
)
|
||||
return True
|
||||
|
||||
def _compose_node_error(self, node: "SyncNode", *, source: str, detail: str) -> str:
|
||||
return (
|
||||
f"{source} | node={node.node_id} | type={node.node_type} | "
|
||||
f"action={node.action.value} | status={node.status.value} | detail={detail}"
|
||||
)
|
||||
|
||||
def _on_node_loaded(self, handler: "NodeHandler", node: "SyncNode") -> None:
|
||||
"""Datasource-specific hook when a loaded node is upserted into collection."""
|
||||
return
|
||||
|
||||
async def _intercept_success_result(
|
||||
self,
|
||||
handler: "NodeHandler",
|
||||
node: "SyncNode",
|
||||
result: "TaskResult",
|
||||
async_tasks: Dict[str, str],
|
||||
) -> bool:
|
||||
"""Datasource-specific hook to intercept SUCCESS result handling.
|
||||
|
||||
Return True if handled/consumed by subclass and no further default SUCCESS flow is needed.
|
||||
"""
|
||||
return False
|
||||
|
||||
def _on_in_progress_result(self, handler: "NodeHandler", node: "SyncNode", result: "TaskResult") -> None:
|
||||
"""Datasource-specific hook after an IN_PROGRESS task is queued."""
|
||||
return
|
||||
|
||||
def _on_poll_result(self, handler: "NodeHandler", node: "SyncNode", task_id: str, result: "TaskResult") -> None:
|
||||
"""Datasource-specific hook after polling result is applied to node."""
|
||||
return
|
||||
|
||||
def _on_failed_result(self, handler: "NodeHandler", node: "SyncNode", result: "TaskResult") -> None:
|
||||
"""Datasource-specific hook after a FAILED result is applied to node."""
|
||||
return
|
||||
|
||||
async def _handle_success_result(self, node: "SyncNode", result: "TaskResult") -> None:
|
||||
previous_data_id = node.data_id
|
||||
ok = self._apply_sync_execute_state(
|
||||
node,
|
||||
handler_status=result.status,
|
||||
detail="远程操作成功完成",
|
||||
result_data_id=result.data_id,
|
||||
)
|
||||
if not ok:
|
||||
raise RuntimeError(f"[{node.node_type}] SUCCESS result cannot map to state machine: node={node.node_id}")
|
||||
self._record_action_result(node, result.status)
|
||||
await self._apply_success_result(node, result.data_id, previous_data_id=previous_data_id)
|
||||
node.error = None
|
||||
|
||||
async def _handle_failed_result(self, node: "SyncNode", result: "TaskResult") -> None:
|
||||
detail = result.error or "Unknown error"
|
||||
ok = self._apply_sync_execute_state(
|
||||
node,
|
||||
handler_status=result.status,
|
||||
detail=f"远程操作失败: {detail}",
|
||||
)
|
||||
if not ok:
|
||||
raise RuntimeError(f"[{node.node_type}] FAILED result cannot map to state machine: node={node.node_id}")
|
||||
self._record_action_result(node, result.status)
|
||||
node.error = self._compose_node_error(node, source="handler_failed", detail=detail)
|
||||
|
||||
async def _handle_skipped_result(self, node: "SyncNode", result: "TaskResult") -> None:
|
||||
from .task_result import TaskStatus as _TaskStatus
|
||||
|
||||
original_action = self._action_by_node_id.get(node.node_id)
|
||||
if original_action is None or original_action.value != "UPDATE":
|
||||
raise RuntimeError(
|
||||
f"[{node.node_type}] SKIPPED is only allowed for UPDATE action: node={node.node_id}, action={original_action.value if original_action is not None else None}"
|
||||
)
|
||||
|
||||
ok = self._apply_sync_execute_state(
|
||||
node,
|
||||
handler_status=_TaskStatus.SKIPPED,
|
||||
detail="远程操作被跳过",
|
||||
action_override=original_action,
|
||||
)
|
||||
if not ok:
|
||||
raise RuntimeError(f"[{node.node_type}] SKIPPED result cannot map to state machine: node={node.node_id}")
|
||||
|
||||
self._record_action_result(node, _TaskStatus.SKIPPED)
|
||||
if result.sync_log:
|
||||
node.append_log(f"同步跳过: {result.sync_log}")
|
||||
node.error = None
|
||||
|
||||
async def _handle_in_progress_result(
|
||||
self,
|
||||
node: "SyncNode",
|
||||
result: "TaskResult",
|
||||
async_tasks: Dict[str, str],
|
||||
) -> None:
|
||||
if not result.task_id:
|
||||
node.error = self._compose_node_error(
|
||||
node,
|
||||
source="handler_in_progress_invalid",
|
||||
detail="missing task_id",
|
||||
)
|
||||
node.append_log("异步任务返回无 task_id，已忽略")
|
||||
return
|
||||
|
||||
if not result.node_id:
|
||||
node.error = self._compose_node_error(
|
||||
node,
|
||||
source="handler_in_progress_invalid",
|
||||
detail="missing node_id",
|
||||
)
|
||||
node.append_log("异步任务返回无 node_id，已忽略")
|
||||
return
|
||||
|
||||
async_tasks[result.node_id] = result.task_id
|
||||
ok = self._apply_sync_execute_state(
|
||||
node,
|
||||
handler_status=result.status,
|
||||
detail=f"异步任务进行中: {result.task_id}",
|
||||
)
|
||||
if not ok:
|
||||
node.append_log(f"异步任务进行中（无状态变更）: {result.task_id}")
|
||||
|
||||
def set_polling_config(self, max_retries: int, interval: float = 0.5) -> None:
|
||||
"""设置异步轮询参数（默认仅轮询一次）"""
|
||||
self._poll_max_retries = max(1, max_retries)
|
||||
self._poll_interval = max(0, interval)
|
||||
|
||||
def reset_stats(self) -> None:
|
||||
"""重置统计信息"""
|
||||
self._stats = {}
|
||||
self._action_summary = {}
|
||||
self._action_by_node_id = {}
|
||||
|
||||
def get_stats(self) -> Dict[str, Dict[str, int]]:
|
||||
"""
|
||||
获取按节点类型的统计信息
|
||||
|
||||
Returns:
|
||||
dict: {"company": {"loaded": 10, "synced": 10, "success": 10, "failed": 0}, ...}
|
||||
"""
|
||||
return self._stats.copy()
|
||||
|
||||
def get_action_summary(self) -> Dict[str, Dict[str, Dict[str, int]]]:
|
||||
"""
|
||||
获取本次 sync 的动作统计（按节点类型）。
|
||||
|
||||
Returns:
|
||||
dict: {
|
||||
"company": {
|
||||
"create": {"total": 10, "success": 8, "failed": 2},
|
||||
"update": {"total": 2, "success": 2, "failed": 0},
|
||||
"delete": {"total": 0, "success": 0, "failed": 0},
|
||||
"none": {"total": 5, "success": 0, "failed": 0},
|
||||
"total": {"total": 17, "success": 10, "failed": 2},
|
||||
},
|
||||
...
|
||||
}
|
||||
"""
|
||||
return {k: {kk: vv.copy() for kk, vv in v.items()} for k, v in self._action_summary.items()}
|
||||
|
||||
def _init_node_stats(self, node_type: str) -> None:
|
||||
"""初始化节点类型的统计信息"""
|
||||
if node_type not in self._stats:
|
||||
self._stats[node_type] = {
|
||||
"loaded": 0,
|
||||
"synced": 0,
|
||||
"success": 0,
|
||||
"failed": 0,
|
||||
}
|
||||
if node_type not in self._action_summary:
|
||||
self._action_summary[node_type] = {
|
||||
"create": {"total": 0, "success": 0, "failed": 0, "skipped": 0},
|
||||
"update": {"total": 0, "success": 0, "failed": 0, "skipped": 0},
|
||||
"delete": {"total": 0, "success": 0, "failed": 0, "skipped": 0},
|
||||
"none": {"total": 0, "success": 0, "failed": 0, "skipped": 0},
|
||||
"total": {"total": 0, "success": 0, "failed": 0, "skipped": 0},
|
||||
}
|
||||
|
||||
def _set_action_totals(self, node_type: str, create: int, update: int, delete: int, none: int) -> None:
|
||||
summary = self._action_summary[node_type]
|
||||
summary["create"]["total"] = create
|
||||
summary["update"]["total"] = update
|
||||
summary["delete"]["total"] = delete
|
||||
summary["none"]["total"] = none
|
||||
summary["total"]["total"] = create + update + delete + none
|
||||
|
||||
def _record_action_result(self, node: "SyncNode", result_status: "TaskStatus") -> None:
|
||||
from ..common.types import SyncAction
|
||||
from .task_result import TaskStatus as _TaskStatus
|
||||
|
||||
# 情况 A: 原始动作就是 NONE
|
||||
# 情况 B: Handler 运行时发现没差异，把 action 降级为了 NONE
|
||||
action = node.action
|
||||
|
||||
# 获取最初锁定的动作（用于修正统计列）
|
||||
original_action = self._action_by_node_id.pop(node.node_id, SyncAction.NONE)
|
||||
|
||||
summary = self._action_summary.get(node.node_type)
|
||||
if summary is None:
|
||||
return
|
||||
|
||||
# 如果当前 action 是 NONE，说明这次“同步”被取消了
|
||||
if action == SyncAction.NONE:
|
||||
# 如果原始动作不是 NONE，说明发生了降级动作，我们需要把 total 从对应列挪到 none 列
|
||||
if original_action != SyncAction.NONE:
|
||||
orig_key = original_action.value.lower()
|
||||
if orig_key in summary and summary[orig_key]["total"] > 0:
|
||||
summary[orig_key]["total"] -= 1
|
||||
summary["none"]["total"] += 1
|
||||
return
|
||||
|
||||
# 正常记录成功/失败/跳过
|
||||
key = action.value.lower()
|
||||
if result_status == _TaskStatus.SUCCESS:
|
||||
summary[key]["success"] += 1
|
||||
summary["total"]["success"] += 1
|
||||
elif result_status == _TaskStatus.FAILED:
|
||||
summary[key]["failed"] += 1
|
||||
summary["total"]["failed"] += 1
|
||||
elif result_status == _TaskStatus.SKIPPED:
|
||||
summary[key]["skipped"] += 1
|
||||
summary["total"]["skipped"] += 1
|
||||
|
||||
def set_collection(self, collection: "DataCollection") -> None:
|
||||
"""
|
||||
设置 Collection（在 load_all/sync_all 前调用）
|
||||
|
||||
Args:
|
||||
collection: 数据集合
|
||||
"""
|
||||
self._collection = collection
|
||||
collection.set_state_machine_runtime(self._ensure_sm_runtime())
|
||||
# 同时设置到所有已注册的 handler
|
||||
for handler in self._handlers.values():
|
||||
handler.set_collection(collection)
|
||||
|
||||
# ========== Handler 管理 ==========
|
||||
|
||||
def register_handler(self, handler: "NodeHandler") -> None:
|
||||
"""
|
||||
注册 Handler
|
||||
|
||||
Args:
|
||||
handler: 节点处理器实例
|
||||
|
||||
示例：
|
||||
datasource.register_handler(ContractNodeHandler(api_client))
|
||||
"""
|
||||
compatible_handler = ensure_handler_compat(handler)
|
||||
self._handlers[compatible_handler.node_type] = compatible_handler
|
||||
self._apply_target_project_ids_to_handler(compatible_handler)
|
||||
|
||||
def set_target_project_ids(self, target_project_ids: Optional[List[str]]) -> None:
|
||||
"""设置项目过滤列表，并在注册阶段下发给所有 handler。"""
|
||||
normalized = [str(pid) for pid in (target_project_ids or []) if str(pid)]
|
||||
self._target_project_ids = normalized
|
||||
for handler in self._handlers.values():
|
||||
self._apply_target_project_ids_to_handler(handler)
|
||||
|
||||
def _apply_target_project_ids_to_handler(self, handler: "NodeHandler") -> None:
|
||||
handler.set_target_project_ids(self._target_project_ids)
|
||||
|
||||
def get_handler(self, node_type: str) -> "NodeHandler":
|
||||
"""获取 Handler"""
|
||||
if node_type not in self._handlers:
|
||||
raise KeyError(f"Handler for '{node_type}' not registered")
|
||||
return self._handlers[node_type]
|
||||
|
||||
async def _upsert_loaded_nodes(
|
||||
self,
|
||||
handler: "NodeHandler",
|
||||
node_type: str,
|
||||
nodes: List["SyncNode"]
|
||||
) -> None:
|
||||
"""将加载的节点写入 Collection（按 data_id 优先命中已有节点）"""
|
||||
if self._collection is None:
|
||||
raise RuntimeError("Collection not set. Call set_collection() first.")
|
||||
|
||||
for node in nodes:
|
||||
existing_node = None
|
||||
if node.data_id:
|
||||
existing_node = self._collection.get_by_data_id(node_type, node.data_id)
|
||||
load_source = f"{self.__class__.__name__}.load"
|
||||
|
||||
if existing_node is not None:
|
||||
data_payload = node.get_data()
|
||||
existing_node.depend_ids = list(node.depend_ids or [])
|
||||
if node.context:
|
||||
existing_node.context.update(node.context)
|
||||
existing_node.set_data(data_payload)
|
||||
existing_node.set_origin_data(data_payload)
|
||||
if existing_node.context.pop("_loaded_from_persistence", False):
|
||||
existing_node.append_log(f"加载来源: persistence -> {load_source} 覆盖刷新")
|
||||
else:
|
||||
existing_node.append_log(f"加载来源: {load_source} 覆盖刷新")
|
||||
|
||||
await self._collection.update(existing_node)
|
||||
self._on_node_loaded(handler, existing_node)
|
||||
else:
|
||||
node.append_log(f"加载来源: {load_source} 新增节点")
|
||||
await self._collection.add(node)
|
||||
self._on_node_loaded(handler, node)
|
||||
|
||||
# ========== 数据加载 ==========
|
||||
|
||||
async def load_all(
|
||||
self,
|
||||
order: Optional[List[str]] = None,
|
||||
data_type: Optional[str] = None
|
||||
) -> None:
|
||||
"""
|
||||
按依赖顺序加载所有数据
|
||||
|
||||
注意：调用前需先 set_collection()
|
||||
|
||||
Args:
|
||||
order: 节点类型顺序（如 ["project", "contract", "construction"])
|
||||
data_type: 仅加载单一类型（与 order 互斥）
|
||||
|
||||
工作流程：
|
||||
1. 按顺序遍历每个节点类型
|
||||
2. 调用 Handler.load(collection)
|
||||
3. Handler 可以从 collection 查询依赖节点（如 Contract 查询 Project）
|
||||
4. 使用 Handler.create_node() 创建 SyncNode
|
||||
5. 添加到 Collection
|
||||
|
||||
示例：
|
||||
datasource.set_collection(collection)
|
||||
await datasource.load_all(order=["project", "contract"])
|
||||
"""
|
||||
if self._collection is None:
|
||||
raise RuntimeError("Collection not set. Call set_collection() first.")
|
||||
|
||||
if data_type is not None:
|
||||
order_to_load = [data_type]
|
||||
elif order is not None:
|
||||
order_to_load = order
|
||||
else:
|
||||
order_to_load = list(self._handlers.keys())
|
||||
|
||||
for node_type in order_to_load:
|
||||
handler = self.get_handler(node_type)
|
||||
|
||||
# 初始化统计
|
||||
self._init_node_stats(node_type)
|
||||
|
||||
try:
|
||||
# Handler 加载数据并创建节点
|
||||
nodes = await handler.load()
|
||||
|
||||
print(f"Loaded nodes for {node_type}: {len(nodes)}")
|
||||
|
||||
# 将节点写入 Collection：优先按 data_id 命中已有节点并更新
|
||||
await self._upsert_loaded_nodes(handler, node_type, nodes)
|
||||
|
||||
# 统计加载的节点数
|
||||
self._stats[node_type]["loaded"] += len(nodes)
|
||||
except Exception as exc:
|
||||
print(f"❌ [{node_type}] 加载失败: {exc}")
|
||||
continue
|
||||
|
||||
# ========== 同步执行 ==========
|
||||
|
||||
async def sync_all(
|
||||
self,
|
||||
order: Optional[List[str]] = None,
|
||||
data_type: Optional[str] = None,
|
||||
poll_async_tasks: bool = True
|
||||
) -> Dict[str, Dict[str, str]]:
|
||||
"""
|
||||
按依赖顺序执行同步
|
||||
|
||||
注意：调用前需先 set_collection()
|
||||
|
||||
Args:
|
||||
order: 节点类型顺序
|
||||
data_type: 仅同步单一类型（与 order 互斥）
|
||||
|
||||
工作流程：
|
||||
1. 按顺序遍历每个节点类型
|
||||
2. 筛选需要同步的节点（status=PENDING, action!=NONE）
|
||||
3. 调用 Handler 的 create/update/delete
|
||||
4. 根据 TaskResult 更新节点状态
|
||||
5. 处理异步任务轮询
|
||||
|
||||
状态流转：
|
||||
- PENDING + action=CREATE/UPDATE/DELETE → IN_PROGRESS → SUCCESS/FAILED
|
||||
- PENDING + action=NONE → 保持 PENDING
|
||||
- DEPENDENCY_ERROR / ABNORMAL / WARNING → SKIPPED
|
||||
|
||||
示例：
|
||||
datasource.set_collection(collection)
|
||||
await datasource.sync_all(order=["project", "contract"])
|
||||
"""
|
||||
if self._collection is None:
|
||||
raise RuntimeError("Collection not set. Call set_collection() first.")
|
||||
|
||||
if data_type is not None:
|
||||
order_to_sync = [data_type]
|
||||
elif order is not None:
|
||||
order_to_sync = order
|
||||
else:
|
||||
order_to_sync = list(self._handlers.keys())
|
||||
|
||||
async_tasks_by_type: Dict[str, Dict[str, str]] = {}
|
||||
|
||||
for node_type in order_to_sync:
|
||||
handler = self.get_handler(node_type)
|
||||
|
||||
# 批量同步该类型的所有节点
|
||||
async_tasks = await self._sync_nodes(handler, poll_async_tasks=poll_async_tasks)
|
||||
if async_tasks:
|
||||
async_tasks_by_type[node_type] = async_tasks
|
||||
|
||||
return async_tasks_by_type
|
||||
|
||||
async def _sync_nodes(
|
||||
self,
|
||||
handler: "NodeHandler",
|
||||
poll_async_tasks: bool = True
|
||||
) -> Dict[str, str]:
|
||||
"""
|
||||
同步一批节点（批量接口）
|
||||
|
||||
Args:
|
||||
handler: 节点处理器
|
||||
|
||||
流程：
|
||||
1. DataSource 筛选需要处理的节点
|
||||
2. 调用 handler.create_all(nodes)
|
||||
3. 调用 handler.update_all(nodes)
|
||||
4. 调用 handler.delete_all(nodes)
|
||||
5. 根据 List[TaskResult] 更新节点状态
|
||||
6. 收集并轮询异步任务
|
||||
"""
|
||||
from ..common.types import SyncAction, SyncStatus
|
||||
from .task_result import TaskStatus as _TaskStatus
|
||||
|
||||
if self._collection is None:
|
||||
raise RuntimeError("Collection not set. Call set_collection() first.")
|
||||
collection = self._collection
|
||||
|
||||
# 初始化统计
|
||||
self._init_node_stats(handler.node_type)
|
||||
|
||||
async_tasks: Dict[str, str] = {} # {node_id: task_id}
|
||||
|
||||
# DataSource 负责筛选需要创建的节点（排除已失败的节点）
|
||||
create_nodes = [
|
||||
n
|
||||
for n in collection.filter_by_state_ids(
|
||||
node_type=handler.node_type,
|
||||
state_ids=["S06"],
|
||||
)
|
||||
if n.action == SyncAction.CREATE
|
||||
]
|
||||
|
||||
# DataSource 负责筛选需要更新的节点（排除已失败的节点）
|
||||
update_nodes = [
|
||||
n
|
||||
for n in collection.filter_by_state_ids(
|
||||
node_type=handler.node_type,
|
||||
state_ids=["S07"],
|
||||
)
|
||||
if n.action == SyncAction.UPDATE
|
||||
]
|
||||
|
||||
# DataSource 负责筛选需要删除的节点（排除已失败的节点）
|
||||
delete_nodes = [
|
||||
n
|
||||
for n in collection.filter_by_state_ids(
|
||||
node_type=handler.node_type,
|
||||
state_ids=["S09"],
|
||||
)
|
||||
if n.action == SyncAction.DELETE
|
||||
]
|
||||
|
||||
if delete_nodes:
|
||||
raise NotImplementedError(
|
||||
f"[{handler.node_type}] delete path is not implemented yet, "
|
||||
f"but found {len(delete_nodes)} node(s) in S09/DELETE."
|
||||
)
|
||||
|
||||
# 记录本次 sync 的动作统计（在 action 被重置前）
|
||||
for node in create_nodes:
|
||||
self._action_by_node_id[node.node_id] = SyncAction.CREATE
|
||||
for node in update_nodes:
|
||||
self._action_by_node_id[node.node_id] = SyncAction.UPDATE
|
||||
for node in delete_nodes:
|
||||
self._action_by_node_id[node.node_id] = SyncAction.DELETE
|
||||
|
||||
none_count = len(
|
||||
self._collection.filter(
|
||||
node_type=handler.node_type,
|
||||
node_filter={"action": SyncAction.NONE}
|
||||
)
|
||||
)
|
||||
self._set_action_totals(
|
||||
handler.node_type,
|
||||
create=len(create_nodes),
|
||||
update=len(update_nodes),
|
||||
delete=len(delete_nodes),
|
||||
none=none_count,
|
||||
)
|
||||
|
||||
# 批量创建（传入节点列表）
|
||||
if create_nodes:
|
||||
logger.info(f"[{handler.node_type}] create提交开始: nodes={len(create_nodes)}")
|
||||
try:
|
||||
create_results = await handler.create_all(create_nodes)
|
||||
create_success = 0
|
||||
create_in_progress = 0
|
||||
create_failed = 0
|
||||
create_skipped = 0
|
||||
for result in create_results:
|
||||
if result.status == _TaskStatus.SUCCESS:
|
||||
create_success += 1
|
||||
elif result.status == _TaskStatus.IN_PROGRESS:
|
||||
create_in_progress += 1
|
||||
elif result.status == _TaskStatus.FAILED:
|
||||
create_failed += 1
|
||||
elif result.status == _TaskStatus.SKIPPED:
|
||||
create_skipped += 1
|
||||
await self._handle_task_result(handler, result, async_tasks)
|
||||
logger.info(
|
||||
f"[{handler.node_type}] create提交完成: success={create_success}, "
|
||||
f"in_progress={create_in_progress}, failed={create_failed}, skipped={create_skipped}"
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"❌ [{handler.node_type}] create_all 异常: {e}")
|
||||
for node in create_nodes:
|
||||
ok = self._apply_sync_execute_state(
|
||||
node,
|
||||
handler_status=_TaskStatus.FAILED,
|
||||
detail=str(e),
|
||||
)
|
||||
if not ok:
|
||||
raise RuntimeError(f"[{handler.node_type}] create_all exception cannot map to state machine")
|
||||
|
||||
# 批量更新（传入节点列表）
|
||||
if update_nodes:
|
||||
try:
|
||||
update_results = await handler.update_all(update_nodes)
|
||||
for result in update_results:
|
||||
await self._handle_task_result(handler, result, async_tasks)
|
||||
except Exception as e:
|
||||
print(f"❌ [{handler.node_type}] update_all 异常: {e}")
|
||||
for node in update_nodes:
|
||||
ok = self._apply_sync_execute_state(
|
||||
node,
|
||||
handler_status=_TaskStatus.FAILED,
|
||||
detail=str(e),
|
||||
)
|
||||
if not ok:
|
||||
raise RuntimeError(f"[{handler.node_type}] update_all exception cannot map to state machine")
|
||||
|
||||
# 批量删除（delete stage 当前未实现，检测到 S09 会在上方直接 fail-fast）
|
||||
|
||||
# 轮询异步任务
|
||||
if async_tasks and poll_async_tasks:
|
||||
logger.info(
|
||||
f"[{handler.node_type}] create已提交，push_id检查即将开始: pending={len(async_tasks)}, "
|
||||
f"first_wait={self._poll_interval:.3f}s"
|
||||
)
|
||||
await self._poll_async_tasks(handler, async_tasks)
|
||||
|
||||
# 统计同步结果
|
||||
all_nodes = self._collection.filter(node_type=handler.node_type)
|
||||
synced_count = len([n for n in all_nodes if n.action != SyncAction.NONE])
|
||||
success_count = len([n for n in all_nodes if n.status == SyncStatus.SUCCESS])
|
||||
failed_count = len([n for n in all_nodes if n.status == SyncStatus.FAILED])
|
||||
|
||||
self._stats[handler.node_type]["synced"] = synced_count
|
||||
self._stats[handler.node_type]["success"] = success_count
|
||||
self._stats[handler.node_type]["failed"] = failed_count
|
||||
|
||||
return async_tasks
|
||||
|
||||
async def poll_async_tasks(self, async_tasks_by_type: Dict[str, Dict[str, str]]) -> None:
|
||||
"""由 Pipeline 调度的异步任务轮询"""
|
||||
for node_type, async_tasks in async_tasks_by_type.items():
|
||||
if not async_tasks:
|
||||
continue
|
||||
handler = self.get_handler(node_type)
|
||||
await self._poll_async_tasks(handler, async_tasks)
|
||||
|
||||
async def _handle_task_result(
|
||||
self,
|
||||
handler: "NodeHandler",
|
||||
result: "TaskResult",
|
||||
async_tasks: Dict[str, str]
|
||||
) -> None:
|
||||
"""
|
||||
处理 Handler 返回的 TaskResult
|
||||
|
||||
Args:
|
||||
collection: Collection 上下文
|
||||
result: Handler 返回的结果（包含 node_id）
|
||||
async_tasks: 异步任务字典
|
||||
"""
|
||||
from .task_result import TaskStatus as _TaskStatus
|
||||
|
||||
if not result.node_id:
|
||||
# 没有 node_id，无法定位节点
|
||||
return
|
||||
|
||||
if self._collection is None:
|
||||
# Collection 未设置，无法处理结果
|
||||
return
|
||||
|
||||
# 从 self._collection 中查找节点
|
||||
node = self._collection.get(result.node_id)
|
||||
if not node:
|
||||
return
|
||||
|
||||
if result.status == _TaskStatus.SUCCESS:
|
||||
if await self._intercept_success_result(handler, node, result, async_tasks):
|
||||
return
|
||||
|
||||
if result.status == _TaskStatus.SUCCESS:
|
||||
await self._handle_success_result(node, result)
|
||||
return
|
||||
|
||||
if result.status == _TaskStatus.FAILED:
|
||||
self._on_failed_result(handler, node, result)
|
||||
await self._handle_failed_result(node, result)
|
||||
return
|
||||
|
||||
if result.status == _TaskStatus.SKIPPED:
|
||||
await self._handle_skipped_result(node, result)
|
||||
return
|
||||
|
||||
if result.status == _TaskStatus.IN_PROGRESS:
|
||||
self._on_in_progress_result(handler, node, result)
|
||||
await self._handle_in_progress_result(node, result, async_tasks)
|
||||
return
|
||||
|
||||
def _write_back_data_id_to_data(self, node, data_id: str) -> None:
|
||||
data = node.get_data()
|
||||
if data is None:
|
||||
return
|
||||
|
||||
schema = node.get_schema()
|
||||
candidate_keys = ["id", "_id"]
|
||||
model_fields = get_pydantic_model_fields(schema)
|
||||
|
||||
for key in candidate_keys:
|
||||
if key in data:
|
||||
data[key] = data_id
|
||||
node.set_data(data)
|
||||
return
|
||||
if key in model_fields:
|
||||
data[key] = data_id
|
||||
node.set_data(data)
|
||||
return
|
||||
|
||||
async def _apply_success_result(self, node, data_id: Optional[str], previous_data_id: str) -> None:
|
||||
from ..common.types import SyncAction
|
||||
|
||||
if data_id and node.action == SyncAction.CREATE:
|
||||
if node.data_id != data_id:
|
||||
raise RuntimeError(
|
||||
f"[{node.node_type}] data_id must be updated in event apply path: expected={data_id}, actual={node.data_id}"
|
||||
)
|
||||
self._write_back_data_id_to_data(node, data_id)
|
||||
|
||||
if self._collection is not None and data_id != previous_data_id:
|
||||
await self._collection.update(node)
|
||||
|
||||
if node.action == SyncAction.UPDATE:
|
||||
current_data = node.get_data()
|
||||
if current_data is not None:
|
||||
node.set_origin_data(current_data)
|
||||
|
||||
if node.action == SyncAction.DELETE:
|
||||
if self._collection is None:
|
||||
return
|
||||
await self._collection.delete(node.node_id)
|
||||
return
|
||||
|
||||
# 成功后 action 是否重置由上层策略决定
|
||||
|
||||
async def _poll_async_tasks(
|
||||
self,
|
||||
handler: "NodeHandler",
|
||||
async_tasks: Dict[str, str]
|
||||
) -> None:
|
||||
"""
|
||||
轮询异步任务直到全部完成
|
||||
|
||||
Args:
|
||||
handler: 节点处理器
|
||||
collection: Collection 上下文
|
||||
async_tasks: {node_id: task_id} 映射
|
||||
|
||||
流程：
|
||||
1. 批量调用 Handler.poll_tasks()
|
||||
2. 更新节点状态
|
||||
3. 如果还有未完成任务，等待后重试
|
||||
"""
|
||||
from .task_result import TaskStatus as _TaskStatus
|
||||
|
||||
max_retries = self._poll_max_retries
|
||||
retry_count = 0
|
||||
poll_interval = self._poll_interval # 轮询间隔（秒）
|
||||
|
||||
if self._collection is None:
|
||||
# Collection 未设置，无法轮询任务
|
||||
return
|
||||
|
||||
if async_tasks and poll_interval > 0:
|
||||
logger.info(
|
||||
f"[{handler.node_type}] push_id首次检查前等待 {poll_interval:.3f}s, pending={len(async_tasks)}"
|
||||
)
|
||||
await asyncio.sleep(poll_interval)
|
||||
|
||||
while async_tasks and retry_count < max_retries:
|
||||
logger.info(
|
||||
f"[{handler.node_type}] push_id检查 {retry_count + 1}/{max_retries}, pending={len(async_tasks)}"
|
||||
)
|
||||
# 批量查询
|
||||
task_ids = list(async_tasks.values())
|
||||
results = await handler.poll_tasks(task_ids)
|
||||
|
||||
# 处理结果
|
||||
completed_nodes = []
|
||||
for node_id, task_id in list(async_tasks.items()):
|
||||
if task_id not in results:
|
||||
continue
|
||||
|
||||
result = results[task_id]
|
||||
node = self._collection.get(node_id)
|
||||
if not node:
|
||||
completed_nodes.append(node_id)
|
||||
continue
|
||||
|
||||
if result.status == _TaskStatus.SUCCESS:
|
||||
self._on_poll_result(handler, node, task_id, result)
|
||||
previous_data_id = node.data_id
|
||||
ok = self._apply_sync_execute_state(
|
||||
node,
|
||||
handler_status=result.status,
|
||||
detail=f"异步任务成功完成: {task_id}",
|
||||
result_data_id=result.data_id,
|
||||
)
|
||||
if not ok:
|
||||
raise RuntimeError(f"[{node.node_type}] async SUCCESS cannot map to state machine: node={node.node_id}")
|
||||
self._record_action_result(node, result.status)
|
||||
await self._apply_success_result(node, result.data_id, previous_data_id=previous_data_id)
|
||||
node.error = None
|
||||
logger.info(
|
||||
f"[{handler.node_type}] polling成功: node={node.node_id}, push_id={task_id}, data_id={result.data_id or node.data_id}"
|
||||
)
|
||||
completed_nodes.append(node_id)
|
||||
|
||||
elif result.status == _TaskStatus.FAILED:
|
||||
self._on_poll_result(handler, node, task_id, result)
|
||||
ok = self._apply_sync_execute_state(
|
||||
node,
|
||||
handler_status=result.status,
|
||||
detail=f"异步任务失败: {result.error}",
|
||||
)
|
||||
if not ok:
|
||||
raise RuntimeError(f"[{node.node_type}] async FAILED cannot map to state machine: node={node.node_id}")
|
||||
self._record_action_result(node, result.status)
|
||||
node.error = result.error
|
||||
completed_nodes.append(node_id)
|
||||
|
||||
# IN_PROGRESS 继续等待
|
||||
|
||||
# 移除已完成的任务
|
||||
for node_id in completed_nodes:
|
||||
async_tasks.pop(node_id, None)
|
||||
|
||||
# 还有未完成任务，等待后重试
|
||||
if async_tasks:
|
||||
retry_count += 1
|
||||
|
||||
# 超时未完成的任务标记为失败
|
||||
if async_tasks:
|
||||
for node_id in async_tasks:
|
||||
node = self._collection.get(node_id)
|
||||
if node:
|
||||
timeout_msg = f"异步任务超时 (等待 {max_retries * poll_interval} 秒后仍未完成)"
|
||||
ok = self._apply_sync_execute_state(
|
||||
node,
|
||||
handler_status=_TaskStatus.FAILED,
|
||||
poll_timeout=True,
|
||||
detail=timeout_msg,
|
||||
)
|
||||
if not ok:
|
||||
raise RuntimeError(f"[{node.node_type}] poll timeout cannot map to state machine: node={node.node_id}")
|
||||
node.error = f"Async task timeout after {max_retries * poll_interval} seconds"
|
||||
self._record_action_result(node, _TaskStatus.FAILED)
|
||||
|
||||
# ========== 统计信息 ==========
|
||||
|
||||
def get_sync_summary(self, collection: "DataCollection") -> Dict[str, Any]:
|
||||
"""
|
||||
获取同步统计信息
|
||||
|
||||
Returns:
|
||||
{
|
||||
"total": 总节点数,
|
||||
"success": 成功数,
|
||||
"failed": 失败数,
|
||||
"in_progress": 进行中,
|
||||
"skipped": 跳过数,
|
||||
"by_type": {...} # 按类型统计
|
||||
}
|
||||
"""
|
||||
nodes = list(collection._nodes.values())
|
||||
|
||||
summary = {
|
||||
"total": len(nodes),
|
||||
"success": sum(1 for n in nodes if n.status == SyncStatus.SUCCESS),
|
||||
"failed": sum(1 for n in nodes if n.status == SyncStatus.FAILED),
|
||||
"in_progress": sum(1 for n in nodes if n.status == SyncStatus.IN_PROGRESS),
|
||||
"skipped": sum(1 for n in nodes if n.status == SyncStatus.SKIPPED),
|
||||
"by_type": {}
|
||||
}
|
||||
|
||||
# 按类型统计
|
||||
for node_type in self._handlers.keys():
|
||||
type_nodes = collection.filter(node_type=node_type)
|
||||
summary["by_type"][node_type] = {
|
||||
"total": len(type_nodes),
|
||||
"success": sum(1 for n in type_nodes if n.status == SyncStatus.SUCCESS),
|
||||
"failed": sum(1 for n in type_nodes if n.status == SyncStatus.FAILED),
|
||||
}
|
||||
|
||||
return summary
|
||||
|
||||
async def close(self) -> None:
|
||||
"""关闭数据源资源（子类可覆盖以实现特定的清理逻辑）"""
|
||||
pass # 基类默认空实现
|
||||
@@ -0,0 +1,579 @@
|
||||
"""
|
||||
节点处理器 V2 (Node Handler - Pure Execution Layer)
|
||||
|
||||
Handler 职责：
|
||||
1. 执行后端 API 调用（load/create/update/delete）
|
||||
2. 返回执行结果（TaskResult）
|
||||
3. 提供异步任务轮询（poll）
|
||||
4. 提供数据质量检查（validate）
|
||||
|
||||
Handler 不负责：
|
||||
- ❌ 状态管理（由 DataSource 负责）
|
||||
- ❌ ID 映射（由 Strategy 负责，已保存在 node.data）
|
||||
- ❌ 依赖关系（由 Strategy 负责）
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import (
|
||||
TypeVar, Generic, Type, Protocol, Dict, List, Any, Optional,
|
||||
runtime_checkable, TYPE_CHECKING
|
||||
)
|
||||
from abc import ABC, abstractmethod
|
||||
from pydantic import BaseModel
|
||||
|
||||
from .task_result import TaskResult
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..common.sync_node import SyncNode
|
||||
from ..common.collection import DataCollection
|
||||
from .api.client import ApiClient
|
||||
|
||||
T = TypeVar("T", bound=BaseModel)
|
||||
|
||||
|
||||
class ValidationResult:
|
||||
"""数据质量检查结果"""
|
||||
def __init__(self, is_valid: bool, errors: Optional[List[str]] = None):
|
||||
self.is_valid = is_valid
|
||||
self.errors = errors or []
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class NodeHandler(Protocol[T]):
|
||||
"""
|
||||
节点处理器协议（纯执行层）
|
||||
|
||||
Handler 只负责调用后端 API，不管理状态。
|
||||
"""
|
||||
|
||||
@property
|
||||
def node_type(self) -> str:
|
||||
"""节点类型（如 'contract'）"""
|
||||
...
|
||||
|
||||
@property
|
||||
def node_class(self) -> Type["SyncNode[T]"]:
|
||||
"""SyncNode 类型"""
|
||||
...
|
||||
|
||||
@property
|
||||
def schema(self) -> Type[T]:
|
||||
"""Schema 类型"""
|
||||
...
|
||||
|
||||
def set_collection(self, collection: "DataCollection") -> None:
|
||||
"""
|
||||
设置 Collection 引用
|
||||
|
||||
Args:
|
||||
collection: 数据集合
|
||||
"""
|
||||
...
|
||||
|
||||
def set_target_project_ids(self, target_project_ids: List[str]) -> None:
|
||||
"""注入项目过滤列表。默认可忽略。"""
|
||||
...
|
||||
|
||||
def set_api_client(self, client: "ApiClient") -> None:
|
||||
"""注入 API 客户端。默认可忽略。"""
|
||||
...
|
||||
|
||||
def set_poll_mode(self, mode: str) -> None:
|
||||
"""设置轮询模式。默认可忽略。"""
|
||||
...
|
||||
|
||||
def get_update_fields(self, *, exclude: frozenset[str] = frozenset({"id"})) -> List[str]:
|
||||
"""返回 post-check 可比较字段列表。默认空。"""
|
||||
...
|
||||
|
||||
async def load(self) -> List['SyncNode']:
|
||||
"""
|
||||
从后端加载全部数据并创建节点
|
||||
|
||||
注意：调用前需先 set_collection()
|
||||
|
||||
Returns:
|
||||
SyncNode 列表
|
||||
|
||||
示例：
|
||||
# Project Handler（无依赖）
|
||||
async def load(self):
|
||||
response = await self.api.get_projects()
|
||||
nodes = []
|
||||
for data in response["data"]:
|
||||
node = self.create_node(data)
|
||||
node.context["project_id"] = data["id"]
|
||||
nodes.append(node)
|
||||
return nodes
|
||||
|
||||
# Contract Handler（依赖 Project）
|
||||
async def load(self):
|
||||
# 使用 self._collection 查询依赖
|
||||
projects = self._collection.filter(node_type="project")
|
||||
project_ids = [p.data_id for p in projects if p.data_id]
|
||||
|
||||
nodes = []
|
||||
for project_id in project_ids:
|
||||
response = await self.api.get_contracts(project_id=project_id)
|
||||
for data in response["data"]:
|
||||
node = self.create_node(data)
|
||||
node.context["project_id"] = project_id
|
||||
node.context["contract_id"] = data["id"]
|
||||
nodes.append(node)
|
||||
return nodes
|
||||
"""
|
||||
...
|
||||
|
||||
async def create_all(
|
||||
self,
|
||||
nodes: List["SyncNode"]
|
||||
) -> List[TaskResult]:
|
||||
"""
|
||||
批量创建节点
|
||||
|
||||
Args:
|
||||
nodes: 需要创建的节点列表（DataSource 已筛选为 CREATE + PENDING）
|
||||
|
||||
Returns:
|
||||
List[TaskResult]:
|
||||
每个 TaskResult 必须包含 node_id，用于定位节点：
|
||||
- TaskResult(status=SUCCESS, node_id="xxx", data_id="yyy")
|
||||
- TaskResult(status=FAILED, node_id="xxx", error="...")
|
||||
- TaskResult(status=IN_PROGRESS, node_id="xxx", task_id="...")
|
||||
|
||||
示例：
|
||||
# JSONL Handler
|
||||
async def create_all(self, nodes):
|
||||
results = []
|
||||
for node in nodes:
|
||||
try:
|
||||
new_id = str(uuid4())
|
||||
data = node.get_data()
|
||||
# ... 存储逻辑 ...
|
||||
results.append(TaskResult.success(
|
||||
node_id=node.node_id,
|
||||
data_id=new_id
|
||||
))
|
||||
except Exception as e:
|
||||
results.append(TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error=str(e)
|
||||
))
|
||||
return results
|
||||
|
||||
# API Handler - 查询依赖
|
||||
async def create_all(self, nodes):
|
||||
results = []
|
||||
for node in nodes:
|
||||
# 查询依赖数据（如合同创建时查项目）
|
||||
project_id = node.context.get("project_id")
|
||||
projects = self._collection.filter(
|
||||
node_type="project",
|
||||
node_filter={"data_id": project_id}
|
||||
)
|
||||
|
||||
if not projects:
|
||||
results.append(TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error=f"Project {project_id} not found"
|
||||
))
|
||||
continue
|
||||
|
||||
# 构造请求数据
|
||||
data = node.get_data()
|
||||
if not data:
|
||||
results.append(TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error="node data is None"
|
||||
))
|
||||
continue
|
||||
|
||||
contract_data = {
|
||||
"project_id": projects[0].data_id,
|
||||
**data
|
||||
}
|
||||
|
||||
# 调用 API
|
||||
response = await self.api.create_contract(contract_data)
|
||||
results.append(TaskResult.success(
|
||||
node_id=node.node_id,
|
||||
data_id=response["id"]
|
||||
))
|
||||
|
||||
return results
|
||||
"""
|
||||
...
|
||||
|
||||
async def update_all(
|
||||
self,
|
||||
nodes: List["SyncNode"]
|
||||
) -> List[TaskResult]:
|
||||
"""
|
||||
批量更新节点
|
||||
|
||||
Args:
|
||||
nodes: 需要更新的节点列表（DataSource 已筛选为 UPDATE + PENDING）
|
||||
|
||||
Returns:
|
||||
List[TaskResult]（同 create_all）
|
||||
"""
|
||||
...
|
||||
|
||||
async def delete_all(
|
||||
self,
|
||||
nodes: List["SyncNode"]
|
||||
) -> List[TaskResult]:
|
||||
"""
|
||||
批量删除节点
|
||||
|
||||
Args:
|
||||
nodes: 需要删除的节点列表（DataSource 已筛选为 DELETE + PENDING）
|
||||
|
||||
Args:
|
||||
collection: Collection 上下文
|
||||
|
||||
Returns:
|
||||
List[TaskResult]（同 create_all）
|
||||
"""
|
||||
...
|
||||
|
||||
async def poll_tasks(self, task_ids: List[str]) -> Dict[str, TaskResult]:
|
||||
"""
|
||||
批量轮询异步任务状态
|
||||
|
||||
Args:
|
||||
task_ids: 任务 ID 列表
|
||||
|
||||
Returns:
|
||||
{task_id: TaskResult} 字典
|
||||
- SUCCESS: 任务完成，返回 data_id
|
||||
- FAILED: 任务失败，返回 error
|
||||
- IN_PROGRESS: 仍在进行中
|
||||
|
||||
示例：
|
||||
async def poll_tasks(self, task_ids):
|
||||
response = await self.api.batch_query_tasks(task_ids)
|
||||
results = {}
|
||||
for task in response["tasks"]:
|
||||
if task["status"] == "completed":
|
||||
results[task["id"]] = TaskResult.success(
|
||||
data_id=task["result"]["contract_id"]
|
||||
)
|
||||
elif task["status"] == "failed":
|
||||
results[task["id"]] = TaskResult.failed(
|
||||
error=task["error"]
|
||||
)
|
||||
else: # running
|
||||
results[task["id"]] = TaskResult.in_progress(
|
||||
task_id=task["id"]
|
||||
)
|
||||
return results
|
||||
"""
|
||||
...
|
||||
|
||||
async def validate(self, data: Dict[str, Any]) -> ValidationResult:
|
||||
"""
|
||||
数据质量检查
|
||||
|
||||
Args:
|
||||
data: 原始数据
|
||||
|
||||
Returns:
|
||||
ValidationResult（是否有效 + 错误列表）
|
||||
|
||||
示例：
|
||||
async def validate(self, data):
|
||||
errors = []
|
||||
if not data.get("code"):
|
||||
errors.append("合同编码为空")
|
||||
if not data.get("project_id"):
|
||||
errors.append("项目 ID 为空")
|
||||
|
||||
return ValidationResult(
|
||||
is_valid=len(errors) == 0,
|
||||
errors=errors
|
||||
)
|
||||
"""
|
||||
...
|
||||
|
||||
def extract_id(self, data: Dict[str, Any]) -> str:
|
||||
"""
|
||||
从原始数据提取 ID
|
||||
|
||||
Args:
|
||||
data: 原始数据
|
||||
|
||||
Returns:
|
||||
业务主键
|
||||
"""
|
||||
...
|
||||
|
||||
def _create_node(self, data: Dict[str, Any]) -> "SyncNode[T]":
|
||||
"""
|
||||
从原始数据创建 SyncNode
|
||||
|
||||
Args:
|
||||
data: 原始数据
|
||||
|
||||
Returns:
|
||||
类型化的 SyncNode
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
class BaseNodeHandler(ABC, Generic[T]):
|
||||
"""
|
||||
节点处理器基类
|
||||
|
||||
子类需实现：
|
||||
- load()
|
||||
- create()
|
||||
- update()
|
||||
- delete()（可选）
|
||||
- poll_tasks()（如果支持异步）
|
||||
- validate()（可选）
|
||||
- extract_id()
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
node_type: str,
|
||||
node_class: Type["SyncNode[T]"],
|
||||
schema: Type[T]
|
||||
):
|
||||
self._node_type = node_type
|
||||
self._node_class = node_class
|
||||
self._schema = schema
|
||||
self._collection: Optional["DataCollection"] = None
|
||||
|
||||
def set_collection(self, collection: "DataCollection") -> None:
|
||||
"""
|
||||
设置 Collection 引用
|
||||
|
||||
Args:
|
||||
collection: 数据集合
|
||||
"""
|
||||
self._collection = collection
|
||||
|
||||
def set_target_project_ids(self, target_project_ids: List[str]) -> None:
|
||||
"""默认忽略项目过滤能力。"""
|
||||
return
|
||||
|
||||
def set_api_client(self, client: "ApiClient") -> None:
|
||||
"""默认忽略 API 客户端注入。"""
|
||||
return
|
||||
|
||||
def set_poll_mode(self, mode: str) -> None:
|
||||
"""默认忽略轮询模式。"""
|
||||
return
|
||||
|
||||
def get_update_fields(self, *, exclude: frozenset[str] = frozenset({"id"})) -> List[str]:
|
||||
"""默认无 post-check 字段白名单。"""
|
||||
return []
|
||||
|
||||
@property
|
||||
def node_type(self) -> str:
|
||||
return self._node_type
|
||||
|
||||
@property
|
||||
def node_class(self) -> Type["SyncNode[T]"]:
|
||||
return self._node_class
|
||||
|
||||
@property
|
||||
def schema(self) -> Type[T]:
|
||||
return self._schema
|
||||
|
||||
# ========== 必须实现 ==========
|
||||
|
||||
@abstractmethod
|
||||
async def load(self) -> List['SyncNode']:
|
||||
"""
|
||||
从数据源加载数据并创建节点
|
||||
|
||||
Handler 负责：
|
||||
1. 加载原始数据
|
||||
2. 创建 SyncNode（调用 create_node）
|
||||
3. 设置 context（如 project_id, contract_id 等）
|
||||
|
||||
Returns:
|
||||
SyncNode 列表
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def create_all(
|
||||
self,
|
||||
nodes: List["SyncNode"]
|
||||
) -> List[TaskResult]:
|
||||
"""批量创建节点"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def update_all(
|
||||
self,
|
||||
nodes: List["SyncNode"]
|
||||
) -> List[TaskResult]:
|
||||
"""批量更新节点"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def extract_id(self, data: Dict[str, Any]) -> str:
|
||||
"""提取 ID"""
|
||||
pass
|
||||
|
||||
# ========== 可选实现 ==========
|
||||
|
||||
async def delete_all(
|
||||
self,
|
||||
nodes: List["SyncNode"]
|
||||
) -> List[TaskResult]:
|
||||
"""批量删除节点（默认不支持）"""
|
||||
return [
|
||||
TaskResult.failed(node_id=node.node_id, error="Delete operation not supported")
|
||||
for node in nodes
|
||||
]
|
||||
|
||||
async def poll_tasks(self, task_ids: List[str]) -> Dict[str, TaskResult]:
|
||||
"""轮询异步任务（默认不支持）"""
|
||||
return {
|
||||
task_id: TaskResult.failed(error="Async tasks not supported")
|
||||
for task_id in task_ids
|
||||
}
|
||||
|
||||
async def validate(self, data: Dict[str, Any]) -> ValidationResult:
|
||||
"""数据质量检查（默认不检查）"""
|
||||
return ValidationResult(is_valid=True)
|
||||
|
||||
def create_node(self, data: Dict[str, Any]) -> "SyncNode[T]":
|
||||
"""创建或复用 SyncNode（默认实现）"""
|
||||
data_id = self.extract_id(data)
|
||||
if self._collection is None:
|
||||
raise RuntimeError("Collection not set. Call set_collection() before create_node().")
|
||||
|
||||
if data_id:
|
||||
existing_node = self._collection.get_by_data_id(self.node_type, data_id)
|
||||
if existing_node is not None:
|
||||
existing_node.set_data(data.copy())
|
||||
existing_node.set_origin_data(data.copy())
|
||||
return existing_node
|
||||
|
||||
node_id = self._collection.generate_node_id()
|
||||
|
||||
return self.node_class(
|
||||
node_id=node_id,
|
||||
data_id=data_id,
|
||||
data=data.copy()
|
||||
)
|
||||
|
||||
def _create_node(self, data: Dict[str, Any]) -> "SyncNode[T]":
|
||||
"""兼容 NodeHandler 协议的默认实现"""
|
||||
return self.create_node(data)
|
||||
|
||||
|
||||
class CompatibleNodeHandler(Generic[T]):
|
||||
"""兼容旧式 Handler，实现扩展后的可选接口并委托给原对象。"""
|
||||
|
||||
def __init__(self, handler: NodeHandler[T]):
|
||||
self._handler = handler
|
||||
|
||||
@property
|
||||
def node_type(self) -> str:
|
||||
return self._handler.node_type
|
||||
|
||||
@property
|
||||
def node_class(self) -> Type["SyncNode[T]"]:
|
||||
return self._handler.node_class
|
||||
|
||||
@property
|
||||
def schema(self) -> Type[T]:
|
||||
return self._handler.schema
|
||||
|
||||
def set_collection(self, collection: "DataCollection") -> None:
|
||||
self._handler.set_collection(collection)
|
||||
|
||||
def set_target_project_ids(self, target_project_ids: List[str]) -> None:
|
||||
method = getattr(self._handler, "set_target_project_ids", None)
|
||||
if callable(method):
|
||||
method(target_project_ids)
|
||||
|
||||
def set_api_client(self, client: "ApiClient") -> None:
|
||||
method = getattr(self._handler, "set_api_client", None)
|
||||
if callable(method):
|
||||
method(client)
|
||||
|
||||
def set_poll_mode(self, mode: str) -> None:
|
||||
method = getattr(self._handler, "set_poll_mode", None)
|
||||
if callable(method):
|
||||
method(mode)
|
||||
|
||||
def get_update_fields(self, *, exclude: frozenset[str] = frozenset({"id"})) -> List[str]:
|
||||
method = getattr(self._handler, "get_update_fields", None)
|
||||
if callable(method):
|
||||
return method(exclude=exclude)
|
||||
return []
|
||||
|
||||
async def load(self) -> List["SyncNode"]:
|
||||
return await self._handler.load()
|
||||
|
||||
async def create_all(self, nodes: List["SyncNode"]) -> List[TaskResult]:
|
||||
return await self._handler.create_all(nodes)
|
||||
|
||||
async def update_all(self, nodes: List["SyncNode"]) -> List[TaskResult]:
|
||||
return await self._handler.update_all(nodes)
|
||||
|
||||
async def delete_all(self, nodes: List["SyncNode"]) -> List[TaskResult]:
|
||||
return await self._handler.delete_all(nodes)
|
||||
|
||||
async def poll_tasks(self, task_ids: List[str]) -> Dict[str, TaskResult]:
|
||||
return await self._handler.poll_tasks(task_ids)
|
||||
|
||||
async def validate(self, data: Dict[str, Any]) -> ValidationResult:
|
||||
return await self._handler.validate(data)
|
||||
|
||||
def extract_id(self, data: Dict[str, Any]) -> str:
|
||||
return self._handler.extract_id(data)
|
||||
|
||||
def _create_node(self, data: Dict[str, Any]) -> "SyncNode[T]":
|
||||
return self._handler._create_node(data)
|
||||
|
||||
def __getattr__(self, item: str) -> Any:
|
||||
return getattr(self._handler, item)
|
||||
|
||||
|
||||
def ensure_handler_compat(handler: NodeHandler[T]) -> NodeHandler[T]:
|
||||
if isinstance(handler, BaseNodeHandler) or isinstance(handler, CompatibleNodeHandler):
|
||||
return handler
|
||||
return CompatibleNodeHandler(handler)
|
||||
|
||||
|
||||
class NodeHandlerRegistry:
|
||||
"""节点处理器注册表"""
|
||||
|
||||
_handlers: Dict[str, NodeHandler] = {}
|
||||
|
||||
@classmethod
|
||||
def register(cls, node_type: str, handler: NodeHandler) -> None:
|
||||
"""注册 handler"""
|
||||
cls._handlers[node_type] = handler
|
||||
|
||||
@classmethod
|
||||
def get(cls, node_type: str) -> NodeHandler:
|
||||
"""获取 handler"""
|
||||
if node_type not in cls._handlers:
|
||||
raise KeyError(f"Handler for '{node_type}' not registered")
|
||||
return cls._handlers[node_type]
|
||||
|
||||
@classmethod
|
||||
def has(cls, node_type: str) -> bool:
|
||||
"""检查是否已注册"""
|
||||
return node_type in cls._handlers
|
||||
|
||||
@classmethod
|
||||
def clear(cls) -> None:
|
||||
"""清空注册表"""
|
||||
cls._handlers.clear()
|
||||
|
||||
@classmethod
|
||||
def list_handlers(cls) -> List[str]:
|
||||
"""列出所有已注册的 node_type"""
|
||||
return list(cls._handlers.keys())
|
||||
@@ -0,0 +1,13 @@
|
||||
"""
|
||||
JSONL DataSource 模块
|
||||
|
||||
提供基于 JSONL 文件的数据源实现，用于测试和本地开发。
|
||||
"""
|
||||
|
||||
from .datasource import JsonlDataSource
|
||||
from .handler import BaseJsonlHandler
|
||||
|
||||
__all__ = [
|
||||
"JsonlDataSource",
|
||||
"BaseJsonlHandler",
|
||||
]
|
||||
@@ -0,0 +1,67 @@
|
||||
"""
|
||||
JsonlDataSource - JSONL 文件数据源
|
||||
|
||||
职责：
|
||||
- 提供 JSONL 文件目录管理
|
||||
- 实现 DataSource 接口（load_all, sync_all）
|
||||
- 管理内存缓存和文件写入
|
||||
- 统一异常处理和状态管理
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, Set
|
||||
|
||||
from ..datasource import BaseDataSource
|
||||
|
||||
|
||||
|
||||
|
||||
class JsonlDataSource(BaseDataSource):
|
||||
"""
|
||||
JSONL 文件数据源
|
||||
|
||||
提供文件路径和基础功能，具体的文件查找和读取由各个 Handler 负责。
|
||||
|
||||
Args:
|
||||
dir_path: JSONL 文件目录
|
||||
read_only: 是否只读模式
|
||||
|
||||
示例：
|
||||
datasource = JsonlDataSource(Path("filtered_datasource/datasource/ecm-2025-12-02"))
|
||||
datasource.register_handler(ContractJsonlHandler(datasource))
|
||||
|
||||
collection = DataCollection()
|
||||
datasource.set_collection(collection)
|
||||
await datasource.load_all(order=["project", "contract"])
|
||||
await datasource.sync_all(order=["project", "contract"])
|
||||
"""
|
||||
|
||||
def __init__(self, dir_path: Path, read_only: bool = False):
|
||||
super().__init__()
|
||||
self.dir_path = Path(dir_path)
|
||||
self.read_only = read_only
|
||||
|
||||
# 用于 create/update/delete 的内存缓存
|
||||
self._data: Dict[str, Dict[str, Dict[str, Any]]] = {} # {node_type: {id: data}}
|
||||
self._dirty_types: Set[str] = set()
|
||||
|
||||
def _on_node_loaded(self, handler, node) -> None:
|
||||
node.append_log(
|
||||
f"JSONL链路[load] dir={self.dir_path.as_posix()} node_type={node.node_type} data_id={node.data_id or '<empty>'}"
|
||||
)
|
||||
|
||||
def _on_in_progress_result(self, handler, node, result) -> None:
|
||||
node.append_log(
|
||||
f"JSONL链路[submit] action={node.action.value} data_id={node.data_id or '<empty>'}"
|
||||
)
|
||||
|
||||
def _on_poll_result(self, handler, node, task_id: str, result) -> None:
|
||||
node.append_log(
|
||||
f"JSONL链路[poll] task_id={task_id} status={result.status.value}"
|
||||
)
|
||||
|
||||
# load_all 和 sync_all 继承自 BaseDataSource
|
||||
# 保存逻辑在 BaseJsonlHandler 中实现
|
||||
|
||||
@@ -0,0 +1,350 @@
|
||||
"""
|
||||
BaseJsonlHandler - JSONL Handler 基类
|
||||
|
||||
职责：
|
||||
- 查找和读取 JSONL 文件
|
||||
- 实现批量 Handler 接口（load, create_all, update_all, delete_all）
|
||||
- 使用 datasource 的内存缓存
|
||||
- 返回 List[TaskResult]（统一状态管理）
|
||||
- 捕获所有异常并转换为 TaskResult.failed()
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Dict, List, Any, TYPE_CHECKING, Optional, Set
|
||||
from uuid import uuid4
|
||||
|
||||
from ..handler import BaseNodeHandler
|
||||
from ..task_result import TaskResult
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ...common.sync_node import SyncNode
|
||||
from .datasource import JsonlDataSource
|
||||
|
||||
|
||||
class BaseJsonlHandler(BaseNodeHandler):
|
||||
"""
|
||||
JSONL Handler 基类
|
||||
|
||||
负责查找和读取对应的 JSONL 文件。
|
||||
|
||||
示例：
|
||||
class ContractJsonlHandler(BaseJsonlHandler):
|
||||
def __init__(self, datasource: JsonlDataSource):
|
||||
super().__init__(
|
||||
datasource=datasource,
|
||||
node_type="contract",
|
||||
node_class=ContractSyncNode,
|
||||
schema=ContractResponse
|
||||
)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
datasource: JsonlDataSource,
|
||||
node_type: str,
|
||||
node_class: type,
|
||||
schema: type
|
||||
):
|
||||
super().__init__(node_type, node_class, schema)
|
||||
self.datasource = datasource
|
||||
self._target_project_ids: Optional[Set[str]] = None
|
||||
|
||||
def set_target_project_ids(self, target_project_ids: List[str]) -> None:
|
||||
"""注入项目过滤列表，供 load 阶段预过滤使用。"""
|
||||
if target_project_ids:
|
||||
self._target_project_ids = {str(pid) for pid in target_project_ids if str(pid)}
|
||||
else:
|
||||
self._target_project_ids = None
|
||||
|
||||
def _should_skip_item_by_project_filter(self, item: Dict[str, Any], item_id: str) -> bool:
|
||||
if not self._target_project_ids:
|
||||
return False
|
||||
if self.node_type == "project":
|
||||
return item_id not in self._target_project_ids
|
||||
|
||||
project_id = item.get("project_id")
|
||||
if project_id:
|
||||
return str(project_id) not in self._target_project_ids
|
||||
return False
|
||||
|
||||
async def load(self) -> List['SyncNode']:
|
||||
"""
|
||||
查找并加载对应的 JSONL 文件并创建节点
|
||||
|
||||
文件命名规则：{node_type}_N.jsonl (N 为数字)
|
||||
例如：contract_1.jsonl, contract_2.jsonl
|
||||
|
||||
注意：load 操作的异常由 DataSource 的 load_all 捕获处理
|
||||
"""
|
||||
nodes = []
|
||||
|
||||
if not self.datasource.dir_path.exists():
|
||||
return nodes
|
||||
|
||||
# 查找所有匹配的文件：{node_type}_N.jsonl
|
||||
pattern = re.compile(rf"^{re.escape(self.node_type)}_\d+\.jsonl$")
|
||||
|
||||
for file_path in self.datasource.dir_path.iterdir():
|
||||
if not pattern.match(file_path.name):
|
||||
continue
|
||||
|
||||
# 读取文件
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
for line_num, line in enumerate(f, 1):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
try:
|
||||
item = json.loads(line)
|
||||
item_id = self.extract_id(item)
|
||||
|
||||
if self._should_skip_item_by_project_filter(item, item_id):
|
||||
continue
|
||||
|
||||
# 创建节点
|
||||
node = self.create_node(item)
|
||||
nodes.append(node)
|
||||
|
||||
# 同时加载到缓存
|
||||
if item_id:
|
||||
bucket = self.datasource._data.setdefault(self.node_type, {})
|
||||
bucket[item_id] = item
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"Warning: Invalid JSON in {file_path.name}:{line_num}: {e}")
|
||||
# 继续处理下一行
|
||||
|
||||
return nodes
|
||||
|
||||
def extract_id(self, data: Dict[str, Any]) -> str:
|
||||
"""
|
||||
提取数据 ID
|
||||
|
||||
默认实现：优先使用 'id'，其次 '_id'
|
||||
子类可重写此方法。
|
||||
|
||||
Args:
|
||||
data: 数据字典
|
||||
|
||||
Returns:
|
||||
数据 ID
|
||||
"""
|
||||
return str(data.get("id") or data.get("_id") or "")
|
||||
|
||||
async def create_all(
|
||||
self,
|
||||
nodes: List['SyncNode']
|
||||
) -> List[TaskResult]:
|
||||
"""
|
||||
批量创建节点
|
||||
|
||||
Args:
|
||||
nodes: 需要创建的节点列表（DataSource 已筛选为 CREATE + PENDING）
|
||||
|
||||
Returns:
|
||||
List[TaskResult]，每个结果关联 node_id
|
||||
"""
|
||||
results = []
|
||||
|
||||
for node in nodes:
|
||||
try:
|
||||
# 生成新 ID（如果没有）
|
||||
if not node.data_id:
|
||||
new_id = str(uuid4())
|
||||
else:
|
||||
new_id = node.data_id
|
||||
|
||||
# 获取数据字典
|
||||
# 使用 exclude_unset=False 以确保测试中的完整性校验（如 amount 字段）能通过
|
||||
# 因为 JSONL 存储通常需要完整记录
|
||||
data_dict = node.get_data(exclude_unset=False)
|
||||
if data_dict is None:
|
||||
results.append(TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error="Cannot get data from node"
|
||||
))
|
||||
continue
|
||||
|
||||
# 设置 ID 和 type
|
||||
data_dict["id"] = new_id
|
||||
data_dict["_id"] = new_id
|
||||
if "type" not in data_dict:
|
||||
data_dict["type"] = self.node_type
|
||||
|
||||
# 存储到缓存
|
||||
bucket = self.datasource._data.setdefault(self.node_type, {})
|
||||
bucket[new_id] = data_dict
|
||||
self.datasource._dirty_types.add(self.node_type)
|
||||
|
||||
results.append(TaskResult.success(
|
||||
node_id=node.node_id,
|
||||
data_id=new_id
|
||||
))
|
||||
|
||||
except Exception as e:
|
||||
results.append(TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error=f"Create failed: {str(e)}"
|
||||
))
|
||||
|
||||
# 自动保存到文件
|
||||
await self._save_to_file()
|
||||
|
||||
return results
|
||||
|
||||
async def update_all(
|
||||
self,
|
||||
nodes: List['SyncNode']
|
||||
) -> List[TaskResult]:
|
||||
"""
|
||||
批量更新节点
|
||||
|
||||
Args:
|
||||
nodes: 需要更新的节点列表（DataSource 已筛选为 UPDATE + PENDING）
|
||||
|
||||
Returns:
|
||||
List[TaskResult]，每个结果关联 node_id
|
||||
"""
|
||||
results = []
|
||||
|
||||
for node in nodes:
|
||||
try:
|
||||
if not node.data_id:
|
||||
results.append(TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error="Cannot update node without data_id"
|
||||
))
|
||||
continue
|
||||
|
||||
# 获取数据字典
|
||||
# JSONL 更新通常也是完整记录替换，所以使用 exclude_unset=False
|
||||
data_dict = node.get_data(exclude_unset=False)
|
||||
if data_dict is None:
|
||||
results.append(TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error="Cannot get data from node"
|
||||
))
|
||||
continue
|
||||
|
||||
# 检查是否存在
|
||||
bucket = self.datasource._data.get(self.node_type, {})
|
||||
existing = bucket.get(node.data_id)
|
||||
if existing is None:
|
||||
results.append(TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error=f"Item {node.data_id} not found"
|
||||
))
|
||||
continue
|
||||
|
||||
# 合并更新
|
||||
existing.update(data_dict)
|
||||
existing["id"] = node.data_id
|
||||
existing["_id"] = node.data_id
|
||||
if "type" not in existing:
|
||||
existing["type"] = self.node_type
|
||||
|
||||
# 更新缓存
|
||||
bucket[node.data_id] = existing
|
||||
self.datasource._dirty_types.add(self.node_type)
|
||||
|
||||
results.append(TaskResult.success(
|
||||
node_id=node.node_id,
|
||||
data_id=node.data_id
|
||||
))
|
||||
|
||||
except Exception as e:
|
||||
results.append(TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error=f"Update failed: {str(e)}"
|
||||
))
|
||||
|
||||
# 自动保存到文件
|
||||
await self._save_to_file()
|
||||
|
||||
return results
|
||||
|
||||
async def _save_to_file(self) -> None:
|
||||
"""
|
||||
将缓存数据写回 JSONL 文件
|
||||
|
||||
在批量操作（create_all/update_all/delete_all）后自动调用。
|
||||
"""
|
||||
if self.datasource.read_only:
|
||||
return
|
||||
|
||||
if self.node_type not in self.datasource._dirty_types:
|
||||
return
|
||||
|
||||
items = self.datasource._data.get(self.node_type, {})
|
||||
if not items:
|
||||
# 如果缓存为空，可能是删除了所有数据，仍需写入空文件
|
||||
pass
|
||||
|
||||
try:
|
||||
# 写入到 {node_type}_1.jsonl
|
||||
file_path = self.datasource.dir_path / f"{self.node_type}_1.jsonl"
|
||||
file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with open(file_path, "w", encoding="utf-8") as f:
|
||||
for item in items.values():
|
||||
f.write(json.dumps(item, ensure_ascii=False) + "\n")
|
||||
|
||||
# 清除该类型的 dirty 标记
|
||||
self.datasource._dirty_types.discard(self.node_type)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error saving {self.node_type}: {str(e)}")
|
||||
# 不抛出异常，继续执行
|
||||
|
||||
async def delete_all(
|
||||
self,
|
||||
nodes: List['SyncNode']
|
||||
) -> List[TaskResult]:
|
||||
"""
|
||||
批量删除节点
|
||||
|
||||
Args:
|
||||
nodes: 需要删除的节点列表（DataSource 已筛选为 DELETE + PENDING）
|
||||
|
||||
Returns:
|
||||
List[TaskResult]，每个结果关联 node_id
|
||||
"""
|
||||
results = []
|
||||
|
||||
for node in nodes:
|
||||
try:
|
||||
if not node.data_id:
|
||||
results.append(TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error="Cannot delete node without data_id"
|
||||
))
|
||||
continue
|
||||
|
||||
# 删除缓存
|
||||
bucket = self.datasource._data.get(self.node_type, {})
|
||||
if node.data_id in bucket:
|
||||
del bucket[node.data_id]
|
||||
self.datasource._dirty_types.add(self.node_type)
|
||||
results.append(TaskResult.success(
|
||||
node_id=node.node_id
|
||||
))
|
||||
else:
|
||||
results.append(TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error=f"Item {node.data_id} not found"
|
||||
))
|
||||
|
||||
except Exception as e:
|
||||
results.append(TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error=f"Delete failed: {str(e)}"
|
||||
))
|
||||
|
||||
# 自动保存到文件
|
||||
await self._save_to_file()
|
||||
|
||||
return results
|
||||
@@ -0,0 +1,122 @@
|
||||
"""
|
||||
TaskResult - Handler 执行结果
|
||||
|
||||
统一的任务执行结果，用于所有 DataSource 的 Handler。
|
||||
"""
|
||||
|
||||
from enum import Enum
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
|
||||
class TaskStatus(Enum):
|
||||
"""任务执行状态"""
|
||||
SUCCESS = "success" # 同步完成，立即成功
|
||||
FAILED = "failed" # 同步失败
|
||||
IN_PROGRESS = "in_progress" # 异步任务进行中
|
||||
SKIPPED = "skipped" # 跳过（无实际更新或不符合更新条件）
|
||||
|
||||
|
||||
class TaskResult:
|
||||
"""
|
||||
Handler 执行结果
|
||||
|
||||
用于封装 create/update/delete 操作的返回值。
|
||||
适用于所有 DataSource（API、JSONL 等）。
|
||||
支持批量操作时通过 node_id 定位结果。
|
||||
"""
|
||||
def __init__(
|
||||
self,
|
||||
status: TaskStatus,
|
||||
node_id: Optional[str] = None,
|
||||
data_id: Optional[str] = None,
|
||||
task_id: Optional[str] = None,
|
||||
push_id: Optional[str] = None,
|
||||
error: Optional[str] = None,
|
||||
sync_log: Optional[str] = None,
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
):
|
||||
"""
|
||||
Args:
|
||||
status: 任务状态（SUCCESS/FAILED/IN_PROGRESS）
|
||||
node_id: 关联的节点 ID（批量操作时用于定位节点）
|
||||
data_id: 成功时返回的业务 ID（CREATE 操作必需）
|
||||
task_id: 异步任务 ID（status=IN_PROGRESS 时必需）
|
||||
push_id: 推送请求 ID（用于统一 push/poll 跟踪）
|
||||
error: 失败原因（status=FAILED 时必需）
|
||||
sync_log: Debug/Info级别日志（SKIPPED时使用）
|
||||
metadata: 其他元数据
|
||||
"""
|
||||
normalized_metadata = dict(metadata or {})
|
||||
if push_id is None:
|
||||
raw_push_id = normalized_metadata.get("push_id")
|
||||
if isinstance(raw_push_id, str) and raw_push_id:
|
||||
push_id = raw_push_id
|
||||
elif push_id:
|
||||
normalized_metadata.setdefault("push_id", push_id)
|
||||
|
||||
self.status = status
|
||||
self.node_id = node_id
|
||||
self.data_id = data_id
|
||||
self.task_id = task_id
|
||||
self.push_id = push_id
|
||||
self.error = error
|
||||
self.sync_log = sync_log
|
||||
self.metadata = normalized_metadata
|
||||
|
||||
@classmethod
|
||||
def success(
|
||||
cls,
|
||||
node_id: Optional[str] = None,
|
||||
data_id: Optional[str] = None,
|
||||
push_id: Optional[str] = None,
|
||||
**metadata,
|
||||
) -> "TaskResult":
|
||||
"""创建成功结果"""
|
||||
return cls(TaskStatus.SUCCESS, node_id=node_id, data_id=data_id, push_id=push_id, metadata=metadata)
|
||||
|
||||
@classmethod
|
||||
def failed(
|
||||
cls,
|
||||
node_id: Optional[str] = None,
|
||||
error: str = "",
|
||||
push_id: Optional[str] = None,
|
||||
**metadata,
|
||||
) -> "TaskResult":
|
||||
"""创建失败结果"""
|
||||
return cls(TaskStatus.FAILED, node_id=node_id, error=error, push_id=push_id, metadata=metadata)
|
||||
|
||||
@classmethod
|
||||
def skipped(
|
||||
cls,
|
||||
node_id: Optional[str] = None,
|
||||
reason: str = "",
|
||||
push_id: Optional[str] = None,
|
||||
**metadata,
|
||||
) -> "TaskResult":
|
||||
"""创建跳过结果（无实际更新等）"""
|
||||
return cls(TaskStatus.SKIPPED, node_id=node_id, sync_log=reason, push_id=push_id, metadata=metadata)
|
||||
|
||||
@classmethod
|
||||
def in_progress(
|
||||
cls,
|
||||
node_id: Optional[str] = None,
|
||||
task_id: str = "",
|
||||
push_id: Optional[str] = None,
|
||||
**metadata,
|
||||
) -> "TaskResult":
|
||||
"""创建进行中结果"""
|
||||
return cls(TaskStatus.IN_PROGRESS, node_id=node_id, task_id=task_id, push_id=push_id, metadata=metadata)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
parts = [f"status={self.status.value}"]
|
||||
if self.node_id:
|
||||
parts.append(f"node_id={self.node_id}")
|
||||
if self.data_id:
|
||||
parts.append(f"data_id={self.data_id}")
|
||||
if self.task_id:
|
||||
parts.append(f"task_id={self.task_id}")
|
||||
if self.push_id:
|
||||
parts.append(f"push_id={self.push_id}")
|
||||
if self.error:
|
||||
parts.append(f"error={self.error!r}")
|
||||
return f"TaskResult({', '.join(parts)})"
|
||||
@@ -0,0 +1,32 @@
|
||||
"""
|
||||
Domain 模块
|
||||
|
||||
导入所有 domain 的 register 模块，触发 DomainRegistry 注册。
|
||||
"""
|
||||
|
||||
# 导入所有 domain register，触发注册
|
||||
from .company import register as _company_register
|
||||
from .user import register as _user_register
|
||||
from .supplier import register as _supplier_register
|
||||
from .project import register as _project_register
|
||||
from .project_detail import register as _project_detail_register
|
||||
from .contract import register as _contract_register
|
||||
from .construction import register as _construction_register
|
||||
from .construction_task_detail import register as _construction_task_detail_register
|
||||
from .material import register as _material_register
|
||||
from .material_detail import register as _material_detail_register
|
||||
from .contract_settlement import register as _contract_settlement_register
|
||||
from .contract_settlement_detail import register as _contract_settlement_detail_register
|
||||
from .personnel import register as _personnel_register
|
||||
from .personnel_detail import register as _personnel_detail_register
|
||||
from .preparation import register as _preparation_register
|
||||
from .production import register as _production_register
|
||||
from .lar import register as _lar_register
|
||||
from .lar_change import register as _lar_change_register
|
||||
from .units import register as _units_register
|
||||
from .contract_change import register as _contract_change_register
|
||||
from .attachment import register as _attachment_register
|
||||
|
||||
# 所有组件通过 DomainRegistry 访问，不再单独导出
|
||||
__all__ = []
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
"""
|
||||
Attachment Domain
|
||||
"""
|
||||
from .sync_node import AttachmentSyncNode
|
||||
from .sync_strategy import AttachmentSyncStrategy
|
||||
from .jsonl_handler import AttachmentJsonlHandler
|
||||
|
||||
__all__ = ["AttachmentSyncNode", "AttachmentSyncStrategy", "AttachmentJsonlHandler"]
|
||||
@@ -0,0 +1,102 @@
|
||||
"""
|
||||
Attachment API Handler
|
||||
|
||||
职责：
|
||||
- 加载附件列表（依赖 Project）
|
||||
- 附件创建/更新/删除（批量接口）
|
||||
"""
|
||||
|
||||
from typing import List, Dict, Any
|
||||
from ...datasource.api.handler import BaseApiHandler
|
||||
from ...datasource.task_result import TaskResult
|
||||
from ...common.sync_node import SyncNode
|
||||
from .sync_node import AttachmentSyncNode, AttachmentSchema
|
||||
|
||||
|
||||
class AttachmentApiHandler(BaseApiHandler):
|
||||
"""附件 API Handler"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._node_type = "attachment"
|
||||
self._node_class = AttachmentSyncNode
|
||||
self._schema = AttachmentSchema
|
||||
|
||||
async def load(self) -> List[SyncNode]:
|
||||
"""加载附件列表（依赖 Project）"""
|
||||
projects = self._collection.filter(node_type="project")
|
||||
project_ids = [p.data_id for p in projects if p.data_id]
|
||||
|
||||
if not project_ids:
|
||||
return []
|
||||
|
||||
all_attachments = []
|
||||
for project_id in project_ids:
|
||||
try:
|
||||
attachments = await api_get_attachment_list(self.api_client, project_id)
|
||||
all_attachments.extend(attachments)
|
||||
except Exception as e:
|
||||
print(f"❌ Failed to load attachments for project {project_id}: {e}")
|
||||
|
||||
return all_attachments
|
||||
|
||||
async def create_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""批量创建附件（已禁用）"""
|
||||
return [
|
||||
TaskResult.failed(node_id=node.node_id, error="Attachment push disabled")
|
||||
for node in nodes
|
||||
]
|
||||
|
||||
async def update_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""批量更新附件（已禁用）"""
|
||||
return [
|
||||
TaskResult.failed(node_id=node.node_id, error="Attachment push disabled")
|
||||
for node in nodes
|
||||
]
|
||||
|
||||
async def delete_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""批量删除附件（已禁用）"""
|
||||
return [
|
||||
TaskResult.failed(node_id=node.node_id, error="Attachment push disabled")
|
||||
for node in nodes
|
||||
]
|
||||
|
||||
async def poll_tasks(self, task_ids: List[str]) -> Dict[str, TaskResult]:
|
||||
"""轮询异步任务状态（占位）"""
|
||||
return await super().poll_tasks(task_ids)
|
||||
|
||||
def _create_node(self, data: Dict[str, Any]) -> SyncNode:
|
||||
node = self._create_basic_node(data, depend_ids=[])
|
||||
|
||||
if data.get("project_id"):
|
||||
node.context["project_id"] = data["project_id"]
|
||||
|
||||
return node
|
||||
|
||||
|
||||
# ========== 静态 API 函数 ==========
|
||||
|
||||
async def api_get_attachment_list(client, project_id: str) -> List[Dict[str, Any]]:
|
||||
response = await client.request(
|
||||
method="GET",
|
||||
endpoint="/attachment/list",
|
||||
params={"project_id": project_id}
|
||||
)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
return response.get("list", [])
|
||||
|
||||
|
||||
async def api_create_attachment(client, data: Dict[str, Any], push_id: str, biz_id: str) -> Dict[str, Any]:
|
||||
request_data = {"push_id": push_id, "biz_id": biz_id, **data}
|
||||
response = await client.request(method="POST", endpoint="/attachment/create", data=request_data)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
return response
|
||||
|
||||
|
||||
async def api_delete_attachment(client, attachment_id: str, push_id: str, biz_id: str) -> None:
|
||||
request_data = {"push_id": push_id, "biz_id": biz_id, "id": attachment_id}
|
||||
response = await client.request(method="DELETE", endpoint="/attachment/delete", data=request_data)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
@@ -0,0 +1,12 @@
|
||||
"""
|
||||
Attachment JSONL Handler
|
||||
"""
|
||||
from typing import Dict, Any
|
||||
from ...datasource import BaseJsonlHandler, JsonlDataSource
|
||||
from .sync_node import AttachmentSyncNode
|
||||
|
||||
|
||||
class AttachmentJsonlHandler(BaseJsonlHandler):
|
||||
"""附件 JSONL Handler"""
|
||||
def __init__(self, datasource: JsonlDataSource):
|
||||
super().__init__(datasource, "attachment", AttachmentSyncNode, Dict[str, Any])
|
||||
@@ -0,0 +1,16 @@
|
||||
"""Attachment domain registration"""
|
||||
from ...common.registry import DomainRegistry
|
||||
from .sync_node import AttachmentSyncNode, AttachmentSchema
|
||||
from .sync_strategy import AttachmentSyncStrategy
|
||||
from .jsonl_handler import AttachmentJsonlHandler
|
||||
from .api_handler import AttachmentApiHandler
|
||||
|
||||
# 注册 attachment domain
|
||||
DomainRegistry.register(
|
||||
node_type="attachment",
|
||||
schema=AttachmentSchema,
|
||||
node_class=AttachmentSyncNode,
|
||||
strategy_class=AttachmentSyncStrategy,
|
||||
jsonl_handler_class=AttachmentJsonlHandler,
|
||||
api_handler_class=AttachmentApiHandler,
|
||||
)
|
||||
@@ -0,0 +1,17 @@
|
||||
"""
|
||||
Attachment Domain
|
||||
"""
|
||||
from typing import Dict, Any
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
from ...common import SyncNode
|
||||
|
||||
|
||||
class AttachmentSchema(BaseModel):
|
||||
"""允许任意附件字段的 schema"""
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
|
||||
class AttachmentSyncNode(SyncNode[Dict[str, Any]]):
|
||||
"""附件同步节点"""
|
||||
node_type = "attachment"
|
||||
schema = AttachmentSchema
|
||||
@@ -0,0 +1,20 @@
|
||||
from ...sync_system.strategy import DefaultSyncStrategy
|
||||
from ...sync_system.config import StrategyConfig, OrphanAction, UpdateDirection
|
||||
from .sync_node import AttachmentSchema
|
||||
|
||||
|
||||
class AttachmentSyncStrategy(DefaultSyncStrategy[AttachmentSchema]):
|
||||
"""
|
||||
Attachment 同步策略（仅作为依赖，不执行同步动作）。
|
||||
"""
|
||||
|
||||
schema = AttachmentSchema
|
||||
|
||||
default_config = StrategyConfig(
|
||||
auto_bind=False,
|
||||
auto_bind_fields=[],
|
||||
depend_fields={},
|
||||
local_orphan_action=OrphanAction.NONE,
|
||||
remote_orphan_action=OrphanAction.NONE,
|
||||
update_direction=UpdateDirection.PUSH,
|
||||
)
|
||||
@@ -0,0 +1,6 @@
|
||||
"""Company domain"""
|
||||
from .sync_node import CompanySyncNode
|
||||
from .sync_strategy import CompanySyncStrategy
|
||||
from .jsonl_handler import CompanyJsonlHandler
|
||||
|
||||
__all__ = ["CompanySyncNode", "CompanySyncStrategy", "CompanyJsonlHandler"]
|
||||
@@ -0,0 +1,131 @@
|
||||
"""
|
||||
Company API Handler (只读)
|
||||
|
||||
职责：
|
||||
- 加载公司列表
|
||||
- 不支持创建/更新/删除
|
||||
"""
|
||||
|
||||
from typing import List, Dict, Any
|
||||
from ...datasource.api.handler import BaseApiHandler
|
||||
from ...datasource.task_result import TaskResult
|
||||
from ...common.sync_node import SyncNode
|
||||
from .sync_node import CompanySyncNode
|
||||
from schemas.company.company import CompanyResponse
|
||||
|
||||
|
||||
class CompanyApiHandler(BaseApiHandler):
|
||||
"""公司 API Handler (只读)"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._node_type = "company"
|
||||
self._node_class = CompanySyncNode
|
||||
self._schema = CompanyResponse
|
||||
|
||||
async def load(self) -> List[SyncNode]:
|
||||
"""加载公司列表（支持分页）"""
|
||||
try:
|
||||
nodes = []
|
||||
page = 1
|
||||
page_size = 1000
|
||||
|
||||
while True:
|
||||
# 获取一页数据
|
||||
companies, total = await api_get_company_list(
|
||||
self.api_client,
|
||||
page=page,
|
||||
page_size=page_size
|
||||
)
|
||||
|
||||
# 没有数据了，退出循环
|
||||
if not companies:
|
||||
break
|
||||
|
||||
# 转换为节点
|
||||
for company_model in companies:
|
||||
company_data = company_model.model_dump()
|
||||
nodes.append(self._create_node(company_data))
|
||||
|
||||
# 检查是否还有下一页
|
||||
if page * page_size >= total:
|
||||
break
|
||||
|
||||
page += 1
|
||||
|
||||
return nodes
|
||||
except Exception as e:
|
||||
print(f"❌ Failed to load companies: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return []
|
||||
|
||||
async def create_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""不支持创建"""
|
||||
return [
|
||||
TaskResult.failed(node_id=node.node_id, error="Company creation not supported")
|
||||
for node in nodes
|
||||
]
|
||||
|
||||
async def update_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""不支持更新"""
|
||||
return [
|
||||
TaskResult.failed(node_id=node.node_id, error="Company update not supported")
|
||||
for node in nodes
|
||||
]
|
||||
|
||||
async def delete_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""不支持删除"""
|
||||
return [
|
||||
TaskResult.failed(node_id=node.node_id, error="Company deletion not supported")
|
||||
for node in nodes
|
||||
]
|
||||
|
||||
async def poll_tasks(self, task_ids: List[str]) -> Dict[str, TaskResult]:
|
||||
"""轮询异步任务状态（占位）"""
|
||||
return await super().poll_tasks(task_ids)
|
||||
|
||||
def _create_node(self, data: Dict[str, Any]) -> SyncNode:
|
||||
return self._create_basic_node(data, depend_ids=[])
|
||||
|
||||
|
||||
# ========== 静态 API 函数 ==========
|
||||
|
||||
async def api_get_company_list(client, page: int = 1, page_size: int = 100) -> tuple[List[CompanyResponse], int]:
|
||||
"""
|
||||
获取公司列表 GET /company/list
|
||||
|
||||
Args:
|
||||
client: API客户端
|
||||
page: 页码，从1开始
|
||||
page_size: 每页大小
|
||||
|
||||
Returns:
|
||||
tuple[List[CompanyResponse], int]: (验证过的公司列表, 总数)
|
||||
"""
|
||||
from pydantic import ValidationError
|
||||
|
||||
response = await client.request(
|
||||
method="GET",
|
||||
endpoint="/company/list",
|
||||
params={"page": page, "page_size": page_size}
|
||||
)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
|
||||
# V2 API 返回格式: {code: 200, data: {list: [...], total: 123, ...}}
|
||||
data = response.get("data", {})
|
||||
items = data.get("list", [])
|
||||
total = data.get("total", 0)
|
||||
|
||||
# 验证每个返回项
|
||||
validated_items = []
|
||||
for item in items:
|
||||
try:
|
||||
validated_item = CompanyResponse.model_validate(item)
|
||||
validated_items.append(validated_item)
|
||||
except ValidationError as e:
|
||||
print(f"⚠️ Skipping invalid company item: {e}")
|
||||
continue
|
||||
|
||||
return validated_items, total
|
||||
@@ -0,0 +1,11 @@
|
||||
"""Company Domain - JSONL Handler"""
|
||||
from ...datasource import BaseJsonlHandler, JsonlDataSource
|
||||
from .sync_node import CompanySyncNode
|
||||
from schemas.company.company import CompanyResponse
|
||||
|
||||
|
||||
class CompanyJsonlHandler(BaseJsonlHandler):
|
||||
"""公司 JSONL Handler"""
|
||||
def __init__(self, datasource: JsonlDataSource):
|
||||
super().__init__(datasource, "company", CompanySyncNode, CompanyResponse)
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Company domain registration"""
|
||||
from ...common.registry import DomainRegistry
|
||||
from schemas.company.company import CompanyResponse
|
||||
from .sync_node import CompanySyncNode
|
||||
from .sync_strategy import CompanySyncStrategy
|
||||
from .jsonl_handler import CompanyJsonlHandler
|
||||
from .api_handler import CompanyApiHandler
|
||||
|
||||
# 注册 company domain
|
||||
DomainRegistry.register(
|
||||
node_type="company",
|
||||
schema=CompanyResponse,
|
||||
node_class=CompanySyncNode,
|
||||
strategy_class=CompanySyncStrategy,
|
||||
jsonl_handler_class=CompanyJsonlHandler,
|
||||
api_handler_class=CompanyApiHandler,
|
||||
)
|
||||
@@ -0,0 +1,11 @@
|
||||
from ...common.sync_node import SyncNode
|
||||
from schemas.company.company import CompanyResponse
|
||||
|
||||
|
||||
class CompanySyncNode(SyncNode[CompanyResponse]):
|
||||
"""公司同步节点"""
|
||||
node_type = "company"
|
||||
schema = CompanyResponse
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
@@ -0,0 +1,29 @@
|
||||
from ...sync_system.strategy import DefaultSyncStrategy
|
||||
from ...sync_system.config import StrategyConfig, OrphanAction, UpdateDirection
|
||||
from schemas.company.company import CompanyResponse
|
||||
|
||||
|
||||
class CompanySyncStrategy(DefaultSyncStrategy[CompanyResponse]):
|
||||
"""
|
||||
Company 同步策略
|
||||
|
||||
业务规则：
|
||||
1. 使用 (code) 作为业务唯一键
|
||||
2. 支持自动绑定
|
||||
3. 默认配置为日常推送模式（本地创建推送到远程，不拉取远程孤儿）
|
||||
|
||||
schema 已在类定义中设置，禁止在初始化时传入其他 schema。
|
||||
"""
|
||||
|
||||
# 类变量：schema 固定为 ContractResponse
|
||||
schema = CompanyResponse
|
||||
|
||||
# 类变量：默认配置
|
||||
default_config = StrategyConfig(
|
||||
auto_bind=True,
|
||||
auto_bind_fields=["credit_code"],
|
||||
depend_fields={},
|
||||
local_orphan_action=OrphanAction.NONE,
|
||||
remote_orphan_action=OrphanAction.CREATE_LOCAL,
|
||||
update_direction=UpdateDirection.PULL,
|
||||
)
|
||||
@@ -0,0 +1,6 @@
|
||||
"""Construction domain"""
|
||||
from .sync_node import ConstructionTaskSyncNode
|
||||
from .sync_strategy import ConstructionTaskSyncStrategy
|
||||
from .jsonl_handler import ConstructionTaskJsonlHandler
|
||||
|
||||
__all__ = ["ConstructionTaskSyncNode", "ConstructionTaskSyncStrategy", "ConstructionTaskJsonlHandler"]
|
||||
@@ -0,0 +1,321 @@
|
||||
"""
|
||||
Construction Task API Handler (V2)
|
||||
|
||||
职责：
|
||||
- 加载施工任务列表（依赖 Project）
|
||||
- 施工任务创建/更新/删除（批量接口）
|
||||
"""
|
||||
|
||||
from typing import List, Dict, Any
|
||||
from ...datasource.api.handler import BaseApiHandler
|
||||
from ...datasource.task_result import TaskResult
|
||||
from ...common.sync_node import SyncNode
|
||||
from ...datasource.api.errors import (
|
||||
ERROR_NODE_DATA_NONE,
|
||||
ERROR_VALIDATION_FAILED,
|
||||
ERROR_PROJECT_ID_NOT_FOUND
|
||||
)
|
||||
from .sync_node import ConstructionTaskSyncNode
|
||||
from schemas.construction.construction import (
|
||||
ConstructionResponse,
|
||||
ConstructionCreate,
|
||||
ConstructionUpdate,
|
||||
ConstructionPlanUpdate
|
||||
)
|
||||
|
||||
|
||||
class ConstructionTaskApiHandler(BaseApiHandler):
|
||||
"""施工任务 API Handler"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._node_type = "construction_task"
|
||||
self._node_class = ConstructionTaskSyncNode
|
||||
self._schema = ConstructionResponse
|
||||
self.update_schemas = [ConstructionCreate, ConstructionUpdate, ConstructionPlanUpdate]
|
||||
|
||||
async def load(self) -> List[SyncNode]:
|
||||
"""加载施工任务列表（依赖 Project）"""
|
||||
projects = self._collection.filter(node_type="project")
|
||||
project_ids = [p.data_id for p in projects if p.data_id]
|
||||
|
||||
if not project_ids:
|
||||
return []
|
||||
|
||||
all_tasks = []
|
||||
for project_id in project_ids:
|
||||
try:
|
||||
tasks = await api_get_construction_task_list(self.api_client, project_id)
|
||||
for task in tasks:
|
||||
# 已验证的 Pydantic model，转换为 dict
|
||||
task_data = task.model_dump()
|
||||
all_tasks.append(self._create_node(task_data))
|
||||
except Exception as e:
|
||||
print(f"❌ Failed to load construction tasks for project {project_id}: {e}")
|
||||
|
||||
return all_tasks
|
||||
|
||||
async def create_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""批量创建施工任务"""
|
||||
results = []
|
||||
for node in nodes:
|
||||
data = node.get_data()
|
||||
if not data:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=ERROR_NODE_DATA_NONE))
|
||||
continue
|
||||
|
||||
project_id = node.context.get("project_id") or data.get("project_id")
|
||||
if not project_id:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=ERROR_PROJECT_ID_NOT_FOUND))
|
||||
continue
|
||||
|
||||
push_id = self._generate_push_id()
|
||||
|
||||
try:
|
||||
# 验证并转换为 Pydantic model
|
||||
from pydantic import ValidationError
|
||||
create_data = {"project_id": project_id, **data}
|
||||
try:
|
||||
create_model = ConstructionCreate.model_validate(create_data)
|
||||
except ValidationError as e:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=f"{ERROR_VALIDATION_FAILED}: {e}"))
|
||||
continue
|
||||
|
||||
response = await api_create_construction_task(self.api_client, create_model, push_id)
|
||||
if self.poll_mode == "sync":
|
||||
created_id = self._extract_created_id_from_response(response)
|
||||
if not created_id:
|
||||
results.append(
|
||||
TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error="Missing created_id in create response"
|
||||
)
|
||||
)
|
||||
continue
|
||||
results.append(TaskResult.success(node_id=node.node_id, data_id=created_id))
|
||||
else:
|
||||
results.append(TaskResult.in_progress(node_id=node.node_id, task_id=push_id))
|
||||
except Exception as e:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=str(e)))
|
||||
|
||||
return results
|
||||
|
||||
async def update_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""
|
||||
批量更新施工任务
|
||||
|
||||
支持两种更新类型：
|
||||
1. ConstructionUpdate - 基础信息更新 (PUT /construction/update)
|
||||
业务逻辑：如果show_name字段不同，那么仅当type == 'QT'时才更新
|
||||
2. ConstructionPlanUpdate - 计划信息更新 (PUT /construction/plan)
|
||||
|
||||
更新逻辑：
|
||||
- 对每种 schema，将原始数据和新数据都转换为 schema 格式
|
||||
- dump 成字典后比较
|
||||
- 如果有差异且满足业务条件，调用对应 API
|
||||
"""
|
||||
results = []
|
||||
|
||||
for node in nodes:
|
||||
data = node.get_data()
|
||||
if not data:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error="node data is None"))
|
||||
continue
|
||||
|
||||
origin_data = node.get_origin_data() or {}
|
||||
biz_id = node.data_id or data.get("id")
|
||||
if not biz_id:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error="Missing biz_id for update"))
|
||||
continue
|
||||
|
||||
node_updated = False
|
||||
error = None
|
||||
schema_status_lines: List[str] = []
|
||||
|
||||
# 1. 基础信息更新 (ConstructionUpdate)
|
||||
# 业务逻辑：如果 show_name 字段不同，那么仅当 type == 'QT' 时才更新
|
||||
base_new_schema_error = self._validate_schema(data, ConstructionUpdate)
|
||||
base_changed_fields = self._schema_changed_fields(ConstructionUpdate, data, origin_data)
|
||||
base_update_called = False
|
||||
base_update_error: str | None = None
|
||||
if base_new_schema_error:
|
||||
base_update_error = base_new_schema_error
|
||||
elif base_changed_fields:
|
||||
should_update = True
|
||||
|
||||
if "show_name" in base_changed_fields:
|
||||
task_type = data.get("task_type") or data.get("type")
|
||||
if task_type != "QT":
|
||||
should_update = False
|
||||
|
||||
if should_update:
|
||||
try:
|
||||
update_model = ConstructionUpdate.model_validate(data)
|
||||
push_id = self._generate_push_id()
|
||||
await api_update_construction_task(self.api_client, update_model, push_id, biz_id)
|
||||
node_updated = True
|
||||
base_update_called = True
|
||||
except Exception as e:
|
||||
error = str(e)
|
||||
base_update_error = str(e)
|
||||
else:
|
||||
base_update_error = "show_name changed but task_type != QT"
|
||||
|
||||
schema_status_lines.append(
|
||||
self._build_update_schema_status(
|
||||
schema_name="ConstructionUpdate",
|
||||
schema=ConstructionUpdate,
|
||||
new_data=data,
|
||||
origin_data=origin_data,
|
||||
will_update=base_update_called,
|
||||
update_error=base_update_error,
|
||||
)
|
||||
)
|
||||
|
||||
# 2. 计划信息更新 (ConstructionPlanUpdate)
|
||||
plan_new_schema_error = self._validate_schema(data, ConstructionPlanUpdate)
|
||||
plan_changed_fields = self._schema_changed_fields(ConstructionPlanUpdate, data, origin_data)
|
||||
plan_update_called = False
|
||||
plan_update_error: str | None = None
|
||||
if not error:
|
||||
if plan_new_schema_error:
|
||||
plan_update_error = plan_new_schema_error
|
||||
elif plan_changed_fields:
|
||||
try:
|
||||
plan_model = ConstructionPlanUpdate.model_validate(data)
|
||||
plan_dict = plan_model.model_dump(exclude_unset=True)
|
||||
push_id = self._generate_push_id()
|
||||
await api_update_construction_task_plan(self.api_client, plan_dict, push_id, biz_id)
|
||||
node_updated = True
|
||||
plan_update_called = True
|
||||
except Exception as e:
|
||||
error = str(e)
|
||||
plan_update_error = str(e)
|
||||
|
||||
schema_status_lines.append(
|
||||
self._build_update_schema_status(
|
||||
schema_name="ConstructionPlanUpdate",
|
||||
schema=ConstructionPlanUpdate,
|
||||
new_data=data,
|
||||
origin_data=origin_data,
|
||||
will_update=plan_update_called,
|
||||
update_error=plan_update_error,
|
||||
)
|
||||
)
|
||||
|
||||
for line in schema_status_lines:
|
||||
node.append_log(f"UPDATE_SCHEMA: {line}")
|
||||
|
||||
# 汇总结果
|
||||
if error:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=error))
|
||||
elif node_updated:
|
||||
results.append(TaskResult.success(node_id=node.node_id))
|
||||
else:
|
||||
# 没有任何字段需要更新
|
||||
results.append(TaskResult.skipped(
|
||||
node_id=node.node_id,
|
||||
reason="No updatable fields changed | " + "; ".join(schema_status_lines)
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
async def delete_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""批量删除施工任务"""
|
||||
results = []
|
||||
for node in nodes:
|
||||
push_id = self._generate_push_id()
|
||||
biz_id = node.data_id
|
||||
if not biz_id:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error="Missing biz_id for delete"))
|
||||
continue
|
||||
|
||||
try:
|
||||
await api_delete_construction_task(self.api_client, biz_id, push_id)
|
||||
results.append(TaskResult.success(node_id=node.node_id))
|
||||
except Exception as e:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=str(e)))
|
||||
|
||||
return results
|
||||
|
||||
async def poll_tasks(self, task_ids: List[str]) -> Dict[str, TaskResult]:
|
||||
"""轮询异步任务状态"""
|
||||
return await super().poll_tasks(task_ids)
|
||||
|
||||
def _create_node(self, data: Dict[str, Any]) -> SyncNode:
|
||||
node = self._create_basic_node(data, depend_ids=[])
|
||||
|
||||
if data.get("project_id"):
|
||||
node.context["project_id"] = data["project_id"]
|
||||
node.context["task_id"] = node.data_id
|
||||
|
||||
return node
|
||||
|
||||
|
||||
# ========== 静态 API 函数 ==========
|
||||
|
||||
async def api_get_construction_task_list(client, project_id: str) -> List[ConstructionResponse]:
|
||||
response = await client.request(
|
||||
method="GET",
|
||||
endpoint="/construction/list",
|
||||
params={"project_id": project_id}
|
||||
)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
|
||||
items = response.get("data", [])
|
||||
|
||||
# 验证每个返回项
|
||||
from pydantic import ValidationError
|
||||
validated_items = []
|
||||
for item in items:
|
||||
try:
|
||||
validated_item = ConstructionResponse.model_validate(item)
|
||||
validated_items.append(validated_item)
|
||||
except ValidationError as e:
|
||||
print(f"⚠️ Skipping invalid item: {e}")
|
||||
continue
|
||||
|
||||
return validated_items
|
||||
|
||||
|
||||
async def api_create_construction_task(client, data: ConstructionCreate, push_id: str) -> Dict[str, Any]:
|
||||
request_data = {"push_id": push_id, "data": data.model_dump()}
|
||||
response = await client.request(method="POST", endpoint="/construction/create", data=request_data)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
return response
|
||||
|
||||
|
||||
async def api_update_construction_task(client, data: ConstructionUpdate, push_id: str, biz_id: str) -> None:
|
||||
request_data = {"push_id": push_id, "biz_id": biz_id, "data": data.model_dump(exclude_unset=True)}
|
||||
response = await client.request(method="PUT", endpoint="/construction/update", data=request_data)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
|
||||
|
||||
async def api_get_construction_task(client, task_id: str) -> Dict[str, Any]:
|
||||
"""GET /construction - 获取单个施工任务"""
|
||||
response = await client.request(
|
||||
method="GET",
|
||||
endpoint="/construction",
|
||||
params={"task_id": task_id}
|
||||
)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
return response.get("data", {})
|
||||
|
||||
|
||||
async def api_update_construction_task_plan(client, data: Dict[str, Any], push_id: str, biz_id: str) -> None:
|
||||
"""PUT /construction/plan - 更新施工计划"""
|
||||
request_data = {"push_id": push_id, "biz_id": biz_id, "data": data}
|
||||
response = await client.request(method="PUT", endpoint="/construction/plan", data=request_data)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
|
||||
|
||||
async def api_delete_construction_task(client, task_id: str, push_id: str) -> None:
|
||||
request_data = {"push_id": push_id, "biz_id": task_id}
|
||||
response = await client.request(method="DELETE", endpoint="/construction/delete", data=request_data)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
@@ -0,0 +1,10 @@
|
||||
"""Construction Task Domain - JSONL Handler"""
|
||||
from ...datasource import BaseJsonlHandler, JsonlDataSource
|
||||
from .sync_node import ConstructionTaskSyncNode
|
||||
from schemas.construction.construction import ConstructionResponse
|
||||
|
||||
|
||||
class ConstructionTaskJsonlHandler(BaseJsonlHandler):
|
||||
"""施工任务 JSONL Handler"""
|
||||
def __init__(self, datasource: JsonlDataSource):
|
||||
super().__init__(datasource, "construction_task", ConstructionTaskSyncNode, ConstructionResponse)
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Construction task domain registration"""
|
||||
from ...common.registry import DomainRegistry
|
||||
from schemas.construction.construction import ConstructionResponse
|
||||
from .sync_node import ConstructionTaskSyncNode
|
||||
from .sync_strategy import ConstructionTaskSyncStrategy
|
||||
from .jsonl_handler import ConstructionTaskJsonlHandler
|
||||
from .api_handler import ConstructionTaskApiHandler
|
||||
|
||||
# 注册 construction_task domain
|
||||
DomainRegistry.register(
|
||||
node_type="construction_task",
|
||||
schema=ConstructionResponse,
|
||||
node_class=ConstructionTaskSyncNode,
|
||||
strategy_class=ConstructionTaskSyncStrategy,
|
||||
jsonl_handler_class=ConstructionTaskJsonlHandler,
|
||||
api_handler_class=ConstructionTaskApiHandler,
|
||||
)
|
||||
@@ -0,0 +1,11 @@
|
||||
from ...common.sync_node import SyncNode
|
||||
from schemas.construction.construction import ConstructionResponse
|
||||
|
||||
|
||||
class ConstructionTaskSyncNode(SyncNode[ConstructionResponse]):
|
||||
"""施工任务同步节点"""
|
||||
node_type = "construction_task"
|
||||
schema = ConstructionResponse
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
@@ -0,0 +1,20 @@
|
||||
from ...sync_system.strategy import DefaultSyncStrategy
|
||||
from ...sync_system.config import StrategyConfig, OrphanAction, UpdateDirection
|
||||
from schemas.construction.construction import ConstructionResponse
|
||||
|
||||
|
||||
class ConstructionTaskSyncStrategy(DefaultSyncStrategy[ConstructionResponse]):
|
||||
"""
|
||||
Construction Task 同步策略（合同子业务）。
|
||||
"""
|
||||
|
||||
schema = ConstructionResponse
|
||||
|
||||
default_config = StrategyConfig(
|
||||
auto_bind=True,
|
||||
auto_bind_fields=["contract_id", "task_type"],
|
||||
depend_fields={"contract_id": "contract", "project_id": "project"},
|
||||
local_orphan_action=OrphanAction.CREATE_REMOTE,
|
||||
remote_orphan_action=OrphanAction.NONE,
|
||||
update_direction=UpdateDirection.PUSH,
|
||||
)
|
||||
@@ -0,0 +1,8 @@
|
||||
"""
|
||||
Construction Task Detail Domain
|
||||
"""
|
||||
from .sync_node import ConstructionTaskDetailSyncNode
|
||||
from .sync_strategy import ConstructionTaskDetailSyncStrategy
|
||||
from .jsonl_handler import ConstructionTaskDetailJsonlHandler
|
||||
|
||||
__all__ = ["ConstructionTaskDetailSyncNode", "ConstructionTaskDetailSyncStrategy", "ConstructionTaskDetailJsonlHandler"]
|
||||
@@ -0,0 +1,283 @@
|
||||
"""
|
||||
Construction Task Detail API Handler (V2)
|
||||
|
||||
职责：
|
||||
- 加载施工明细列表（依赖 ConstructionTask）
|
||||
- 施工明细创建/更新/删除（批量接口）
|
||||
"""
|
||||
|
||||
from typing import List, Dict, Any
|
||||
from ...datasource.api.handler import BaseApiHandler
|
||||
from ...datasource.task_result import TaskResult
|
||||
from ...common.sync_node import SyncNode
|
||||
from ...datasource.api.errors import ERROR_NODE_DATA_NONE, ERROR_VALIDATION_FAILED, ERROR_ID_NOT_FOUND
|
||||
from .sync_node import ConstructionTaskDetailSyncNode
|
||||
from schemas.construction.construction_detail import (
|
||||
ConstructionDetailResponse,
|
||||
ConstructionDetailCreate,
|
||||
ConstructionDetailUpdate
|
||||
)
|
||||
|
||||
|
||||
class ConstructionTaskDetailApiHandler(BaseApiHandler):
|
||||
"""施工明细 API Handler"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._node_type = "construction_task_detail"
|
||||
self._node_class = ConstructionTaskDetailSyncNode
|
||||
self._schema = ConstructionDetailResponse
|
||||
self.update_schemas = [ConstructionDetailCreate, ConstructionDetailUpdate]
|
||||
|
||||
async def load(self) -> List[SyncNode]:
|
||||
"""加载施工明细列表（依赖 ConstructionTask）"""
|
||||
tasks = self._collection.filter(node_type="construction_task")
|
||||
task_ids = [t.data_id for t in tasks if t.data_id]
|
||||
|
||||
if not task_ids:
|
||||
return []
|
||||
|
||||
all_details = []
|
||||
for task_id in task_ids:
|
||||
try:
|
||||
details = await api_get_construction_task_detail_list(self.api_client, task_id)
|
||||
for detail_model in details:
|
||||
# 已验证的 Pydantic model，转换为 dict
|
||||
detail_data = detail_model.model_dump()
|
||||
all_details.append(self._create_node(detail_data))
|
||||
except Exception as e:
|
||||
print(f"❌ Failed to load construction details for task {task_id}: {e}")
|
||||
|
||||
return all_details
|
||||
|
||||
async def create_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""批量创建施工明细"""
|
||||
from pydantic import ValidationError
|
||||
|
||||
results = []
|
||||
for node in nodes:
|
||||
data = node.get_data()
|
||||
if not data:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=ERROR_NODE_DATA_NONE))
|
||||
continue
|
||||
|
||||
# 从data的parent_id获取task_id
|
||||
task_id = data.get("parent_id")
|
||||
if not task_id:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=ERROR_ID_NOT_FOUND))
|
||||
continue
|
||||
|
||||
# 验证并转换为 Pydantic model
|
||||
try:
|
||||
create_data = {"task_id": task_id, **data}
|
||||
create_model = ConstructionDetailCreate.model_validate(create_data)
|
||||
except ValidationError as e:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=f"{ERROR_VALIDATION_FAILED}: {e}"))
|
||||
continue
|
||||
|
||||
push_id = self._generate_push_id()
|
||||
|
||||
try:
|
||||
response = await api_create_construction_task_detail(self.api_client, create_model, push_id)
|
||||
if self.poll_mode == "sync":
|
||||
created_id = self._extract_created_id_from_response(response)
|
||||
if not created_id:
|
||||
results.append(
|
||||
TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error="Missing created_id in create response"
|
||||
)
|
||||
)
|
||||
continue
|
||||
results.append(TaskResult.success(node_id=node.node_id, data_id=created_id))
|
||||
else:
|
||||
results.append(TaskResult.in_progress(node_id=node.node_id, task_id=push_id))
|
||||
except Exception as e:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=str(e)))
|
||||
|
||||
return results
|
||||
|
||||
async def update_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""
|
||||
批量更新施工明细
|
||||
|
||||
按 task_id 分组，每组使用同一个 push_id 批量提交
|
||||
"""
|
||||
from pydantic import ValidationError
|
||||
from collections import defaultdict
|
||||
|
||||
results = []
|
||||
|
||||
# 按 task_id 分组
|
||||
groups: Dict[str, List[tuple[SyncNode, Dict[str, Any]]]] = defaultdict(list)
|
||||
|
||||
for node in nodes:
|
||||
data = node.get_data()
|
||||
if not data:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=ERROR_NODE_DATA_NONE))
|
||||
continue
|
||||
|
||||
origin_data = node.get_origin_data() or {}
|
||||
|
||||
# 使用 _get_schema_diff 检查差异并打印日志
|
||||
diff_fields = self._get_schema_diff(ConstructionDetailUpdate, data, origin_data, node.node_id)
|
||||
if not diff_fields:
|
||||
from ...common.types import SyncAction
|
||||
results.append(TaskResult.skipped(node_id=node.node_id, reason="No updatable fields changed"))
|
||||
continue
|
||||
|
||||
# 获取 task_id（parent_id）
|
||||
task_id = data.get("parent_id") or node.context.get("task_id")
|
||||
if not task_id:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error="Missing task_id (parent_id)"))
|
||||
continue
|
||||
|
||||
# 需要完整的 schema 数据（包括必填字段）
|
||||
update_data = self._filter_update_data(data, origin_data, ConstructionDetailUpdate)
|
||||
|
||||
# 验证并转换为 Pydantic model
|
||||
try:
|
||||
update_model = ConstructionDetailUpdate.model_validate(update_data)
|
||||
groups[task_id].append((node, update_model))
|
||||
except ValidationError as e:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=f"{ERROR_VALIDATION_FAILED}: {e}"))
|
||||
|
||||
# 逐组批量更新
|
||||
for task_id, group_items in groups.items():
|
||||
push_id = self._generate_push_id()
|
||||
|
||||
# 准备批量数据
|
||||
detail_list = [model.model_dump(mode='json', exclude_unset=True) for node, model in group_items]
|
||||
|
||||
# 使用 parent_id（task_id）作为 biz_id
|
||||
biz_id = task_id
|
||||
|
||||
try:
|
||||
# 批量调用 API
|
||||
await api_update_construction_task_detail(
|
||||
self.api_client,
|
||||
{"detail_list": detail_list},
|
||||
push_id,
|
||||
biz_id,
|
||||
)
|
||||
# 整组成功
|
||||
for node, _ in group_items:
|
||||
if self.poll_mode == "sync":
|
||||
results.append(TaskResult.success(node_id=node.node_id))
|
||||
else:
|
||||
results.append(TaskResult.in_progress(node_id=node.node_id, task_id=push_id))
|
||||
except Exception as e:
|
||||
# 整组失败
|
||||
error_msg = str(e)
|
||||
for node, _ in group_items:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=error_msg))
|
||||
|
||||
return results
|
||||
|
||||
async def delete_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""批量删除施工明细"""
|
||||
results = []
|
||||
for node in nodes:
|
||||
push_id = self._generate_push_id()
|
||||
biz_id = node.data_id
|
||||
if not biz_id:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error="Missing biz_id for delete"))
|
||||
continue
|
||||
|
||||
try:
|
||||
await api_delete_construction_task_detail(self.api_client, biz_id, push_id)
|
||||
results.append(TaskResult.success(node_id=node.node_id))
|
||||
except Exception as e:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=str(e)))
|
||||
|
||||
return results
|
||||
|
||||
async def poll_tasks(self, task_ids: List[str]) -> Dict[str, TaskResult]:
|
||||
"""轮询异步任务状态"""
|
||||
return await super().poll_tasks(task_ids)
|
||||
|
||||
def _create_node(self, data: Dict[str, Any]) -> SyncNode:
|
||||
node = self._create_basic_node(data, depend_ids=[])
|
||||
|
||||
if data.get("task_id"):
|
||||
node.context["task_id"] = data["task_id"]
|
||||
|
||||
return node
|
||||
|
||||
|
||||
# ========== 静态 API 函数 ==========
|
||||
|
||||
async def api_get_construction_task_detail_list(client, task_id: str) -> List[ConstructionDetailResponse]:
|
||||
"""
|
||||
获取施工明细列表 GET /construction_task_detail/list
|
||||
|
||||
Args:
|
||||
client: API 客户端
|
||||
task_id: 施工任务ID
|
||||
|
||||
Returns:
|
||||
List[ConstructionDetailResponse]: 验证过的明细列表
|
||||
"""
|
||||
from pydantic import ValidationError
|
||||
|
||||
response = await client.request(
|
||||
method="GET",
|
||||
endpoint="/construction/detail/list",
|
||||
params={"task_id": task_id}
|
||||
)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
|
||||
items = response.get("data", [])
|
||||
|
||||
# 验证每个返回项
|
||||
validated_items = []
|
||||
for item in items:
|
||||
try:
|
||||
validated_item = ConstructionDetailResponse.model_validate(item)
|
||||
validated_items.append(validated_item)
|
||||
except ValidationError as e:
|
||||
print(f"⚠️ Skipping invalid detail item: {e}")
|
||||
continue
|
||||
|
||||
return validated_items
|
||||
|
||||
|
||||
async def api_create_construction_task_detail(client, data: ConstructionDetailCreate, push_id: str) -> Dict[str, Any]:
|
||||
"""
|
||||
创建施工明细 POST /construction/detail/create
|
||||
|
||||
Args:
|
||||
client: API 客户端
|
||||
data: 创建数据（Pydantic model）
|
||||
push_id: 推送ID
|
||||
biz_id: 业务ID
|
||||
"""
|
||||
request_data = {"push_id": push_id, "data": data.model_dump(mode='json')}
|
||||
response = await client.request(method="POST", endpoint="/construction/detail/create", data=request_data)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
return response
|
||||
|
||||
|
||||
async def api_update_construction_task_detail(client, data: Dict[str, Any], push_id: str, biz_id: str) -> None:
|
||||
"""
|
||||
更新施工明细 PUT /construction/detail/update
|
||||
|
||||
Args:
|
||||
client: API 客户端
|
||||
data: 更新数据（Pydantic model）
|
||||
push_id: 推送ID
|
||||
biz_id: 业务ID
|
||||
"""
|
||||
request_data = {"push_id": push_id, "biz_id": biz_id, "data": data}
|
||||
response = await client.request(method="PUT", endpoint="/construction/detail/update", data=request_data)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
|
||||
|
||||
async def api_delete_construction_task_detail(client, detail_id: str, push_id: str) -> None:
|
||||
request_data = {"push_id": push_id, "biz_id": detail_id}
|
||||
response = await client.request(method="DELETE", endpoint="/construction/detail/delete", data=request_data)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
@@ -0,0 +1,12 @@
|
||||
"""
|
||||
Construction Task Detail JSONL Handler
|
||||
"""
|
||||
from ...datasource import BaseJsonlHandler, JsonlDataSource
|
||||
from .sync_node import ConstructionTaskDetailSyncNode
|
||||
from schemas.construction.construction_detail import ConstructionDetailResponse
|
||||
|
||||
|
||||
class ConstructionTaskDetailJsonlHandler(BaseJsonlHandler):
|
||||
"""施工任务明细 JSONL Handler"""
|
||||
def __init__(self, datasource: JsonlDataSource):
|
||||
super().__init__(datasource, "construction_task_detail", ConstructionTaskDetailSyncNode, ConstructionDetailResponse)
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Construction task detail domain registration"""
|
||||
from ...common.registry import DomainRegistry
|
||||
from schemas.construction.construction_detail import ConstructionDetailResponse
|
||||
from .sync_node import ConstructionTaskDetailSyncNode
|
||||
from .sync_strategy import ConstructionTaskDetailSyncStrategy
|
||||
from .jsonl_handler import ConstructionTaskDetailJsonlHandler
|
||||
from .api_handler import ConstructionTaskDetailApiHandler
|
||||
|
||||
# 注册 construction_task_detail domain
|
||||
DomainRegistry.register(
|
||||
node_type="construction_task_detail",
|
||||
schema=ConstructionDetailResponse,
|
||||
node_class=ConstructionTaskDetailSyncNode,
|
||||
strategy_class=ConstructionTaskDetailSyncStrategy,
|
||||
jsonl_handler_class=ConstructionTaskDetailJsonlHandler,
|
||||
api_handler_class=ConstructionTaskDetailApiHandler,
|
||||
)
|
||||
@@ -0,0 +1,11 @@
|
||||
"""
|
||||
Construction Task Detail Domain
|
||||
"""
|
||||
from ...common import SyncNode
|
||||
from schemas.construction.construction_detail import ConstructionDetailResponse
|
||||
|
||||
|
||||
class ConstructionTaskDetailSyncNode(SyncNode[ConstructionDetailResponse]):
|
||||
"""施工任务明细同步节点"""
|
||||
node_type = "construction_task_detail"
|
||||
schema = ConstructionDetailResponse
|
||||
@@ -0,0 +1,20 @@
|
||||
from ...sync_system.strategy import DefaultSyncStrategy
|
||||
from ...sync_system.config import StrategyConfig, OrphanAction, UpdateDirection
|
||||
from schemas.construction.construction_detail import ConstructionDetailResponse
|
||||
|
||||
|
||||
class ConstructionTaskDetailSyncStrategy(DefaultSyncStrategy[ConstructionDetailResponse]):
|
||||
"""
|
||||
Construction Task Detail 同步策略（合同子业务明细）。
|
||||
"""
|
||||
|
||||
schema = ConstructionDetailResponse
|
||||
|
||||
default_config = StrategyConfig(
|
||||
auto_bind=True,
|
||||
auto_bind_fields=["parent_id", "date"],
|
||||
depend_fields={"parent_id": "construction_task"},
|
||||
local_orphan_action=OrphanAction.CREATE_REMOTE,
|
||||
remote_orphan_action=OrphanAction.NONE,
|
||||
update_direction=UpdateDirection.PUSH,
|
||||
)
|
||||
@@ -0,0 +1,7 @@
|
||||
"""Contract domain"""
|
||||
from .sync_node import ContractSyncNode
|
||||
from .sync_strategy import ContractSyncStrategy
|
||||
from .jsonl_handler import ContractJsonlHandler
|
||||
|
||||
__all__ = ["ContractSyncNode", "ContractSyncStrategy", "ContractJsonlHandler"]
|
||||
|
||||
@@ -0,0 +1,329 @@
|
||||
"""
|
||||
Contract API Handler
|
||||
|
||||
职责：
|
||||
- 加载合同列表（依赖 Project）
|
||||
- 合同创建（POST /contract/create，异步）
|
||||
- 合同更新（PUT /contract/update，同步）
|
||||
- 合同删除（DELETE /contract/delete，同步）
|
||||
"""
|
||||
|
||||
from typing import List, Dict, Any, Optional
|
||||
|
||||
from ...datasource.api.handler import BaseApiHandler
|
||||
from ...datasource.task_result import TaskResult
|
||||
from ...common.sync_node import SyncNode
|
||||
from ...datasource.api.client import ApiClient
|
||||
from .sync_node import ContractSyncNode
|
||||
from schemas.contract.contract import ContractResponse, ContractCreate, ContractUpdate
|
||||
|
||||
|
||||
# ========== 静态 API 函数 ==========
|
||||
|
||||
async def api_get_contract_list(client: ApiClient, project_id: str) -> List[ContractResponse]:
|
||||
"""GET /contract/list - 获取项目合同列表"""
|
||||
response = await client.request(
|
||||
method="GET",
|
||||
endpoint="/contract/list",
|
||||
params={"project_id": project_id}
|
||||
)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"Failed to get contract list: {response.get('message', 'Unknown error')}")
|
||||
|
||||
# V2 API 返回格式：{code: 200, data: [...]}
|
||||
items = response.get("data", [])
|
||||
|
||||
return [ContractResponse.model_validate(item) for item in items]
|
||||
|
||||
|
||||
async def api_get_contract(client: ApiClient, contract_id: str) -> ContractResponse:
|
||||
"""GET /contract - 获取合同详情"""
|
||||
response = await client.request(
|
||||
method="GET",
|
||||
endpoint="/contract",
|
||||
params={"contract_id": contract_id}
|
||||
)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"Failed to get contract: {response.get('message', 'Unknown error')}")
|
||||
return ContractResponse.model_validate(response["data"])
|
||||
|
||||
|
||||
async def api_create_contract(client: ApiClient, data: Dict[str, Any], push_id: str) -> Dict[str, Any]:
|
||||
"""POST /contract/create - 创建合同（异步）"""
|
||||
validated_data = ContractCreate.model_validate(data)
|
||||
request_data = {"push_id": push_id, "data": validated_data.model_dump(exclude_none=True)}
|
||||
response = await client.request("POST", "/contract/create", data=request_data)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"Failed to create contract: {response.get('message', 'Unknown error')}")
|
||||
return response
|
||||
|
||||
|
||||
async def api_update_contract(client: ApiClient, data_id: str, data: Dict[str, Any], push_id: str) -> Dict[str, Any]:
|
||||
"""PUT /contract/update - 更新合同（同步）"""
|
||||
validated_data = ContractUpdate.model_validate(data)
|
||||
request_data = {"push_id": push_id, "biz_id": data_id, "data": validated_data.model_dump(exclude_none=True)}
|
||||
response = await client.request("PUT", "/contract/update", data=request_data)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"Failed to update contract: {response.get('message', 'Unknown error')}")
|
||||
return response
|
||||
|
||||
|
||||
async def api_delete_contract(client: ApiClient, data_id: str, push_id: str) -> Dict[str, Any]:
|
||||
"""DELETE /contract/delete - 删除合同（同步）"""
|
||||
request_data = {"push_id": push_id, "biz_id": data_id}
|
||||
response = await client.request("DELETE", "/contract/delete", data=request_data)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"Failed to delete contract: {response.get('message', 'Unknown error')}")
|
||||
return response
|
||||
|
||||
|
||||
# ========== Handler ==========
|
||||
|
||||
class ContractApiHandler(BaseApiHandler):
|
||||
"""合同 API Handler"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._node_type = "contract"
|
||||
self._node_class = ContractSyncNode
|
||||
self._schema = ContractResponse
|
||||
self.update_schemas = [ContractCreate, ContractUpdate]
|
||||
|
||||
# ========== Handler 接口实现 ==========
|
||||
|
||||
async def load(self) -> List[SyncNode]:
|
||||
"""
|
||||
加载合同列表
|
||||
|
||||
依赖 Project：从 collection 获取项目列表，然后加载这些项目的合同
|
||||
|
||||
Returns:
|
||||
合同数据列表
|
||||
"""
|
||||
try:
|
||||
# 从 collection 获取项目列表
|
||||
projects = self._collection.filter(node_type="project")
|
||||
project_ids = [p.data_id for p in projects if p.data_id]
|
||||
|
||||
if not project_ids:
|
||||
print("⚠️ No projects found, skipping contract load")
|
||||
return []
|
||||
|
||||
# 为每个项目加载合同
|
||||
all_nodes = []
|
||||
for project_id in project_ids:
|
||||
try:
|
||||
contracts = await api_get_contract_list(self.api_client, project_id)
|
||||
|
||||
# 已验证的 Pydantic model，转换为 dict 并创建 node
|
||||
for contract in contracts:
|
||||
contract_data = contract.model_dump()
|
||||
all_nodes.append(self._create_node(contract_data))
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Failed to load contracts for project {project_id}: {e}")
|
||||
continue
|
||||
|
||||
return all_nodes
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Failed to load contracts: {e}")
|
||||
return []
|
||||
|
||||
async def create_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""批量创建合同"""
|
||||
results = []
|
||||
for node in nodes:
|
||||
result = await self._create_single(node)
|
||||
results.append(result)
|
||||
return results
|
||||
|
||||
async def update_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""批量更新合同"""
|
||||
results = []
|
||||
for node in nodes:
|
||||
result = await self._update_single(node)
|
||||
results.append(result)
|
||||
return results
|
||||
|
||||
async def delete_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""批量删除合同"""
|
||||
results = []
|
||||
for node in nodes:
|
||||
result = await self._delete_single(node)
|
||||
results.append(result)
|
||||
return results
|
||||
|
||||
async def poll_tasks(self, task_ids: List[str]) -> Dict[str, TaskResult]:
|
||||
"""轮询异步任务状态"""
|
||||
return await super().poll_tasks(task_ids)
|
||||
|
||||
async def _create_single(self, node: SyncNode) -> TaskResult:
|
||||
"""
|
||||
创建合同（异步）
|
||||
|
||||
Args:
|
||||
node: 同步节点
|
||||
|
||||
Returns:
|
||||
TaskResult (IN_PROGRESS with push_id)
|
||||
"""
|
||||
push_id: str | None = None
|
||||
try:
|
||||
# 获取数据
|
||||
data = node.get_data()
|
||||
if not data:
|
||||
return TaskResult.failed(node_id=node.node_id, error="Node data is None or validation failed")
|
||||
|
||||
# 从 context 获取 project_id
|
||||
project_id = node.context.get("project_id")
|
||||
if not project_id:
|
||||
project_id = data.get("project_id")
|
||||
|
||||
if not project_id:
|
||||
return TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error="project_id not found in context or data"
|
||||
)
|
||||
|
||||
# 构建请求数据（确保包含 project_id）
|
||||
request_data = {
|
||||
"project_id": project_id,
|
||||
**data
|
||||
}
|
||||
|
||||
# 调用 API（直接调用顶层函数）
|
||||
push_id = self._generate_push_id()
|
||||
response = await api_create_contract(self.api_client, request_data, push_id)
|
||||
if self.poll_mode == "sync":
|
||||
created_id = self._extract_created_id_from_response(response)
|
||||
if not created_id:
|
||||
return TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error="Missing created_id in create response"
|
||||
)
|
||||
return TaskResult.success(node_id=node.node_id, data_id=created_id)
|
||||
else:
|
||||
return TaskResult.in_progress(node_id=node.node_id, task_id=push_id)
|
||||
|
||||
except Exception as e:
|
||||
return TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error=f"Create failed: {str(e)}",
|
||||
push_id=push_id,
|
||||
)
|
||||
|
||||
async def _update_single(self, node: SyncNode) -> TaskResult:
|
||||
"""
|
||||
更新合同（同步）
|
||||
|
||||
自动选择需要更新的字段（基于 update_schema）
|
||||
|
||||
Args:
|
||||
node: 同步节点
|
||||
|
||||
Returns:
|
||||
TaskResult (SUCCESS or FAILED)
|
||||
"""
|
||||
try:
|
||||
data = node.get_data()
|
||||
if not data:
|
||||
return TaskResult.failed(node_id=node.node_id, error="Missing data for update")
|
||||
|
||||
# 过滤出需要更新的字段（排除空值、未变化的字段）
|
||||
origin_data = node.get_origin_data() or {}
|
||||
update_data = self._filter_update_data(
|
||||
new_data=data,
|
||||
original_data=origin_data,
|
||||
schema=ContractUpdate,
|
||||
)
|
||||
|
||||
# 如果没有需要更新的字段，降级 action 并返回跳过
|
||||
if not update_data:
|
||||
return TaskResult.skipped(node_id=node.node_id, reason="No updatable fields changed")
|
||||
|
||||
# Schema 验证
|
||||
error = self._validate_schema(update_data, ContractUpdate)
|
||||
if error:
|
||||
return TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error=f"Schema validation failed: {error}"
|
||||
)
|
||||
|
||||
# 生成 push_id 和 biz_id
|
||||
push_id = self._generate_push_id()
|
||||
biz_id = node.data_id or update_data.get("id")
|
||||
if not biz_id:
|
||||
return TaskResult.failed(node_id=node.node_id, error="Missing biz_id for update")
|
||||
|
||||
# 调用 API
|
||||
response = await api_update_contract(self.api_client, biz_id, update_data, push_id)
|
||||
|
||||
# 检查响应状态（更新接口是同步的）
|
||||
if response.get("code") == 200:
|
||||
return TaskResult.success(node_id=node.node_id)
|
||||
else:
|
||||
return TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error=f"API returned error: {response.get('message', 'Unknown error')}"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
return TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error=f"Update failed: {str(e)}"
|
||||
)
|
||||
|
||||
async def _delete_single(self, node: SyncNode) -> TaskResult:
|
||||
"""
|
||||
删除合同（同步）
|
||||
|
||||
Args:
|
||||
node: 同步节点
|
||||
|
||||
Returns:
|
||||
TaskResult (SUCCESS or FAILED)
|
||||
"""
|
||||
if not self.api_client:
|
||||
return TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error="api_client not initialized"
|
||||
)
|
||||
|
||||
try:
|
||||
# 生成 push_id 和 biz_id
|
||||
push_id = self._generate_push_id()
|
||||
biz_id = node.data_id
|
||||
if not biz_id:
|
||||
return TaskResult.failed(node_id=node.node_id, error="Missing biz_id for delete")
|
||||
|
||||
# 调用 API
|
||||
response = await api_delete_contract(self.api_client, biz_id, push_id)
|
||||
|
||||
# 检查响应状态
|
||||
if response.get("code") == 200:
|
||||
return TaskResult.success(node_id=node.node_id)
|
||||
else:
|
||||
return TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error=f"API returned error: {response.get('message', 'Unknown error')}"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
return TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error=f"Delete failed: {str(e)}"
|
||||
)
|
||||
|
||||
# ========== 辅助方法 ==========
|
||||
|
||||
def _create_node(self, data: Dict[str, Any]) -> SyncNode:
|
||||
"""创建合同同步节点（设置 project_id 到 context）"""
|
||||
node = self._create_basic_node(data, depend_ids=[])
|
||||
|
||||
# 设置 project_id 到 context（供子节点使用）
|
||||
if data.get("project_id"):
|
||||
node.context["project_id"] = data["project_id"]
|
||||
|
||||
return node
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
"""
|
||||
Contract JSONL Handler
|
||||
"""
|
||||
from ...datasource import BaseJsonlHandler, JsonlDataSource
|
||||
from .sync_node import ContractSyncNode
|
||||
from schemas.contract.contract import ContractResponse
|
||||
|
||||
|
||||
class ContractJsonlHandler(BaseJsonlHandler):
|
||||
"""合同 JSONL Handler"""
|
||||
def __init__(self, datasource: JsonlDataSource):
|
||||
super().__init__(datasource, "contract", ContractSyncNode, ContractResponse)
|
||||
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Contract domain registration"""
|
||||
from ...common.registry import DomainRegistry
|
||||
from schemas.contract.contract import ContractResponse
|
||||
from .sync_node import ContractSyncNode
|
||||
from .sync_strategy import ContractSyncStrategy
|
||||
from .jsonl_handler import ContractJsonlHandler
|
||||
from .api_handler import ContractApiHandler
|
||||
|
||||
# 注册 contract domain
|
||||
DomainRegistry.register(
|
||||
node_type="contract",
|
||||
schema=ContractResponse,
|
||||
node_class=ContractSyncNode,
|
||||
strategy_class=ContractSyncStrategy,
|
||||
jsonl_handler_class=ContractJsonlHandler,
|
||||
api_handler_class=ContractApiHandler,
|
||||
)
|
||||
@@ -0,0 +1,11 @@
|
||||
from ...common.sync_node import SyncNode
|
||||
from schemas.contract.contract import ContractResponse
|
||||
|
||||
|
||||
class ContractSyncNode(SyncNode[ContractResponse]):
|
||||
"""合同同步节点"""
|
||||
node_type = "contract"
|
||||
schema = ContractResponse
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
@@ -0,0 +1,30 @@
|
||||
from ...sync_system.strategy import DefaultSyncStrategy
|
||||
from ...sync_system.config import StrategyConfig, OrphanAction, UpdateDirection
|
||||
from schemas.contract.contract import ContractResponse
|
||||
|
||||
|
||||
class ContractSyncStrategy(DefaultSyncStrategy[ContractResponse]):
|
||||
"""
|
||||
Contract 同步策略
|
||||
|
||||
业务规则：
|
||||
1. 使用 (project_id, code) 作为业务唯一键
|
||||
2. 依赖于 project 和 company
|
||||
3. 支持自动绑定
|
||||
4. 默认配置为日常推送模式（本地创建推送到远程，不拉取远程孤儿）
|
||||
|
||||
schema 已在类定义中设置，禁止在初始化时传入其他 schema。
|
||||
"""
|
||||
|
||||
# 类变量：schema 固定为 ContractResponse
|
||||
schema = ContractResponse
|
||||
|
||||
# 类变量：默认配置
|
||||
default_config = StrategyConfig(
|
||||
auto_bind=True,
|
||||
auto_bind_fields=["project_id","company_id", "code"],
|
||||
depend_fields={"project_id": "project", "company_id": "supplier"},
|
||||
local_orphan_action=OrphanAction.CREATE_REMOTE,
|
||||
remote_orphan_action=OrphanAction.NONE,
|
||||
update_direction=UpdateDirection.PUSH,
|
||||
)
|
||||
@@ -0,0 +1,13 @@
|
||||
"""Contract Change (合同变更) Domain Module"""
|
||||
|
||||
from .sync_node import ContractChangeSyncNode
|
||||
from .sync_strategy import ContractChangeSyncStrategy
|
||||
from .api_handler import ContractChangeApiHandler
|
||||
from .jsonl_handler import ContractChangeJsonlHandler
|
||||
|
||||
__all__ = [
|
||||
"ContractChangeSyncNode",
|
||||
"ContractChangeSyncStrategy",
|
||||
"ContractChangeApiHandler",
|
||||
"ContractChangeJsonlHandler",
|
||||
]
|
||||
@@ -0,0 +1,171 @@
|
||||
"""
|
||||
Contract Change (合同变更) API Handler
|
||||
|
||||
职责：
|
||||
- 创建和更新合同变更记录（依赖 Project 和 Contract）
|
||||
|
||||
实现的接口：
|
||||
POST:
|
||||
- /contract_change/create - 创建合同变更
|
||||
PUT:
|
||||
- /contract_change/update - 更新合同变更
|
||||
"""
|
||||
|
||||
from typing import List, Dict, Any
|
||||
from ...datasource.api.handler import BaseApiHandler
|
||||
from ...datasource.task_result import TaskResult
|
||||
from ...common.sync_node import SyncNode
|
||||
from ...datasource.api.errors import ERROR_NODE_DATA_NONE, ERROR_VALIDATION_FAILED
|
||||
from .sync_node import ContractChangeSyncNode
|
||||
from schemas.project_extensions.contract_change import ContractChangeResponse, PostContractChange, CreateContractChange
|
||||
|
||||
|
||||
class ContractChangeApiHandler(BaseApiHandler):
|
||||
"""合同变更 API Handler"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._node_type = "contract_change"
|
||||
self._node_class = ContractChangeSyncNode
|
||||
# TODO: ContractChangeResponse schema in schemas/project_extensions/contract_change.py is missing 'project_id'
|
||||
self._schema = ContractChangeResponse
|
||||
|
||||
async def load(self) -> List[SyncNode]:
|
||||
"""加载合同变更数据"""
|
||||
return []
|
||||
|
||||
async def create_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""批量创建合同变更"""
|
||||
from pydantic import ValidationError
|
||||
|
||||
results = []
|
||||
for node in nodes:
|
||||
node_data = node.get_data()
|
||||
if node_data is None:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=ERROR_NODE_DATA_NONE))
|
||||
continue
|
||||
|
||||
# 验证并转换为 Pydantic model
|
||||
try:
|
||||
create_model = CreateContractChange.model_validate(node_data)
|
||||
except ValidationError as e:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=f"{ERROR_VALIDATION_FAILED}: {e}"))
|
||||
continue
|
||||
|
||||
push_id = self._generate_push_id()
|
||||
|
||||
try:
|
||||
response = await api_create_contract_change(self.api_client, create_model, push_id)
|
||||
if self.poll_mode == "sync":
|
||||
created_id = self._extract_created_id_from_response(response)
|
||||
if not created_id:
|
||||
results.append(
|
||||
TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error="Missing created_id in create response"
|
||||
)
|
||||
)
|
||||
continue
|
||||
results.append(TaskResult.success(node_id=node.node_id, data_id=created_id))
|
||||
else:
|
||||
results.append(TaskResult.in_progress(node_id=node.node_id, task_id=push_id))
|
||||
except Exception as e:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=str(e)))
|
||||
|
||||
return results
|
||||
|
||||
async def update_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""批量更新合同变更"""
|
||||
from pydantic import ValidationError
|
||||
|
||||
results = []
|
||||
for node in nodes:
|
||||
node_data = node.get_data()
|
||||
if node_data is None:
|
||||
error_msg = node.error or "Node data is missing or invalid"
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=f"Cannot update: {error_msg}"))
|
||||
continue
|
||||
|
||||
origin_data = node.get_origin_data() or {}
|
||||
update_data = self._filter_update_data(node_data, origin_data, PostContractChange)
|
||||
if not update_data:
|
||||
from ...common.types import SyncAction
|
||||
results.append(TaskResult.skipped(node_id=node.node_id, reason="No updatable fields changed"))
|
||||
continue
|
||||
|
||||
# 验证并转换为 Pydantic model
|
||||
try:
|
||||
update_model = PostContractChange.model_validate(update_data)
|
||||
except ValidationError as e:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=f"{ERROR_VALIDATION_FAILED}: {e}"))
|
||||
continue
|
||||
|
||||
push_id = self._generate_push_id()
|
||||
biz_id = node.data_id or update_model.id
|
||||
if not biz_id:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error="Missing biz_id for update"))
|
||||
continue
|
||||
|
||||
try:
|
||||
await api_update_contract_change(self.api_client, update_model, push_id, biz_id)
|
||||
results.append(TaskResult.success(node_id=node.node_id))
|
||||
except Exception as e:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=str(e)))
|
||||
|
||||
return results
|
||||
|
||||
async def delete_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""不支持删除"""
|
||||
return [
|
||||
TaskResult.failed(node_id=node.node_id, error="Contract change deletion not supported")
|
||||
for node in nodes
|
||||
]
|
||||
|
||||
async def poll_tasks(self, task_ids: List[str]) -> Dict[str, TaskResult]:
|
||||
"""轮询异步任务状态"""
|
||||
return await super().poll_tasks(task_ids)
|
||||
|
||||
def _create_node(self, data: Dict[str, Any]) -> SyncNode:
|
||||
node = self._create_basic_node(data, depend_ids=[])
|
||||
|
||||
if data.get("project_id"):
|
||||
node.context["project_id"] = data["project_id"]
|
||||
if data.get("contract_id"):
|
||||
node.context["contract_id"] = data["contract_id"]
|
||||
|
||||
return node
|
||||
|
||||
|
||||
# ========== 静态 API 函数 ==========
|
||||
|
||||
async def api_create_contract_change(client, data: CreateContractChange, push_id: str) -> Dict[str, Any]:
|
||||
"""
|
||||
POST /contract_change/create - 创建合同变更
|
||||
|
||||
Args:
|
||||
client: API 客户端
|
||||
data: 创建数据（Pydantic model）
|
||||
push_id: 推送ID
|
||||
biz_id: 业务ID
|
||||
"""
|
||||
request_data = {"push_id": push_id, "data": data.model_dump()}
|
||||
response = await client.request(method="POST", endpoint="/contract_change/create", data=request_data)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
return response
|
||||
|
||||
|
||||
async def api_update_contract_change(client, data: PostContractChange, push_id: str, biz_id: str) -> None:
|
||||
"""
|
||||
PUT /contract_change/update - 更新合同变更
|
||||
|
||||
Args:
|
||||
client: API 客户端
|
||||
data: 更新数据（Pydantic model）
|
||||
push_id: 推送ID
|
||||
biz_id: 业务ID
|
||||
"""
|
||||
request_data = {"push_id": push_id, "biz_id": biz_id, "data": data.model_dump(exclude_unset=True)}
|
||||
response = await client.request(method="PUT", endpoint="/contract_change/update", data=request_data)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
@@ -0,0 +1,10 @@
|
||||
"""Contract Change Domain - JSONL Handler"""
|
||||
from ...datasource import BaseJsonlHandler, JsonlDataSource
|
||||
from .sync_node import ContractChangeSyncNode
|
||||
from schemas.project_extensions.contract_change import ContractChangeResponse
|
||||
|
||||
|
||||
class ContractChangeJsonlHandler(BaseJsonlHandler):
|
||||
"""合同变更 JSONL Handler"""
|
||||
def __init__(self, datasource: JsonlDataSource):
|
||||
super().__init__(datasource, "contract_change", ContractChangeSyncNode, ContractChangeResponse)
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Contract change domain registration"""
|
||||
from ...common.registry import DomainRegistry
|
||||
from schemas.project_extensions.contract_change import ContractChangeResponse
|
||||
from .sync_node import ContractChangeSyncNode
|
||||
from .sync_strategy import ContractChangeSyncStrategy
|
||||
from .api_handler import ContractChangeApiHandler
|
||||
from .jsonl_handler import ContractChangeJsonlHandler
|
||||
|
||||
# 注册 contract_change domain
|
||||
DomainRegistry.register(
|
||||
node_type="contract_change",
|
||||
schema=ContractChangeResponse,
|
||||
node_class=ContractChangeSyncNode,
|
||||
strategy_class=ContractChangeSyncStrategy,
|
||||
jsonl_handler_class=ContractChangeJsonlHandler,
|
||||
api_handler_class=ContractChangeApiHandler,
|
||||
)
|
||||
@@ -0,0 +1,10 @@
|
||||
"""Contract Change SyncNode"""
|
||||
|
||||
from ...common.sync_node import SyncNode
|
||||
from schemas.project_extensions.contract_change import ContractChangeResponse
|
||||
|
||||
|
||||
class ContractChangeSyncNode(SyncNode[ContractChangeResponse]):
|
||||
"""合同变更同步节点"""
|
||||
node_type = "contract_change"
|
||||
schema = ContractChangeResponse
|
||||
@@ -0,0 +1,20 @@
|
||||
from ...sync_system.strategy import DefaultSyncStrategy
|
||||
from ...sync_system.config import StrategyConfig, OrphanAction, UpdateDirection
|
||||
from schemas.project_extensions.contract_change import ContractChangeResponse
|
||||
|
||||
|
||||
class ContractChangeSyncStrategy(DefaultSyncStrategy[ContractChangeResponse]):
|
||||
"""
|
||||
Contract Change 同步策略。
|
||||
"""
|
||||
|
||||
schema = ContractChangeResponse
|
||||
|
||||
default_config = StrategyConfig(
|
||||
auto_bind=True,
|
||||
auto_bind_fields=["contract_id", "change_time", "title"],
|
||||
depend_fields={"project_id": "project", "contract_id": "contract"},
|
||||
local_orphan_action=OrphanAction.CREATE_REMOTE,
|
||||
remote_orphan_action=OrphanAction.NONE,
|
||||
update_direction=UpdateDirection.PUSH,
|
||||
)
|
||||
@@ -0,0 +1,6 @@
|
||||
"""Contract settlement domain"""
|
||||
from .sync_node import ContractSettlementSyncNode
|
||||
from .sync_strategy import ContractSettlementSyncStrategy
|
||||
from .jsonl_handler import ContractSettlementJsonlHandler
|
||||
|
||||
__all__ = ["ContractSettlementSyncNode", "ContractSettlementSyncStrategy", "ContractSettlementJsonlHandler"]
|
||||
@@ -0,0 +1,313 @@
|
||||
"""
|
||||
Contract Settlement API Handler
|
||||
|
||||
职责：
|
||||
- 加载合同结算列表（依赖 Project）
|
||||
- 合同结算创建/更新/删除（批量接口）
|
||||
"""
|
||||
|
||||
from typing import List, Dict, Any
|
||||
from ...datasource.api.handler import BaseApiHandler
|
||||
from ...datasource.task_result import TaskResult
|
||||
from ...common.sync_node import SyncNode
|
||||
from ...datasource.api.errors import (
|
||||
ERROR_NODE_DATA_NONE,
|
||||
ERROR_VALIDATION_FAILED,
|
||||
ERROR_PROJECT_ID_NOT_FOUND
|
||||
)
|
||||
from .sync_node import ContractSettlementSyncNode
|
||||
from schemas.contract_settlement.contract_settlement import (
|
||||
ContractSettlementResponse,
|
||||
ContractSettlementCreate,
|
||||
ContractSettlementUpdate,
|
||||
ContractSettlementPlanUpdate,
|
||||
)
|
||||
|
||||
|
||||
class ContractSettlementApiHandler(BaseApiHandler):
|
||||
"""合同结算 API Handler"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._node_type = "contract_settlement"
|
||||
self._node_class = ContractSettlementSyncNode
|
||||
self._schema = ContractSettlementResponse
|
||||
self.update_schemas = [ContractSettlementCreate, ContractSettlementUpdate, ContractSettlementPlanUpdate]
|
||||
|
||||
async def load(self) -> List[SyncNode]:
|
||||
"""加载合同结算列表（依赖 Project）"""
|
||||
projects = self._collection.filter(node_type="project")
|
||||
project_ids = [p.data_id for p in projects if p.data_id]
|
||||
|
||||
if not project_ids:
|
||||
return []
|
||||
|
||||
all_settlements = []
|
||||
for project_id in project_ids:
|
||||
try:
|
||||
settlements = await api_get_contract_settlement_list(self.api_client, project_id)
|
||||
for settlement_model in settlements:
|
||||
# 已经过验证的 Pydantic model，转换为 dict
|
||||
settlement_data = settlement_model.model_dump()
|
||||
all_settlements.append(self._create_node(settlement_data))
|
||||
except Exception as e:
|
||||
print(f"❌ Failed to load settlements for project {project_id}: {e}")
|
||||
|
||||
return all_settlements
|
||||
|
||||
async def create_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""批量创建合同结算"""
|
||||
from pydantic import ValidationError
|
||||
|
||||
results = []
|
||||
for node in nodes:
|
||||
data = node.get_data()
|
||||
if not data:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=ERROR_NODE_DATA_NONE))
|
||||
continue
|
||||
|
||||
project_id = node.context.get("project_id") or data.get("project_id")
|
||||
if not project_id:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=ERROR_PROJECT_ID_NOT_FOUND))
|
||||
continue
|
||||
|
||||
# 验证并转换为 Pydantic model
|
||||
try:
|
||||
create_data = {"project_id": project_id, **data}
|
||||
create_model = ContractSettlementCreate.model_validate(create_data)
|
||||
except ValidationError as e:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=f"{ERROR_VALIDATION_FAILED}: {e}"))
|
||||
continue
|
||||
|
||||
push_id = self._generate_push_id()
|
||||
|
||||
try:
|
||||
response = await api_create_contract_settlement(self.api_client, create_model, push_id)
|
||||
if self.poll_mode == "sync":
|
||||
created_id = self._extract_created_id_from_response(response)
|
||||
if not created_id:
|
||||
results.append(
|
||||
TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error="Missing created_id in create response"
|
||||
)
|
||||
)
|
||||
continue
|
||||
results.append(TaskResult.success(node_id=node.node_id, data_id=created_id))
|
||||
else:
|
||||
results.append(TaskResult.in_progress(node_id=node.node_id, task_id=push_id))
|
||||
except Exception as e:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=str(e)))
|
||||
|
||||
return results
|
||||
|
||||
async def update_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""
|
||||
批量更新合同结算
|
||||
|
||||
支持两种更新类型：
|
||||
1. ContractSettlementUpdate - 基础信息更新
|
||||
2. ContractSettlementPlanUpdate - 计划信息更新
|
||||
|
||||
更新逻辑：
|
||||
- 对每种 schema，检查是否有差异
|
||||
- 如果有差异，调用对应 API
|
||||
"""
|
||||
from pydantic import ValidationError
|
||||
|
||||
results = []
|
||||
for node in nodes:
|
||||
data = node.get_data()
|
||||
if not data:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=ERROR_NODE_DATA_NONE))
|
||||
continue
|
||||
|
||||
origin_data = node.get_origin_data() or {}
|
||||
biz_id = node.data_id or data.get("id")
|
||||
if not biz_id:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error="Missing biz_id for update"))
|
||||
continue
|
||||
|
||||
node_updated = False
|
||||
error = None
|
||||
|
||||
# 1. 基础信息更新 (ContractSettlementUpdate)
|
||||
base_diff = self._get_schema_diff(ContractSettlementUpdate, data, origin_data, node.node_id)
|
||||
if base_diff:
|
||||
try:
|
||||
update_model = ContractSettlementUpdate.model_validate(data)
|
||||
push_id = self._generate_push_id()
|
||||
await api_update_contract_settlement(self.api_client, update_model, push_id, biz_id)
|
||||
node_updated = True
|
||||
except ValidationError as e:
|
||||
error = f"Validation failed: {e}"
|
||||
except Exception as e:
|
||||
error = f"API error: {e}"
|
||||
|
||||
# 2. 计划信息更新 (ContractSettlementPlanUpdate)
|
||||
plan_diff = self._get_schema_diff(ContractSettlementPlanUpdate, data, origin_data, node.node_id)
|
||||
if not error and plan_diff:
|
||||
try:
|
||||
plan_model = ContractSettlementPlanUpdate.model_validate(data)
|
||||
push_id = self._generate_push_id()
|
||||
# 使用计划更新专用 API
|
||||
await api_update_settlement_plan(self.api_client, plan_model.model_dump(exclude_unset=True), push_id, biz_id)
|
||||
node_updated = True
|
||||
except ValidationError as e:
|
||||
error = f"Validation failed: {e}"
|
||||
except Exception as e:
|
||||
error = f"API error: {e}"
|
||||
|
||||
# 汇总结果
|
||||
if error:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=error))
|
||||
elif node_updated:
|
||||
results.append(TaskResult.success(node_id=node.node_id))
|
||||
else:
|
||||
# 没有任何字段需要更新
|
||||
from ...common.types import SyncAction
|
||||
results.append(TaskResult.skipped(node_id=node.node_id, reason="No updatable fields changed"))
|
||||
|
||||
return results
|
||||
|
||||
async def delete_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""批量删除合同结算"""
|
||||
results = []
|
||||
for node in nodes:
|
||||
push_id = self._generate_push_id()
|
||||
biz_id = node.data_id
|
||||
if not biz_id:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error="Missing biz_id for delete"))
|
||||
continue
|
||||
|
||||
try:
|
||||
await api_delete_contract_settlement(self.api_client, biz_id, push_id)
|
||||
results.append(TaskResult.success(node_id=node.node_id))
|
||||
except Exception as e:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=str(e)))
|
||||
|
||||
return results
|
||||
|
||||
async def poll_tasks(self, task_ids: List[str]) -> Dict[str, TaskResult]:
|
||||
"""轮询异步任务状态（占位）"""
|
||||
return await super().poll_tasks(task_ids)
|
||||
|
||||
def _create_node(self, data: Dict[str, Any]) -> SyncNode:
|
||||
node = self._create_basic_node(data, depend_ids=[])
|
||||
|
||||
if data.get("project_id"):
|
||||
node.context["project_id"] = data["project_id"]
|
||||
node.context["settlement_id"] = node.data_id
|
||||
|
||||
return node
|
||||
|
||||
|
||||
# ========== 静态 API 函数 ==========
|
||||
|
||||
async def api_get_settlement(client, settlement_id: str) -> Dict[str, Any]:
|
||||
"""获取单个合同结算数据 GET /contract_settlement"""
|
||||
response = await client.request(
|
||||
method="GET",
|
||||
endpoint="/contract_settlement",
|
||||
params={"settlement_id": settlement_id}
|
||||
)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
return response.get("data", {})
|
||||
|
||||
|
||||
async def api_get_contract_settlement_list(client, project_id: str) -> List[ContractSettlementResponse]:
|
||||
"""
|
||||
获取合同结算列表 GET /contract_settlement/list
|
||||
|
||||
Args:
|
||||
client: API 客户端
|
||||
project_id: 项目ID
|
||||
|
||||
Returns:
|
||||
List[ContractSettlementResponse]: 验证过的结算列表
|
||||
|
||||
Raises:
|
||||
Exception: API 返回错误或数据验证失败
|
||||
"""
|
||||
from pydantic import ValidationError
|
||||
|
||||
response = await client.request(
|
||||
method="GET",
|
||||
endpoint="/contract_settlement/list",
|
||||
params={"project_id": project_id}
|
||||
)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
|
||||
items = response.get("data", [])
|
||||
|
||||
# 验证每个返回项
|
||||
validated_items = []
|
||||
for item in items:
|
||||
try:
|
||||
validated_item = ContractSettlementResponse.model_validate(item)
|
||||
validated_items.append(validated_item)
|
||||
except ValidationError as e:
|
||||
print(f"⚠️ Skipping invalid settlement item: {e}")
|
||||
continue
|
||||
|
||||
return validated_items
|
||||
|
||||
|
||||
async def api_create_contract_settlement(client, data: ContractSettlementCreate, push_id: str) -> Dict[str, Any]:
|
||||
"""
|
||||
创建合同结算 POST /contract_settlement/create
|
||||
|
||||
Args:
|
||||
client: API 客户端
|
||||
data: 创建数据（Pydantic model）
|
||||
push_id: 推送ID
|
||||
biz_id: 业务ID
|
||||
|
||||
Returns:
|
||||
str: push_id
|
||||
|
||||
Raises:
|
||||
Exception: API 返回错误
|
||||
"""
|
||||
request_data = {"push_id": push_id, "data": data.model_dump()}
|
||||
response = await client.request(method="POST", endpoint="/contract_settlement/create", data=request_data)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
return response
|
||||
|
||||
|
||||
async def api_update_contract_settlement(client, data: ContractSettlementUpdate, push_id: str, biz_id: str) -> None:
|
||||
"""
|
||||
更新合同结算 PUT /contract_settlement/update
|
||||
|
||||
Args:
|
||||
client: API 客户端
|
||||
data: 更新数据（Pydantic model）
|
||||
push_id: 推送ID
|
||||
biz_id: 业务ID
|
||||
|
||||
Raises:
|
||||
Exception: API 返回错误
|
||||
"""
|
||||
request_data = {"push_id": push_id, "biz_id": biz_id, "data": data.model_dump(exclude_unset=True)}
|
||||
response = await client.request(method="PUT", endpoint="/contract_settlement/update", data=request_data)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
|
||||
|
||||
async def api_delete_contract_settlement(client, settlement_id: str, push_id: str) -> None:
|
||||
"""删除合同结算 DELETE /contract_settlement/delete"""
|
||||
request_data = {"push_id": push_id, "biz_id": settlement_id}
|
||||
response = await client.request(method="DELETE", endpoint="/contract_settlement/delete", data=request_data)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
|
||||
|
||||
async def api_update_settlement_plan(client, data: Dict[str, Any], push_id: str, biz_id: str) -> None:
|
||||
"""更新合同结算计划 PUT /contract_settlement/plan"""
|
||||
request_data = {"push_id": push_id, "biz_id": biz_id, "data": data}
|
||||
response = await client.request(method="PUT", endpoint="/contract_settlement/plan", data=request_data)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
@@ -0,0 +1,10 @@
|
||||
"""Contract Settlement Domain - JSONL Handler"""
|
||||
from ...datasource import BaseJsonlHandler, JsonlDataSource
|
||||
from .sync_node import ContractSettlementSyncNode
|
||||
from schemas.contract_settlement.contract_settlement import ContractSettlementResponse
|
||||
|
||||
|
||||
class ContractSettlementJsonlHandler(BaseJsonlHandler):
|
||||
"""合同结算 JSONL Handler"""
|
||||
def __init__(self, datasource: JsonlDataSource):
|
||||
super().__init__(datasource, "contract_settlement", ContractSettlementSyncNode, ContractSettlementResponse)
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Contract settlement domain registration"""
|
||||
from ...common.registry import DomainRegistry
|
||||
from schemas.contract_settlement.contract_settlement import ContractSettlementResponse
|
||||
from .sync_node import ContractSettlementSyncNode
|
||||
from .sync_strategy import ContractSettlementSyncStrategy
|
||||
from .jsonl_handler import ContractSettlementJsonlHandler
|
||||
from .api_handler import ContractSettlementApiHandler
|
||||
|
||||
# 注册 contract_settlement domain
|
||||
DomainRegistry.register(
|
||||
node_type="contract_settlement",
|
||||
schema=ContractSettlementResponse,
|
||||
node_class=ContractSettlementSyncNode,
|
||||
strategy_class=ContractSettlementSyncStrategy,
|
||||
jsonl_handler_class=ContractSettlementJsonlHandler,
|
||||
api_handler_class=ContractSettlementApiHandler,
|
||||
)
|
||||
@@ -0,0 +1,11 @@
|
||||
from ...common.sync_node import SyncNode
|
||||
from schemas.contract_settlement.contract_settlement import ContractSettlementResponse
|
||||
|
||||
|
||||
class ContractSettlementSyncNode(SyncNode[ContractSettlementResponse]):
|
||||
"""合同结算同步节点"""
|
||||
node_type = "contract_settlement"
|
||||
schema = ContractSettlementResponse
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
@@ -0,0 +1,20 @@
|
||||
from ...sync_system.strategy import DefaultSyncStrategy
|
||||
from ...sync_system.config import StrategyConfig, OrphanAction, UpdateDirection
|
||||
from schemas.contract_settlement.contract_settlement import ContractSettlementResponse
|
||||
|
||||
|
||||
class ContractSettlementSyncStrategy(DefaultSyncStrategy[ContractSettlementResponse]):
|
||||
"""
|
||||
Contract Settlement 同步策略（合同子业务）。
|
||||
"""
|
||||
|
||||
schema = ContractSettlementResponse
|
||||
|
||||
default_config = StrategyConfig(
|
||||
auto_bind=True,
|
||||
auto_bind_fields=["contract_id"],
|
||||
depend_fields={"contract_id": "contract", "project_id": "project"},
|
||||
local_orphan_action=OrphanAction.CREATE_REMOTE,
|
||||
remote_orphan_action=OrphanAction.NONE,
|
||||
update_direction=UpdateDirection.PUSH,
|
||||
)
|
||||
@@ -0,0 +1,8 @@
|
||||
"""
|
||||
Contract Settlement Detail Domain
|
||||
"""
|
||||
from .sync_node import ContractSettlementDetailSyncNode
|
||||
from .sync_strategy import ContractSettlementDetailSyncStrategy
|
||||
from .jsonl_handler import ContractSettlementDetailJsonlHandler
|
||||
|
||||
__all__ = ["ContractSettlementDetailSyncNode", "ContractSettlementDetailSyncStrategy", "ContractSettlementDetailJsonlHandler"]
|
||||
@@ -0,0 +1,288 @@
|
||||
"""
|
||||
Contract Settlement Detail API Handler
|
||||
|
||||
职责：
|
||||
- 加载合同结算明细列表（依赖 ContractSettlement）
|
||||
- 合同结算明细创建/更新/删除（批量接口）
|
||||
"""
|
||||
|
||||
from typing import List, Dict, Any
|
||||
from ...datasource.api.handler import BaseApiHandler
|
||||
from ...datasource.task_result import TaskResult
|
||||
from ...common.sync_node import SyncNode
|
||||
from ...datasource.api.errors import ERROR_NODE_DATA_NONE, ERROR_VALIDATION_FAILED, ERROR_ID_NOT_FOUND
|
||||
from .sync_node import ContractSettlementDetailSyncNode
|
||||
from schemas.contract_settlement.contract_settlement_detail import (
|
||||
ContractSettlementDetailResponse,
|
||||
ContractSettlementDetailCreate,
|
||||
ContractSettlementDetailUpdate
|
||||
)
|
||||
|
||||
|
||||
class ContractSettlementDetailApiHandler(BaseApiHandler):
|
||||
"""合同结算明细 API Handler"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._node_type = "contract_settlement_detail"
|
||||
self._node_class = ContractSettlementDetailSyncNode
|
||||
self._schema = ContractSettlementDetailResponse
|
||||
self.update_schemas = [ContractSettlementDetailCreate, ContractSettlementDetailUpdate]
|
||||
|
||||
async def load(self) -> List[SyncNode]:
|
||||
"""加载合同结算明细列表（依赖 ContractSettlement）"""
|
||||
settlements = self._collection.filter(node_type="contract_settlement")
|
||||
settlement_ids = [s.data_id for s in settlements if s.data_id]
|
||||
|
||||
if not settlement_ids:
|
||||
return []
|
||||
|
||||
all_details = []
|
||||
for settlement_id in settlement_ids:
|
||||
try:
|
||||
details = await api_get_contract_settlement_detail_list(self.api_client, settlement_id)
|
||||
for detail_model in details:
|
||||
detail_data = detail_model.model_dump()
|
||||
all_details.append(self._create_node(detail_data))
|
||||
except Exception as e:
|
||||
print(f"❌ Failed to load settlement details for {settlement_id}: {e}")
|
||||
|
||||
return all_details
|
||||
|
||||
async def create_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""批量创建合同结算明细"""
|
||||
from pydantic import ValidationError
|
||||
|
||||
results = []
|
||||
for node in nodes:
|
||||
data = node.get_data()
|
||||
if not data:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=ERROR_NODE_DATA_NONE))
|
||||
continue
|
||||
|
||||
# 从data的parent_id获取settlement_id
|
||||
settlement_id = data.get("parent_id")
|
||||
if not settlement_id:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=ERROR_ID_NOT_FOUND))
|
||||
continue
|
||||
|
||||
try:
|
||||
create_data = {"settlement_id": settlement_id, **data}
|
||||
create_model = ContractSettlementDetailCreate.model_validate(create_data)
|
||||
except ValidationError as e:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=f"{ERROR_VALIDATION_FAILED}: {e}"))
|
||||
continue
|
||||
|
||||
push_id = self._generate_push_id()
|
||||
|
||||
try:
|
||||
response = await api_create_contract_settlement_detail(self.api_client, create_model, push_id)
|
||||
if self.poll_mode == "sync":
|
||||
created_id = self._extract_created_id_from_response(response)
|
||||
if not created_id:
|
||||
results.append(
|
||||
TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error="Missing created_id in create response"
|
||||
)
|
||||
)
|
||||
continue
|
||||
results.append(TaskResult.success(node_id=node.node_id, data_id=created_id))
|
||||
else:
|
||||
results.append(TaskResult.in_progress(node_id=node.node_id, task_id=push_id))
|
||||
except Exception as e:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=str(e)))
|
||||
|
||||
return results
|
||||
|
||||
async def update_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""
|
||||
批量更新合同结算明细
|
||||
|
||||
按 settlement_id 分组，每组使用同一个 push_id 批量提交
|
||||
"""
|
||||
from pydantic import ValidationError
|
||||
from collections import defaultdict
|
||||
|
||||
results = []
|
||||
|
||||
# 按 settlement_id 分组
|
||||
groups: Dict[str, List[tuple[SyncNode, Dict[str, Any]]]] = defaultdict(list)
|
||||
|
||||
for node in nodes:
|
||||
data = node.get_data()
|
||||
if not data:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=ERROR_NODE_DATA_NONE))
|
||||
continue
|
||||
|
||||
origin_data = node.get_origin_data() or {}
|
||||
|
||||
# 使用 _get_schema_diff 检查差异并打印日志
|
||||
diff_fields = self._get_schema_diff(ContractSettlementDetailUpdate, data, origin_data, node.node_id)
|
||||
if not diff_fields:
|
||||
from ...common.types import SyncAction
|
||||
results.append(TaskResult.skipped(node_id=node.node_id, reason="No updatable fields changed"))
|
||||
continue
|
||||
|
||||
# 获取 settlement_id（parent_id）
|
||||
settlement_id = data.get("parent_id") or node.context.get("settlement_id")
|
||||
if not settlement_id:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error="Missing settlement_id (parent_id)"))
|
||||
continue
|
||||
|
||||
# 需要完整的 schema 数据（包括必填字段）
|
||||
update_data = self._filter_update_data(data, origin_data, ContractSettlementDetailUpdate)
|
||||
|
||||
try:
|
||||
update_model = ContractSettlementDetailUpdate.model_validate(update_data)
|
||||
groups[settlement_id].append((node, update_model))
|
||||
except ValidationError as e:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=f"{ERROR_VALIDATION_FAILED}: {e}"))
|
||||
|
||||
# 逐组批量更新
|
||||
for settlement_id, group_items in groups.items():
|
||||
push_id = self._generate_push_id()
|
||||
|
||||
# 准备批量数据
|
||||
detail_list = [model.model_dump(mode='json', exclude_unset=True) for node, model in group_items]
|
||||
|
||||
# 使用第一个节点的 biz_id（对于批量更新，通常使用 parent_id）
|
||||
biz_id = settlement_id
|
||||
|
||||
try:
|
||||
# 批量调用 API
|
||||
await api_update_contract_settlement_detail(
|
||||
self.api_client,
|
||||
{"detail_list": detail_list},
|
||||
push_id,
|
||||
biz_id,
|
||||
)
|
||||
# 整组成功
|
||||
for node, _ in group_items:
|
||||
results.append(TaskResult.success(node_id=node.node_id, push_id=push_id))
|
||||
except Exception as e:
|
||||
# 整组失败
|
||||
error_msg = str(e)
|
||||
for node, _ in group_items:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=error_msg))
|
||||
|
||||
return results
|
||||
|
||||
async def delete_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""批量删除合同结算明细"""
|
||||
results = []
|
||||
for node in nodes:
|
||||
push_id = self._generate_push_id()
|
||||
biz_id = node.data_id
|
||||
if not biz_id:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error="Missing biz_id for delete"))
|
||||
continue
|
||||
|
||||
try:
|
||||
await api_delete_contract_settlement_detail(self.api_client, biz_id, push_id)
|
||||
results.append(TaskResult.success(node_id=node.node_id))
|
||||
except Exception as e:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=str(e)))
|
||||
|
||||
return results
|
||||
|
||||
async def poll_tasks(self, task_ids: List[str]) -> Dict[str, TaskResult]:
|
||||
"""轮询异步任务状态（占位）"""
|
||||
return await super().poll_tasks(task_ids)
|
||||
|
||||
def _create_node(self, data: Dict[str, Any]) -> SyncNode:
|
||||
node = self._create_basic_node(data, depend_ids=[])
|
||||
|
||||
if data.get("settlement_id"):
|
||||
node.context["settlement_id"] = data["settlement_id"]
|
||||
|
||||
return node
|
||||
|
||||
|
||||
# ========== 静态 API 函数 ==========
|
||||
|
||||
async def api_get_contract_settlement_detail_list(client, settlement_id: str) -> List[ContractSettlementDetailResponse]:
|
||||
"""
|
||||
获取合同结算明细列表
|
||||
|
||||
Args:
|
||||
client: API 客户端
|
||||
settlement_id: 结算ID
|
||||
|
||||
Returns:
|
||||
List[ContractSettlementDetailResponse]: 验证过的明细列表
|
||||
|
||||
Raises:
|
||||
Exception: API 返回错误或数据验证失败
|
||||
"""
|
||||
from pydantic import ValidationError
|
||||
|
||||
response = await client.request(
|
||||
method="GET",
|
||||
endpoint="/contract_settlement/detail/list",
|
||||
params={"settlement_id": settlement_id}
|
||||
)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
|
||||
items = response.get("data", [])
|
||||
|
||||
validated_items = []
|
||||
for item in items:
|
||||
try:
|
||||
validated_item = ContractSettlementDetailResponse.model_validate(item)
|
||||
validated_items.append(validated_item)
|
||||
except ValidationError as e:
|
||||
print(f"⚠️ Skipping invalid detail item: {e}")
|
||||
continue
|
||||
|
||||
return validated_items
|
||||
|
||||
|
||||
async def api_create_contract_settlement_detail(client, data: ContractSettlementDetailCreate, push_id: str) -> Dict[str, Any]:
|
||||
"""
|
||||
创建合同结算明细
|
||||
|
||||
Args:
|
||||
client: API 客户端
|
||||
data: 创建数据（Pydantic model）
|
||||
push_id: 推送ID
|
||||
biz_id: 业务ID
|
||||
|
||||
Returns:
|
||||
str: push_id
|
||||
|
||||
Raises:
|
||||
Exception: API 返回错误
|
||||
"""
|
||||
request_data = {"push_id": push_id, "data": data.model_dump(mode='json')}
|
||||
response = await client.request(method="POST", endpoint="/contract_settlement/detail/create", data=request_data)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
return response
|
||||
|
||||
|
||||
async def api_update_contract_settlement_detail(client, data: Dict[str, Any], push_id: str, biz_id: str) -> None:
|
||||
"""
|
||||
更新合同结算明细
|
||||
|
||||
Args:
|
||||
client: API 客户端
|
||||
data: 更新数据（Pydantic model）
|
||||
push_id: 推送ID
|
||||
biz_id: 业务ID
|
||||
|
||||
Raises:
|
||||
Exception: API 返回错误
|
||||
"""
|
||||
request_data = {"push_id": push_id, "biz_id": biz_id, "data": data}
|
||||
response = await client.request(method="PUT", endpoint="/contract_settlement/detail/update", data=request_data)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
|
||||
|
||||
async def api_delete_contract_settlement_detail(client, detail_id: str, push_id: str) -> None:
|
||||
request_data = {"push_id": push_id, "biz_id": detail_id}
|
||||
response = await client.request(method="DELETE", endpoint="/contract_settlement/detail/delete", data=request_data)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
@@ -0,0 +1,12 @@
|
||||
"""
|
||||
Contract Settlement Detail JSONL Handler
|
||||
"""
|
||||
from ...datasource import BaseJsonlHandler, JsonlDataSource
|
||||
from .sync_node import ContractSettlementDetailSyncNode
|
||||
from schemas.contract_settlement.contract_settlement_detail import ContractSettlementDetailResponse
|
||||
|
||||
|
||||
class ContractSettlementDetailJsonlHandler(BaseJsonlHandler):
|
||||
"""合同结算明细 JSONL Handler"""
|
||||
def __init__(self, datasource: JsonlDataSource):
|
||||
super().__init__(datasource, "contract_settlement_detail", ContractSettlementDetailSyncNode, ContractSettlementDetailResponse)
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Contract settlement detail domain registration"""
|
||||
from ...common.registry import DomainRegistry
|
||||
from schemas.contract_settlement.contract_settlement_detail import ContractSettlementDetailResponse
|
||||
from .sync_node import ContractSettlementDetailSyncNode
|
||||
from .sync_strategy import ContractSettlementDetailSyncStrategy
|
||||
from .jsonl_handler import ContractSettlementDetailJsonlHandler
|
||||
from .api_handler import ContractSettlementDetailApiHandler
|
||||
|
||||
# 注册 contract_settlement_detail domain
|
||||
DomainRegistry.register(
|
||||
node_type="contract_settlement_detail",
|
||||
schema=ContractSettlementDetailResponse,
|
||||
node_class=ContractSettlementDetailSyncNode,
|
||||
strategy_class=ContractSettlementDetailSyncStrategy,
|
||||
jsonl_handler_class=ContractSettlementDetailJsonlHandler,
|
||||
api_handler_class=ContractSettlementDetailApiHandler,
|
||||
)
|
||||
@@ -0,0 +1,11 @@
|
||||
"""
|
||||
Contract Settlement Detail Domain
|
||||
"""
|
||||
from ...common import SyncNode
|
||||
from schemas.contract_settlement.contract_settlement_detail import ContractSettlementDetailResponse
|
||||
|
||||
|
||||
class ContractSettlementDetailSyncNode(SyncNode[ContractSettlementDetailResponse]):
|
||||
"""合同结算明细同步节点"""
|
||||
node_type = "contract_settlement_detail"
|
||||
schema = ContractSettlementDetailResponse
|
||||
@@ -0,0 +1,20 @@
|
||||
from ...sync_system.strategy import DefaultSyncStrategy
|
||||
from ...sync_system.config import StrategyConfig, OrphanAction, UpdateDirection
|
||||
from schemas.contract_settlement.contract_settlement_detail import ContractSettlementDetailResponse
|
||||
|
||||
|
||||
class ContractSettlementDetailSyncStrategy(DefaultSyncStrategy[ContractSettlementDetailResponse]):
|
||||
"""
|
||||
Contract Settlement Detail 同步策略（合同子业务明细）。
|
||||
"""
|
||||
|
||||
schema = ContractSettlementDetailResponse
|
||||
|
||||
default_config = StrategyConfig(
|
||||
auto_bind=True,
|
||||
auto_bind_fields=["parent_id", "date"],
|
||||
depend_fields={"parent_id": "contract_settlement"},
|
||||
local_orphan_action=OrphanAction.CREATE_REMOTE,
|
||||
remote_orphan_action=OrphanAction.NONE,
|
||||
update_direction=UpdateDirection.PUSH,
|
||||
)
|
||||
@@ -0,0 +1,6 @@
|
||||
"""LAR domain"""
|
||||
from .sync_node import LarSyncNode
|
||||
from .sync_strategy import LarSyncStrategy
|
||||
from .jsonl_handler import LarJsonlHandler
|
||||
|
||||
__all__ = ["LarSyncNode", "LarSyncStrategy", "LarJsonlHandler"]
|
||||
@@ -0,0 +1,218 @@
|
||||
"""
|
||||
LAR (移民征地) API Handler
|
||||
|
||||
职责：
|
||||
- 更新移民征地数据（依赖 Project）
|
||||
- 支持主数据、搬迁信息、投资信息三种更新
|
||||
- 不支持创建和删除（数据随项目自动生成）
|
||||
|
||||
实现的接口：
|
||||
PUT:
|
||||
- /lar/main - 更新主数据
|
||||
- /lar/resettlement - 更新搬迁信息
|
||||
- /lar/investment - 更新投资信息
|
||||
"""
|
||||
|
||||
from typing import List, Dict, Any
|
||||
from ...datasource.api.handler import BaseApiHandler
|
||||
from ...datasource.task_result import TaskResult
|
||||
from ...common.sync_node import SyncNode
|
||||
from ...datasource.api.errors import ERROR_VALIDATION_FAILED
|
||||
from .sync_node import LarSyncNode
|
||||
from schemas.project_extensions.lar import (
|
||||
LarDetailSchema,
|
||||
LarMainInfoUpdateSchema,
|
||||
LarResettlementSchema,
|
||||
LarInvestmentSchema
|
||||
)
|
||||
|
||||
|
||||
class LarApiHandler(BaseApiHandler):
|
||||
"""移民征地 API Handler"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._node_type = "lar"
|
||||
self._node_class = LarSyncNode
|
||||
self._schema = LarDetailSchema
|
||||
|
||||
async def load(self) -> List[SyncNode]:
|
||||
"""
|
||||
加载移民征地数据（从 project all_info 中提取）
|
||||
|
||||
注意：LAR 是可选的，只有特定项目类型（水电、抽蓄）才有
|
||||
"""
|
||||
from ..project.api_handler import ProjectApiHandler
|
||||
|
||||
# 获取已加载的项目
|
||||
projects = self._collection.filter(node_type="project")
|
||||
if not projects:
|
||||
print("⚠️ No projects found, skipping LAR load")
|
||||
return []
|
||||
|
||||
nodes = []
|
||||
for project in projects:
|
||||
project_id = project.data_id
|
||||
|
||||
try:
|
||||
# 从 project context 或缓存获取 all_info
|
||||
all_info = project.context.get("all_info")
|
||||
if not all_info:
|
||||
all_info = await ProjectApiHandler.get_all_info(
|
||||
self.api_client,
|
||||
project_id
|
||||
)
|
||||
|
||||
# 提取 lar（注意：lar 是单个对象，不是列表）
|
||||
lar_data = all_info.get("lar")
|
||||
if lar_data:
|
||||
try:
|
||||
validated = self._schema.model_validate(lar_data)
|
||||
node = self._create_node(validated.model_dump())
|
||||
node.context["project_id"] = project_id
|
||||
nodes.append(node)
|
||||
print(f" Project {project_id}: 1 LAR")
|
||||
except Exception as e:
|
||||
print(f"⚠️ Skipping invalid LAR for project {project_id}: {e}")
|
||||
continue
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Failed to load LAR for project {project_id}: {e}")
|
||||
continue
|
||||
|
||||
return nodes
|
||||
|
||||
async def create_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""不支持创建（LAR随项目自动生成）"""
|
||||
return [
|
||||
TaskResult.failed(node_id=node.node_id, error="LAR creation not supported (auto-generated with project)")
|
||||
for node in nodes
|
||||
]
|
||||
|
||||
async def update_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""批量更新移民征地数据"""
|
||||
from pydantic import ValidationError
|
||||
|
||||
results = []
|
||||
|
||||
# 定义三种更新类型
|
||||
update_configs = [
|
||||
(LarMainInfoUpdateSchema, api_update_lar_main),
|
||||
(LarResettlementSchema, api_update_lar_resettlement),
|
||||
(LarInvestmentSchema, api_update_lar_investment),
|
||||
]
|
||||
|
||||
for node in nodes:
|
||||
node_data = node.get_data()
|
||||
if not node_data:
|
||||
error_msg = node.error or "Node data is missing or invalid"
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=f"Cannot update: {error_msg}"))
|
||||
continue
|
||||
|
||||
origin_data = node.get_origin_data() or {}
|
||||
node_updated = False
|
||||
|
||||
for schema, api_func in update_configs:
|
||||
# 获取过滤后的更新数据
|
||||
update_data = self._filter_update_data(node_data, origin_data, schema)
|
||||
if update_data:
|
||||
# 验证并转换为 Pydantic model
|
||||
try:
|
||||
update_model = schema.model_validate(update_data)
|
||||
except ValidationError as e:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=f"{ERROR_VALIDATION_FAILED}: {e}"))
|
||||
break
|
||||
|
||||
push_id = self._generate_push_id()
|
||||
update_model_data = update_model.model_dump()
|
||||
biz_id = node.data_id or update_model_data.get("id")
|
||||
if not biz_id:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error="Missing biz_id for update"))
|
||||
break
|
||||
|
||||
try:
|
||||
await api_func(self.api_client, update_model, push_id, biz_id)
|
||||
node_updated = True
|
||||
except Exception as e:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=str(e)))
|
||||
break
|
||||
|
||||
if node_updated:
|
||||
results.append(TaskResult.success(node_id=node.node_id))
|
||||
elif not any(r.node_id == node.node_id for r in results):
|
||||
# 如果没有任何更新且没有报错，视为跳过（归一化后无差异）
|
||||
reason = f"No physical diff found across all 3 LAR sub-schemas for {node.data_id} despite strategy update request (Normalization Discrepancy)."
|
||||
print(f" [WARNING] [lar] {reason}")
|
||||
results.append(TaskResult.skipped(node_id=node.node_id, reason=reason))
|
||||
|
||||
return results
|
||||
|
||||
async def delete_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""不支持删除（LAR随项目存在）"""
|
||||
return [
|
||||
TaskResult.failed(node_id=node.node_id, error="LAR deletion not supported")
|
||||
for node in nodes
|
||||
]
|
||||
|
||||
async def poll_tasks(self, task_ids: List[str]) -> Dict[str, TaskResult]:
|
||||
"""轮询异步任务状态"""
|
||||
return await super().poll_tasks(task_ids)
|
||||
|
||||
def _create_node(self, data: Dict[str, Any]) -> SyncNode:
|
||||
node = self._create_basic_node(data, depend_ids=[])
|
||||
|
||||
if data.get("project_id"):
|
||||
node.context["project_id"] = data["project_id"]
|
||||
node.context["lar_id"] = node.data_id
|
||||
|
||||
return node
|
||||
|
||||
|
||||
# ========== 静态 API 函数 ==========
|
||||
|
||||
async def api_update_lar_main(client, data: LarMainInfoUpdateSchema, push_id: str, biz_id: str) -> None:
|
||||
"""
|
||||
PUT /lar/main - 更新主数据
|
||||
|
||||
Args:
|
||||
client: API 客户端
|
||||
data: 更新数据（Pydantic model）
|
||||
push_id: 推送ID
|
||||
biz_id: 业务ID
|
||||
"""
|
||||
request_data = {"push_id": push_id, "biz_id": biz_id, "data": data.model_dump(exclude_unset=True)}
|
||||
response = await client.request(method="PUT", endpoint="/lar/main", data=request_data)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
|
||||
|
||||
async def api_update_lar_resettlement(client, data: LarResettlementSchema, push_id: str, biz_id: str) -> None:
|
||||
"""
|
||||
PUT /lar/resettlement - 更新搬迁信息
|
||||
|
||||
Args:
|
||||
client: API 客户端
|
||||
data: 更新数据（Pydantic model）
|
||||
push_id: 推送ID
|
||||
biz_id: 业务ID
|
||||
"""
|
||||
request_data = {"push_id": push_id, "biz_id": biz_id, "data": data.model_dump(exclude_unset=True)}
|
||||
response = await client.request(method="PUT", endpoint="/lar/resettlement", data=request_data)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
|
||||
|
||||
async def api_update_lar_investment(client, data: LarInvestmentSchema, push_id: str, biz_id: str) -> None:
|
||||
"""
|
||||
PUT /lar/investment - 更新投资信息
|
||||
|
||||
Args:
|
||||
client: API 客户端
|
||||
data: 更新数据（Pydantic model）
|
||||
push_id: 推送ID
|
||||
biz_id: 业务ID
|
||||
"""
|
||||
request_data = {"push_id": push_id, "biz_id": biz_id, "data": data.model_dump(exclude_unset=True)}
|
||||
response = await client.request(method="PUT", endpoint="/lar/investment", data=request_data)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
@@ -0,0 +1,10 @@
|
||||
"""LAR Domain - JSONL Handler"""
|
||||
from ...datasource import BaseJsonlHandler, JsonlDataSource
|
||||
from .sync_node import LarSyncNode
|
||||
from schemas.project_extensions.lar import LarDetailSchema
|
||||
|
||||
|
||||
class LarJsonlHandler(BaseJsonlHandler):
|
||||
"""移民征地 JSONL Handler"""
|
||||
def __init__(self, datasource: JsonlDataSource):
|
||||
super().__init__(datasource, "lar", LarSyncNode, LarDetailSchema)
|
||||
@@ -0,0 +1,17 @@
|
||||
"""LAR domain registration"""
|
||||
from ...common.registry import DomainRegistry
|
||||
from schemas.project_extensions.lar import LarDetailSchema
|
||||
from .sync_node import LarSyncNode
|
||||
from .sync_strategy import LarSyncStrategy
|
||||
from .jsonl_handler import LarJsonlHandler
|
||||
from .api_handler import LarApiHandler
|
||||
|
||||
# 注册 lar domain
|
||||
DomainRegistry.register(
|
||||
node_type="lar",
|
||||
schema=LarDetailSchema,
|
||||
node_class=LarSyncNode,
|
||||
strategy_class=LarSyncStrategy,
|
||||
jsonl_handler_class=LarJsonlHandler,
|
||||
api_handler_class=LarApiHandler,
|
||||
)
|
||||
@@ -0,0 +1,14 @@
|
||||
"""LAR SyncNode"""
|
||||
|
||||
from typing import Dict, Any
|
||||
from ...common.sync_node import SyncNode
|
||||
from schemas.project_extensions.lar import LarDetailSchema
|
||||
|
||||
|
||||
class LarSyncNode(SyncNode[LarDetailSchema]):
|
||||
"""移民征地同步节点"""
|
||||
node_type = "lar"
|
||||
schema = LarDetailSchema
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
@@ -0,0 +1,20 @@
|
||||
from ...sync_system.strategy import DefaultSyncStrategy
|
||||
from ...sync_system.config import StrategyConfig, OrphanAction, UpdateDirection
|
||||
from schemas.project_extensions.lar import LarDetailSchema
|
||||
|
||||
|
||||
class LarSyncStrategy(DefaultSyncStrategy[LarDetailSchema]):
|
||||
"""
|
||||
LAR 同步策略（项目扩展）。
|
||||
"""
|
||||
|
||||
schema = LarDetailSchema
|
||||
|
||||
default_config = StrategyConfig(
|
||||
auto_bind=True,
|
||||
auto_bind_fields=["project_id"],
|
||||
depend_fields={"project_id": "project"},
|
||||
local_orphan_action=OrphanAction.NONE,
|
||||
remote_orphan_action=OrphanAction.NONE,
|
||||
update_direction=UpdateDirection.PUSH,
|
||||
)
|
||||
@@ -0,0 +1,8 @@
|
||||
"""LAR Change (移民变更) Domain Module"""
|
||||
|
||||
from .sync_node import LarChangeSyncNode
|
||||
from .sync_strategy import LarChangeSyncStrategy
|
||||
from .api_handler import LarChangeApiHandler
|
||||
from .jsonl_handler import LarChangeJsonlHandler
|
||||
|
||||
__all__ = ["LarChangeSyncNode", "LarChangeSyncStrategy", "LarChangeApiHandler", "LarChangeJsonlHandler"]
|
||||
@@ -0,0 +1,180 @@
|
||||
"""
|
||||
LAR Change (移民变更) API Handler
|
||||
|
||||
职责：
|
||||
- 创建和更新移民变更记录（依赖 Project）
|
||||
|
||||
实现的接口：
|
||||
POST:
|
||||
- /lar/change - 创建移民变更
|
||||
PUT:
|
||||
- /lar/change - 更新移民变更
|
||||
"""
|
||||
|
||||
from typing import List, Dict, Any
|
||||
from ...datasource.api.handler import BaseApiHandler
|
||||
from ...datasource.task_result import TaskResult
|
||||
from ...common.sync_node import SyncNode
|
||||
from ...datasource.api.errors import ERROR_NODE_DATA_NONE, ERROR_VALIDATION_FAILED
|
||||
from .sync_node import LarChangeSyncNode
|
||||
from schemas.project_extensions.lar import (
|
||||
LarChangeDetailSchema,
|
||||
LarChangeCreateSchema,
|
||||
LarChangeUpdateSchema
|
||||
)
|
||||
|
||||
|
||||
class LarChangeApiHandler(BaseApiHandler):
|
||||
"""移民变更 API Handler"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._node_type = "lar_change"
|
||||
self._node_class = LarChangeSyncNode
|
||||
self._schema = LarChangeDetailSchema
|
||||
|
||||
async def load(self) -> List[SyncNode]:
|
||||
"""
|
||||
加载移民变更数据（依赖 Project）
|
||||
"""
|
||||
projects = self._collection.filter(node_type="project")
|
||||
project_ids = [p.data_id for p in projects if p.data_id]
|
||||
|
||||
if not project_ids:
|
||||
return []
|
||||
|
||||
# 移民变更通常需要单独查询，这里暂时返回空
|
||||
return []
|
||||
|
||||
async def create_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""批量创建移民变更"""
|
||||
from pydantic import ValidationError
|
||||
|
||||
results = []
|
||||
for node in nodes:
|
||||
node_data = node.get_data()
|
||||
if node_data is None:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=ERROR_NODE_DATA_NONE))
|
||||
continue
|
||||
|
||||
# 验证并转换为 Pydantic model
|
||||
try:
|
||||
create_model = LarChangeCreateSchema.model_validate(node_data)
|
||||
except ValidationError as e:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=f"{ERROR_VALIDATION_FAILED}: {e}"))
|
||||
continue
|
||||
|
||||
push_id = self._generate_push_id()
|
||||
|
||||
try:
|
||||
response = await api_create_lar_change(self.api_client, create_model, push_id)
|
||||
if self.poll_mode == "sync":
|
||||
created_id = self._extract_created_id_from_response(response)
|
||||
if not created_id:
|
||||
results.append(
|
||||
TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error="Missing created_id in create response"
|
||||
)
|
||||
)
|
||||
continue
|
||||
results.append(TaskResult.success(node_id=node.node_id, data_id=created_id))
|
||||
else:
|
||||
results.append(TaskResult.in_progress(node_id=node.node_id, task_id=push_id))
|
||||
except Exception as e:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=str(e)))
|
||||
|
||||
return results
|
||||
|
||||
async def update_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""批量更新移民变更"""
|
||||
from pydantic import ValidationError
|
||||
|
||||
results = []
|
||||
for node in nodes:
|
||||
node_data = node.get_data()
|
||||
if node_data is None:
|
||||
error_msg = node.error or "Node data is missing or invalid"
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=f"Cannot update: {error_msg}"))
|
||||
continue
|
||||
origin_data = node.get_origin_data() or {}
|
||||
update_data = self._filter_update_data(node_data, origin_data, LarChangeUpdateSchema)
|
||||
if not update_data:
|
||||
from ...common.types import SyncAction
|
||||
results.append(TaskResult.skipped(node_id=node.node_id, reason="No updatable fields changed"))
|
||||
continue
|
||||
|
||||
# 验证并转换为 Pydantic model
|
||||
try:
|
||||
update_model = LarChangeUpdateSchema.model_validate(update_data)
|
||||
except ValidationError as e:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=f"{ERROR_VALIDATION_FAILED}: {e}"))
|
||||
continue
|
||||
|
||||
push_id = self._generate_push_id()
|
||||
biz_id = node.data_id or update_model.id
|
||||
if not biz_id:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error="Missing biz_id for update"))
|
||||
continue
|
||||
|
||||
try:
|
||||
await api_update_lar_change(self.api_client, update_model, push_id, biz_id)
|
||||
results.append(TaskResult.success(node_id=node.node_id))
|
||||
except Exception as e:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=str(e)))
|
||||
|
||||
return results
|
||||
|
||||
async def delete_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""不支持删除"""
|
||||
return [
|
||||
TaskResult.failed(node_id=node.node_id, error="LAR change deletion not supported")
|
||||
for node in nodes
|
||||
]
|
||||
|
||||
async def poll_tasks(self, task_ids: List[str]) -> Dict[str, TaskResult]:
|
||||
"""轮询异步任务状态"""
|
||||
return await super().poll_tasks(task_ids)
|
||||
|
||||
def _create_node(self, data: Dict[str, Any]) -> SyncNode:
|
||||
node = self._create_basic_node(data, depend_ids=[])
|
||||
|
||||
if data.get("project_id"):
|
||||
node.context["project_id"] = data["project_id"]
|
||||
|
||||
return node
|
||||
|
||||
|
||||
# ========== 静态 API 函数 ==========
|
||||
|
||||
async def api_create_lar_change(client, data: LarChangeCreateSchema, push_id: str) -> Dict[str, Any]:
|
||||
"""
|
||||
POST /lar/change - 创建移民变更
|
||||
|
||||
Args:
|
||||
client: API 客户端
|
||||
data: 创建数据（Pydantic model）
|
||||
push_id: 推送ID
|
||||
biz_id: 业务ID
|
||||
"""
|
||||
request_data = {"push_id": push_id, "data": data.model_dump()}
|
||||
response = await client.request(method="POST", endpoint="/lar/change", data=request_data)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
return response
|
||||
|
||||
|
||||
async def api_update_lar_change(client, data: LarChangeUpdateSchema, push_id: str, biz_id: str) -> None:
|
||||
"""
|
||||
PUT /lar/change - 更新移民变更
|
||||
|
||||
Args:
|
||||
client: API 客户端
|
||||
data: 更新数据（Pydantic model）
|
||||
push_id: 推送ID
|
||||
biz_id: 业务ID
|
||||
"""
|
||||
request_data = {"push_id": push_id, "biz_id": biz_id, "data": data.model_dump(exclude_unset=True)}
|
||||
response = await client.request(method="PUT", endpoint="/lar/change", data=request_data)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
@@ -0,0 +1,12 @@
|
||||
"""LAR Change JSONL Data Handler"""
|
||||
from ...datasource.jsonl.handler import BaseJsonlHandler
|
||||
from ...datasource.jsonl.datasource import JsonlDataSource
|
||||
from .sync_node import LarChangeSyncNode
|
||||
from schemas.project_extensions.lar import LarChangeDetailSchema
|
||||
|
||||
|
||||
class LarChangeJsonlHandler(BaseJsonlHandler):
|
||||
"""LAR Change (移民变更) JSONL 数据处理器"""
|
||||
|
||||
def __init__(self, datasource: JsonlDataSource):
|
||||
super().__init__(datasource, "lar_change", LarChangeSyncNode, LarChangeDetailSchema)
|
||||
@@ -0,0 +1,17 @@
|
||||
"""LAR change domain registration"""
|
||||
from ...common.registry import DomainRegistry
|
||||
from schemas.project_extensions.lar import LarChangeDetailSchema
|
||||
from .sync_node import LarChangeSyncNode
|
||||
from .sync_strategy import LarChangeSyncStrategy
|
||||
from .api_handler import LarChangeApiHandler
|
||||
from .jsonl_handler import LarChangeJsonlHandler
|
||||
|
||||
# 注册 lar_change domain
|
||||
DomainRegistry.register(
|
||||
node_type="lar_change",
|
||||
schema=LarChangeDetailSchema,
|
||||
node_class=LarChangeSyncNode,
|
||||
strategy_class=LarChangeSyncStrategy,
|
||||
api_handler_class=LarChangeApiHandler,
|
||||
jsonl_handler_class=LarChangeJsonlHandler,
|
||||
)
|
||||
@@ -0,0 +1,10 @@
|
||||
"""LAR Change SyncNode"""
|
||||
|
||||
from ...common.sync_node import SyncNode
|
||||
from schemas.project_extensions.lar import LarChangeDetailSchema
|
||||
|
||||
|
||||
class LarChangeSyncNode(SyncNode[LarChangeDetailSchema]):
|
||||
"""移民变更同步节点"""
|
||||
node_type = "lar_change"
|
||||
schema = LarChangeDetailSchema
|
||||
@@ -0,0 +1,20 @@
|
||||
from ...sync_system.strategy import DefaultSyncStrategy
|
||||
from ...sync_system.config import StrategyConfig, OrphanAction, UpdateDirection
|
||||
from schemas.project_extensions.lar import LarChangeDetailSchema
|
||||
|
||||
|
||||
class LarChangeSyncStrategy(DefaultSyncStrategy[LarChangeDetailSchema]):
|
||||
"""
|
||||
LAR Change 同步策略。
|
||||
"""
|
||||
|
||||
schema = LarChangeDetailSchema
|
||||
|
||||
default_config = StrategyConfig(
|
||||
auto_bind=True,
|
||||
auto_bind_fields=["project_id", "change_time", "title"],
|
||||
depend_fields={"project_id": "project"},
|
||||
local_orphan_action=OrphanAction.NONE,
|
||||
remote_orphan_action=OrphanAction.NONE,
|
||||
update_direction=UpdateDirection.PUSH,
|
||||
)
|
||||
@@ -0,0 +1,6 @@
|
||||
"""Material domain"""
|
||||
from .sync_node import MaterialSyncNode
|
||||
from .sync_strategy import MaterialSyncStrategy
|
||||
from .jsonl_handler import MaterialJsonlHandler
|
||||
|
||||
__all__ = ["MaterialSyncNode", "MaterialSyncStrategy", "MaterialJsonlHandler"]
|
||||
@@ -0,0 +1,355 @@
|
||||
"""
|
||||
Material API Handler (V2)
|
||||
|
||||
职责：
|
||||
- 加载物资列表（依赖 Project）
|
||||
- 物资创建/更新/删除（批量接口）
|
||||
"""
|
||||
|
||||
from typing import List, Dict, Any
|
||||
from ...datasource.api.handler import BaseApiHandler
|
||||
from ...datasource.task_result import TaskResult
|
||||
from ...common.sync_node import SyncNode
|
||||
from ...datasource.api.errors import (
|
||||
ERROR_NODE_DATA_NONE,
|
||||
ERROR_VALIDATION_FAILED,
|
||||
ERROR_PROJECT_ID_NOT_FOUND
|
||||
)
|
||||
from .sync_node import MaterialSyncNode
|
||||
from schemas.material.material import MaterialResponse, MaterialCreate, MaterialUpdate, MaterialPlanUpdate
|
||||
|
||||
|
||||
class MaterialApiHandler(BaseApiHandler):
|
||||
"""物资 API Handler"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._node_type = "material"
|
||||
self._node_class = MaterialSyncNode
|
||||
self._schema = MaterialResponse
|
||||
self.update_schemas = [MaterialCreate, MaterialUpdate, MaterialPlanUpdate]
|
||||
|
||||
async def load(self) -> List[SyncNode]:
|
||||
"""
|
||||
加载物资列表
|
||||
|
||||
依赖 Project
|
||||
使用 GET /material/list?project_id={id} 接口
|
||||
"""
|
||||
try:
|
||||
projects = self._collection.filter(node_type="project")
|
||||
project_ids = [p.data_id for p in projects if p.data_id]
|
||||
|
||||
if not project_ids:
|
||||
print("⚠️ No projects found, skipping material load")
|
||||
return []
|
||||
|
||||
all_materials = []
|
||||
for project_id in project_ids:
|
||||
try:
|
||||
materials = await api_get_material_list(self.api_client, project_id)
|
||||
|
||||
for material in materials:
|
||||
# 已验证的 Pydantic model，转换为 dict
|
||||
material_data = material.model_dump()
|
||||
all_materials.append(self._create_node(material_data))
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Failed to load materials for project {project_id}: {e}")
|
||||
continue
|
||||
|
||||
return all_materials
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Failed to load materials: {e}")
|
||||
return []
|
||||
|
||||
async def create_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""批量创建物资"""
|
||||
results = []
|
||||
for node in nodes:
|
||||
result = await self._create_single(node)
|
||||
results.append(result)
|
||||
return results
|
||||
|
||||
async def update_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""
|
||||
批量更新物资
|
||||
|
||||
支持两种更新类型：
|
||||
1. MaterialUpdate - 基础信息更新
|
||||
2. MaterialPlanUpdate - 计划信息更新
|
||||
|
||||
更新逻辑：
|
||||
- 对每种 schema，检查是否有差异
|
||||
- 如果有差异，调用对应 API
|
||||
"""
|
||||
from pydantic import ValidationError
|
||||
|
||||
results = []
|
||||
for node in nodes:
|
||||
data = node.get_data()
|
||||
if not data:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=ERROR_NODE_DATA_NONE))
|
||||
continue
|
||||
|
||||
origin_data = node.get_origin_data() or {}
|
||||
biz_id = node.data_id or data.get("id")
|
||||
if not biz_id:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error="Missing biz_id for update"))
|
||||
continue
|
||||
|
||||
node_updated = False
|
||||
error = None
|
||||
|
||||
# 1. 基础信息更新 (MaterialUpdate)
|
||||
base_diff = self._get_schema_diff(MaterialUpdate, data, origin_data, node.node_id)
|
||||
if base_diff:
|
||||
try:
|
||||
update_model = MaterialUpdate.model_validate(data)
|
||||
push_id = self._generate_push_id()
|
||||
await api_update_material(self.api_client, update_model, push_id, biz_id)
|
||||
node_updated = True
|
||||
except ValidationError as e:
|
||||
error = f"{ERROR_VALIDATION_FAILED}: {e}"
|
||||
|
||||
# 2. 计划信息更新 (MaterialPlanUpdate)
|
||||
plan_diff = self._get_schema_diff(MaterialPlanUpdate, data, origin_data, node.node_id)
|
||||
if not error and plan_diff:
|
||||
try:
|
||||
plan_model = MaterialPlanUpdate.model_validate(data)
|
||||
push_id = self._generate_push_id()
|
||||
# 需要调用对应的计划更新 API
|
||||
await api_update_material_plan(
|
||||
self.api_client,
|
||||
plan_model.model_dump(exclude_unset=True),
|
||||
push_id,
|
||||
biz_id
|
||||
)
|
||||
node_updated = True
|
||||
except ValidationError as e:
|
||||
error = f"{ERROR_VALIDATION_FAILED}: {e}"
|
||||
|
||||
# 汇总结果
|
||||
if error:
|
||||
results.append(TaskResult.failed(node_id=node.node_id, error=error))
|
||||
elif node_updated:
|
||||
results.append(TaskResult.success(node_id=node.node_id))
|
||||
else:
|
||||
# 没有任何字段需要更新
|
||||
results.append(TaskResult.skipped(node_id=node.node_id, reason="No updatable fields changed"))
|
||||
|
||||
return results
|
||||
|
||||
async def delete_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
|
||||
"""批量删除物资"""
|
||||
results = []
|
||||
for node in nodes:
|
||||
result = await self._delete_single(node)
|
||||
results.append(result)
|
||||
return results
|
||||
|
||||
async def _create_single(self, node: SyncNode) -> TaskResult:
|
||||
"""创建物资"""
|
||||
try:
|
||||
data = node.get_data()
|
||||
if not data:
|
||||
return TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error=ERROR_NODE_DATA_NONE
|
||||
)
|
||||
|
||||
push_id = self._generate_push_id()
|
||||
|
||||
project_id = node.context.get("project_id")
|
||||
if not project_id:
|
||||
project_id = data.get("project_id")
|
||||
|
||||
if not project_id:
|
||||
return TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error=ERROR_PROJECT_ID_NOT_FOUND
|
||||
)
|
||||
|
||||
# 验证并转换为 Pydantic model
|
||||
from pydantic import ValidationError
|
||||
create_data = {"project_id": project_id, **data}
|
||||
try:
|
||||
create_model = MaterialCreate.model_validate(create_data)
|
||||
except ValidationError as e:
|
||||
return TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error=f"{ERROR_VALIDATION_FAILED}: {e}"
|
||||
)
|
||||
|
||||
response = await api_create_material(self.api_client, create_model, push_id)
|
||||
if self.poll_mode == "sync":
|
||||
created_id = self._extract_created_id_from_response(response)
|
||||
if not created_id:
|
||||
return TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error="Missing created_id in create response"
|
||||
)
|
||||
return TaskResult.success(node_id=node.node_id, data_id=created_id)
|
||||
else:
|
||||
return TaskResult.in_progress(node_id=node.node_id, task_id=push_id)
|
||||
|
||||
except Exception as e:
|
||||
return TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error=f"Create failed: {str(e)}"
|
||||
)
|
||||
|
||||
async def _update_single(self, node: SyncNode) -> TaskResult:
|
||||
"""更新物资"""
|
||||
if not self.api_client:
|
||||
return TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error="api_client not initialized"
|
||||
)
|
||||
|
||||
try:
|
||||
data = node.get_data()
|
||||
if not data:
|
||||
return TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error="Node data is None or validation failed"
|
||||
)
|
||||
|
||||
push_id = self._generate_push_id()
|
||||
biz_id = node.data_id or data.get("id")
|
||||
if not biz_id:
|
||||
return TaskResult.failed(node_id=node.node_id, error="Missing biz_id for update")
|
||||
|
||||
# 验证并转换为 Pydantic model
|
||||
from pydantic import ValidationError
|
||||
try:
|
||||
update_model = MaterialUpdate.model_validate(data)
|
||||
except ValidationError as e:
|
||||
return TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error=f"{ERROR_VALIDATION_FAILED}: {e}"
|
||||
)
|
||||
|
||||
await api_update_material(self.api_client, update_model, push_id, biz_id)
|
||||
return TaskResult.success(node_id=node.node_id)
|
||||
|
||||
except Exception as e:
|
||||
return TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error=f"Update failed: {str(e)}"
|
||||
)
|
||||
|
||||
async def _delete_single(self, node: SyncNode) -> TaskResult:
|
||||
"""删除物资"""
|
||||
try:
|
||||
push_id = self._generate_push_id()
|
||||
biz_id = node.data_id
|
||||
if not biz_id:
|
||||
return TaskResult.failed(node_id=node.node_id, error="Missing biz_id for delete")
|
||||
|
||||
request_data = {
|
||||
"push_id": push_id,
|
||||
"biz_id": biz_id,
|
||||
}
|
||||
|
||||
response = await self.api_client.request(
|
||||
method="DELETE",
|
||||
endpoint="/material/delete",
|
||||
data=request_data
|
||||
)
|
||||
|
||||
if response.get("code") == 200:
|
||||
return TaskResult.success(node_id=node.node_id)
|
||||
else:
|
||||
return TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error=f"API returned error: {response.get('message', 'Unknown error')}"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
return TaskResult.failed(
|
||||
node_id=node.node_id,
|
||||
error=f"Delete failed: {str(e)}"
|
||||
)
|
||||
|
||||
def _create_node(self, data: Dict[str, Any]) -> SyncNode:
|
||||
"""
|
||||
创建物资同步节点
|
||||
|
||||
设置 project_id 和 material_id 到 context
|
||||
"""
|
||||
node = self._create_basic_node(data, depend_ids=[])
|
||||
|
||||
if data.get("project_id"):
|
||||
node.context["project_id"] = data["project_id"]
|
||||
node.context["material_id"] = node.data_id
|
||||
|
||||
return node
|
||||
|
||||
|
||||
# ========== 静态 API 函数 ==========
|
||||
|
||||
async def api_get_material(client, material_id: str) -> Dict[str, Any]:
|
||||
"""获取单个物资数据 GET /material"""
|
||||
response = await client.request(
|
||||
method="GET",
|
||||
endpoint="/material",
|
||||
params={"material_id": material_id}
|
||||
)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
return response.get("data", {})
|
||||
|
||||
|
||||
# ========== 静态 API 函数 ==========
|
||||
|
||||
async def api_get_material_list(client, project_id: str) -> List[MaterialResponse]:
|
||||
"""GET /material/list - 获取物资列表"""
|
||||
response = await client.request(
|
||||
method="GET",
|
||||
endpoint="/material/list",
|
||||
params={"project_id": project_id}
|
||||
)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
|
||||
items = response.get("data", [])
|
||||
|
||||
# 验证每个返回项
|
||||
from pydantic import ValidationError
|
||||
validated_items = []
|
||||
for item in items:
|
||||
try:
|
||||
validated_item = MaterialResponse.model_validate(item)
|
||||
validated_items.append(validated_item)
|
||||
except ValidationError as e:
|
||||
print(f"⚠️ Skipping invalid material item: {e}")
|
||||
continue
|
||||
|
||||
return validated_items
|
||||
|
||||
|
||||
async def api_create_material(client, data: MaterialCreate, push_id: str) -> Dict[str, Any]:
|
||||
"""创建物资 POST /material/create"""
|
||||
request_data = {"push_id": push_id, "data": data.model_dump()}
|
||||
response = await client.request(method="POST", endpoint="/material/create", data=request_data)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
return response
|
||||
|
||||
|
||||
async def api_update_material(client, data: MaterialUpdate, push_id: str, biz_id: str) -> None:
|
||||
"""更新物资 PUT /material/update"""
|
||||
request_data = {"push_id": push_id, "biz_id": biz_id, "data": data.model_dump(exclude_unset=True)}
|
||||
response = await client.request(method="PUT", endpoint="/material/update", data=request_data)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
|
||||
|
||||
async def api_update_material_plan(client, data: Dict[str, Any], push_id: str, biz_id: str) -> None:
|
||||
"""更新物资计划 PUT /material/plan"""
|
||||
request_data = {"push_id": push_id, "biz_id": biz_id, "data": data}
|
||||
response = await client.request(method="PUT", endpoint="/material/plan", data=request_data)
|
||||
if response.get("code") != 200:
|
||||
raise Exception(f"API error: {response.get('message', 'Unknown error')}")
|
||||
@@ -0,0 +1,10 @@
|
||||
"""Material Domain - JSONL Handler"""
|
||||
from ...datasource import BaseJsonlHandler, JsonlDataSource
|
||||
from .sync_node import MaterialSyncNode
|
||||
from schemas.material.material import MaterialResponse
|
||||
|
||||
|
||||
class MaterialJsonlHandler(BaseJsonlHandler):
|
||||
"""物资 JSONL Handler"""
|
||||
def __init__(self, datasource: JsonlDataSource):
|
||||
super().__init__(datasource, "material", MaterialSyncNode, MaterialResponse)
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Material domain registration"""
|
||||
from ...common.registry import DomainRegistry
|
||||
from schemas.material.material import MaterialResponse
|
||||
from .sync_node import MaterialSyncNode
|
||||
from .sync_strategy import MaterialSyncStrategy
|
||||
from .jsonl_handler import MaterialJsonlHandler
|
||||
from .api_handler import MaterialApiHandler
|
||||
|
||||
# 注册 material domain
|
||||
DomainRegistry.register(
|
||||
node_type="material",
|
||||
schema=MaterialResponse,
|
||||
node_class=MaterialSyncNode,
|
||||
strategy_class=MaterialSyncStrategy,
|
||||
jsonl_handler_class=MaterialJsonlHandler,
|
||||
api_handler_class=MaterialApiHandler,
|
||||
)
|
||||
@@ -0,0 +1,11 @@
|
||||
from ...common.sync_node import SyncNode
|
||||
from schemas.material.material import MaterialResponse
|
||||
|
||||
|
||||
class MaterialSyncNode(SyncNode[MaterialResponse]):
|
||||
"""物资同步节点"""
|
||||
node_type = "material"
|
||||
schema = MaterialResponse
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
@@ -0,0 +1,20 @@
|
||||
from ...sync_system.strategy import DefaultSyncStrategy
|
||||
from ...sync_system.config import StrategyConfig, OrphanAction, UpdateDirection
|
||||
from schemas.material.material import MaterialResponse
|
||||
|
||||
|
||||
class MaterialSyncStrategy(DefaultSyncStrategy[MaterialResponse]):
|
||||
"""
|
||||
Material 同步策略（合同子业务）。
|
||||
"""
|
||||
|
||||
schema = MaterialResponse
|
||||
|
||||
default_config = StrategyConfig(
|
||||
auto_bind=True,
|
||||
auto_bind_fields=["contract_id", "material_type"],
|
||||
depend_fields={"contract_id": "contract", "project_id": "project"},
|
||||
local_orphan_action=OrphanAction.CREATE_REMOTE,
|
||||
remote_orphan_action=OrphanAction.NONE,
|
||||
update_direction=UpdateDirection.PUSH,
|
||||
)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user