优化sync_system代码质量
This commit is contained in:
@@ -362,7 +362,7 @@ class DataCollection:
|
||||
- data/origin_data 保留(由 DataSource 覆盖)
|
||||
|
||||
**初始化归并(E01)**:
|
||||
- 由 Pipeline 在加载后调用 run_reset() 触发 E01
|
||||
- 由 Pipeline 在加载后调用 `run_phase2_cleanup()` 触发 E01
|
||||
- CREATE 僵尸节点进入 S15 并删除
|
||||
- 其余节点进入 S00(UNCHECKED/NONE/PENDING)
|
||||
"""
|
||||
|
||||
@@ -9,7 +9,7 @@ from ..common.registry import DomainRegistry
|
||||
from .datasource_config import DataSourceConfig
|
||||
from .pipeline_config import PipelineRunConfig
|
||||
from .run_presets import load_overrides_from_file
|
||||
from .strategy_config import ConfigPresets, StrategyRuntimeConfig
|
||||
from .strategy_config import ConfigPresets, StrategyRuntimeConfig, resolve_domain_option_config
|
||||
|
||||
|
||||
def _deep_merge(base: Mapping[str, Any], override: Mapping[str, Any]) -> Dict[str, Any]:
|
||||
@@ -28,19 +28,76 @@ def _deep_merge(base: Mapping[str, Any], override: Mapping[str, Any]) -> Dict[st
|
||||
return merged
|
||||
|
||||
|
||||
def _build_default_strategy_runtime_config(node_type: str) -> StrategyRuntimeConfig | None:
|
||||
strategy_class = DomainRegistry.get_strategy_class(node_type)
|
||||
if strategy_class is None:
|
||||
return None
|
||||
|
||||
config = strategy_class.default_config.model_copy(deep=True)
|
||||
config.domain_option = resolve_domain_option_config(
|
||||
getattr(strategy_class, "domain_option_model", None),
|
||||
config.domain_option,
|
||||
)
|
||||
|
||||
return StrategyRuntimeConfig(
|
||||
node_type=node_type,
|
||||
config=config,
|
||||
)
|
||||
|
||||
|
||||
def _default_strategy_map(node_types: list[str]) -> Dict[str, StrategyRuntimeConfig]:
|
||||
strategy_map: Dict[str, StrategyRuntimeConfig] = {}
|
||||
for node_type in node_types:
|
||||
strategy_class = DomainRegistry.get_strategy_class(node_type)
|
||||
if strategy_class is None:
|
||||
runtime_cfg = _build_default_strategy_runtime_config(node_type)
|
||||
if runtime_cfg is None:
|
||||
continue
|
||||
strategy_map[node_type] = StrategyRuntimeConfig(
|
||||
node_type=node_type,
|
||||
config=strategy_class.default_config.model_copy(deep=True),
|
||||
)
|
||||
strategy_map[node_type] = runtime_cfg
|
||||
return strategy_map
|
||||
|
||||
|
||||
def _merge_strategy_runtime_config(
|
||||
runtime_cfg: StrategyRuntimeConfig,
|
||||
raw_override: Any,
|
||||
) -> StrategyRuntimeConfig:
|
||||
override = dict(raw_override or {})
|
||||
override.pop("node_type", None)
|
||||
|
||||
deprecated_keys = [key for key in ("force_sync", "load_mode") if key in override]
|
||||
if deprecated_keys:
|
||||
joined = ", ".join(deprecated_keys)
|
||||
raise ValueError(
|
||||
f"Strategy override for {runtime_cfg.node_type!r} contains unsupported legacy keys: {joined}. "
|
||||
"Use skip_sync under strategies.<node_type> or datasource.handler_configs instead."
|
||||
)
|
||||
|
||||
preset = override.pop("preset", None)
|
||||
if preset:
|
||||
runtime_cfg.config = runtime_cfg.config.model_validate(
|
||||
_deep_merge(
|
||||
runtime_cfg.config.model_dump(),
|
||||
ConfigPresets.get_preset(str(preset)).model_dump(exclude_defaults=True),
|
||||
)
|
||||
)
|
||||
|
||||
if "config" in override and isinstance(override["config"], dict):
|
||||
runtime_cfg.config = runtime_cfg.config.model_validate(
|
||||
_deep_merge(runtime_cfg.config.model_dump(), override.pop("config"))
|
||||
)
|
||||
|
||||
if override:
|
||||
runtime_cfg.config = runtime_cfg.config.model_validate(
|
||||
_deep_merge(runtime_cfg.config.model_dump(), override)
|
||||
)
|
||||
|
||||
strategy_class = DomainRegistry.get_strategy_class(runtime_cfg.node_type)
|
||||
runtime_cfg.config.domain_option = resolve_domain_option_config(
|
||||
getattr(strategy_class, "domain_option_model", None) if strategy_class is not None else None,
|
||||
runtime_cfg.config.domain_option,
|
||||
)
|
||||
|
||||
return runtime_cfg
|
||||
|
||||
|
||||
def _apply_strategy_overrides(
|
||||
node_types: list[str],
|
||||
strategy_overrides: Any,
|
||||
@@ -49,8 +106,16 @@ def _apply_strategy_overrides(
|
||||
|
||||
if isinstance(strategy_overrides, list):
|
||||
for item in strategy_overrides:
|
||||
runtime_cfg = StrategyRuntimeConfig.model_validate(item)
|
||||
strategy_map[runtime_cfg.node_type] = runtime_cfg
|
||||
raw_item = dict(item or {})
|
||||
node_type = str(raw_item.get("node_type") or "")
|
||||
if not node_type:
|
||||
raise ValueError("Strategy override list item missing required node_type")
|
||||
|
||||
runtime_cfg = strategy_map.get(node_type)
|
||||
if runtime_cfg is None:
|
||||
runtime_cfg = _build_default_strategy_runtime_config(node_type) or StrategyRuntimeConfig(node_type=node_type)
|
||||
|
||||
strategy_map[node_type] = _merge_strategy_runtime_config(runtime_cfg, raw_item)
|
||||
return [strategy_map[n] for n in node_types if n in strategy_map]
|
||||
|
||||
if not isinstance(strategy_overrides, dict):
|
||||
@@ -58,41 +123,10 @@ def _apply_strategy_overrides(
|
||||
|
||||
for node_type, raw_override in strategy_overrides.items():
|
||||
if node_type not in strategy_map:
|
||||
strategy_map[node_type] = StrategyRuntimeConfig(node_type=node_type)
|
||||
strategy_map[node_type] = _build_default_strategy_runtime_config(node_type) or StrategyRuntimeConfig(node_type=node_type)
|
||||
|
||||
runtime_cfg = strategy_map[node_type]
|
||||
override = dict(raw_override or {})
|
||||
|
||||
deprecated_keys = [key for key in ("force_sync", "load_mode") if key in override]
|
||||
if deprecated_keys:
|
||||
joined = ", ".join(deprecated_keys)
|
||||
raise ValueError(
|
||||
f"Strategy override for {node_type!r} contains unsupported legacy keys: {joined}. "
|
||||
"Use skip_sync under strategies.<node_type> or datasource.handler_configs instead."
|
||||
)
|
||||
|
||||
preset = override.pop("preset", None)
|
||||
if preset:
|
||||
runtime_cfg.config = ConfigPresets.get_preset(str(preset))
|
||||
|
||||
skip_sync = override.pop("skip_sync", None)
|
||||
skip_post_check = override.pop("skip_post_check", None)
|
||||
|
||||
if skip_sync is not None:
|
||||
runtime_cfg.skip_sync = bool(skip_sync)
|
||||
|
||||
if skip_post_check is not None:
|
||||
runtime_cfg.skip_post_check = bool(skip_post_check)
|
||||
|
||||
if "config" in override and isinstance(override["config"], dict):
|
||||
runtime_cfg.config = runtime_cfg.config.model_validate(
|
||||
_deep_merge(runtime_cfg.config.model_dump(), override.pop("config"))
|
||||
)
|
||||
|
||||
if override:
|
||||
runtime_cfg.config = runtime_cfg.config.model_validate(
|
||||
_deep_merge(runtime_cfg.config.model_dump(), override)
|
||||
)
|
||||
strategy_map[node_type] = _merge_strategy_runtime_config(runtime_cfg, raw_override)
|
||||
|
||||
return [strategy_map[n] for n in node_types if n in strategy_map]
|
||||
|
||||
|
||||
@@ -1,11 +1,21 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from enum import Enum
|
||||
from typing import Dict, List, Optional
|
||||
from typing import Any, Dict, List, Mapping, Type
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
|
||||
def resolve_domain_option_config(
|
||||
domain_option_model: Type[BaseModel] | None,
|
||||
domain_option: Mapping[str, Any] | None,
|
||||
) -> Dict[str, Any]:
|
||||
if domain_option_model is None:
|
||||
return dict(domain_option or {})
|
||||
return domain_option_model.model_validate(domain_option or {}).model_dump()
|
||||
|
||||
|
||||
class OrphanAction(str, Enum):
|
||||
CREATE_REMOTE = "create_remote"
|
||||
CREATE_LOCAL = "create_local"
|
||||
@@ -17,7 +27,6 @@ class OrphanAction(str, Enum):
|
||||
class UpdateDirection(str, Enum):
|
||||
PUSH = "push"
|
||||
PULL = "pull"
|
||||
BIDIRECTIONAL = "bidirectional"
|
||||
NONE = "none"
|
||||
|
||||
|
||||
@@ -40,13 +49,6 @@ class StrategyConfig(BaseModel):
|
||||
remote_orphan_action: OrphanAction = OrphanAction.NONE
|
||||
update_direction: UpdateDirection = UpdateDirection.PUSH
|
||||
|
||||
# 字段级同步方向覆盖。
|
||||
# field_direction_key: 节点 data 中用作查表键的字段名,如 "key"。
|
||||
# field_direction_overrides: 字段值 → UpdateDirection 的映射。
|
||||
# 未命中映射的节点沿用 update_direction 作为默认方向。
|
||||
field_direction_key: Optional[str] = None
|
||||
field_direction_overrides: Dict[str, UpdateDirection] = Field(default_factory=dict)
|
||||
|
||||
# post-check 一致性比较时忽略的顶层字段。
|
||||
# 只影响最终一致性评估,不影响 create/update 的差异检测。
|
||||
post_check_ignore_fields: List[str] = Field(default_factory=list)
|
||||
@@ -63,6 +65,60 @@ class StrategyConfig(BaseModel):
|
||||
# 默认关闭,避免在一般场景下引入隐式匹配。
|
||||
allow_many_to_one_auto_bind: bool = False
|
||||
|
||||
# 是否跳过该类型的 create/update 阶段。
|
||||
skip_sync: bool = False
|
||||
|
||||
# 是否跳过该类型的 post-check reload 与一致性校验。
|
||||
skip_post_check: bool = False
|
||||
|
||||
# 领域策略运行参数,由各 strategy 的 domain_option_model 负责解释与校验。
|
||||
domain_option: Dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
def log_logic_warnings(self, *, node_type: str, logger: logging.Logger) -> None:
|
||||
direction = self.update_direction
|
||||
local_action = self.local_orphan_action
|
||||
remote_action = self.remote_orphan_action
|
||||
|
||||
if direction == UpdateDirection.PUSH:
|
||||
if local_action == OrphanAction.CREATE_LOCAL:
|
||||
logger.warning(
|
||||
f"[{node_type}] Config inconsistency: "
|
||||
f"update_direction=PUSH but local_orphan_action=CREATE_LOCAL (本地创建)。"
|
||||
f"应使用 CREATE_REMOTE (推送到远程)"
|
||||
)
|
||||
if remote_action == OrphanAction.CREATE_LOCAL:
|
||||
logger.warning(
|
||||
f"[{node_type}] Config inconsistency: "
|
||||
f"update_direction=PUSH but remote_orphan_action=CREATE_LOCAL (本地创建)。"
|
||||
f"PUSH 模式应设为 NONE 或 DELETE_REMOTE"
|
||||
)
|
||||
elif direction == UpdateDirection.PULL:
|
||||
if local_action == OrphanAction.CREATE_REMOTE:
|
||||
logger.warning(
|
||||
f"[{node_type}] Config inconsistency: "
|
||||
f"update_direction=PULL but local_orphan_action=CREATE_REMOTE (推送到远程)。"
|
||||
f"PULL 模式应设为 NONE 或 DELETE_LOCAL"
|
||||
)
|
||||
if remote_action in {OrphanAction.CREATE_REMOTE, OrphanAction.DELETE_LOCAL}:
|
||||
logger.warning(
|
||||
f"[{node_type}] Config inconsistency: "
|
||||
f"update_direction=PULL but remote_orphan_action={remote_action}。"
|
||||
f"PULL 模式建议使用 CREATE_LOCAL 或 NONE"
|
||||
)
|
||||
elif direction == UpdateDirection.NONE:
|
||||
if local_action != OrphanAction.NONE:
|
||||
logger.warning(
|
||||
f"[{node_type}] Config inconsistency: "
|
||||
f"update_direction=NONE but local_orphan_action={local_action}。"
|
||||
f"应设为 NONE"
|
||||
)
|
||||
if remote_action != OrphanAction.NONE:
|
||||
logger.warning(
|
||||
f"[{node_type}] Config inconsistency: "
|
||||
f"update_direction=NONE but remote_orphan_action={remote_action}。"
|
||||
f"应设为 NONE"
|
||||
)
|
||||
|
||||
|
||||
class StrategyRuntimeConfig(BaseModel):
|
||||
"""Pipeline 层策略运行配置(按 node_type)。"""
|
||||
@@ -70,8 +126,6 @@ class StrategyRuntimeConfig(BaseModel):
|
||||
model_config = ConfigDict(validate_assignment=True, extra="forbid")
|
||||
|
||||
node_type: str
|
||||
skip_sync: bool = False
|
||||
skip_post_check: bool = False
|
||||
config: StrategyConfig = Field(default_factory=StrategyConfig)
|
||||
|
||||
|
||||
|
||||
@@ -33,6 +33,7 @@ def _build_default_push_steps() -> List[Dict[str, str]]:
|
||||
class ApiClient:
|
||||
"""API HTTP 客户端"""
|
||||
_TRACE_TEXT_MAX_LEN = 1000
|
||||
_REQUEST_STATS_LOG_PREFIX = "📊 ApiClient request stats"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -72,6 +73,9 @@ class ApiClient:
|
||||
self._rate_limit_window_seconds = float(rate_limit_window_seconds)
|
||||
self._rate_limit_lock = asyncio.Lock()
|
||||
self._rate_limit_timestamps: deque[float] = deque()
|
||||
self._request_total = 0
|
||||
self._request_success = 0
|
||||
self._request_failure = 0
|
||||
|
||||
def _is_rate_limit_enabled(self) -> bool:
|
||||
return self._rate_limit_max_requests > 0 and self._rate_limit_window_seconds > 0
|
||||
@@ -105,9 +109,38 @@ class ApiClient:
|
||||
|
||||
async def close(self):
|
||||
"""关闭 HTTP 客户端"""
|
||||
self.log_request_stats()
|
||||
if self._client:
|
||||
await self._client.aclose()
|
||||
self._client = None
|
||||
|
||||
def get_request_stats(self) -> Dict[str, int]:
|
||||
"""返回当前客户端生命周期内的请求统计。"""
|
||||
return {
|
||||
"total": self._request_total,
|
||||
"success": self._request_success,
|
||||
"failure": self._request_failure,
|
||||
}
|
||||
|
||||
def format_request_stats(self) -> str:
|
||||
"""格式化当前客户端生命周期内的请求统计。"""
|
||||
stats = self.get_request_stats()
|
||||
return (
|
||||
f"{self._REQUEST_STATS_LOG_PREFIX}: "
|
||||
f"total={stats['total']}, success={stats['success']}, failure={stats['failure']}"
|
||||
)
|
||||
|
||||
def log_request_stats(self, level: str = "INFO") -> None:
|
||||
"""输出当前客户端生命周期内的请求统计。"""
|
||||
self._log(self.format_request_stats(), level=level)
|
||||
|
||||
def _log_request_summary(self, *, method: str, endpoint: str, elapsed_seconds: float) -> None:
|
||||
if not self._debug:
|
||||
return
|
||||
self._log(
|
||||
f"ApiClient request: method={method.upper()} endpoint={endpoint} elapsed={elapsed_seconds:.3f}s",
|
||||
level="DEBUG",
|
||||
)
|
||||
|
||||
def _log(self, message: str, level: str = "DEBUG"):
|
||||
"""
|
||||
@@ -382,6 +415,7 @@ class ApiClient:
|
||||
client = self._get_client()
|
||||
url = f"{self._base_url}{endpoint}"
|
||||
await self._acquire_rate_limit_slot()
|
||||
self._request_total += 1
|
||||
|
||||
# 记录请求信息
|
||||
self._log(f"\n{'='*60}", level="DEBUG")
|
||||
@@ -433,6 +467,7 @@ class ApiClient:
|
||||
|
||||
# 记录响应时间
|
||||
elapsed = time.time() - start_time
|
||||
self._log_request_summary(method=method, endpoint=endpoint, elapsed_seconds=elapsed)
|
||||
self._log(f"Response Time: {elapsed:.3f}s", level="DEBUG")
|
||||
|
||||
# 检查 HTTP 状态
|
||||
@@ -453,14 +488,17 @@ class ApiClient:
|
||||
response=result,
|
||||
elapsed_ms=int(elapsed * 1000),
|
||||
)
|
||||
self._request_success += 1
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
import traceback
|
||||
self._request_failure += 1
|
||||
|
||||
# 记录错误
|
||||
elapsed = time.time() - start_time
|
||||
self._log_request_summary(method=method, endpoint=endpoint, elapsed_seconds=elapsed)
|
||||
error_msg = "\n========== [ApiClient HTTP Error] ==========\n"
|
||||
error_msg += f"Request: {method} {url}\n"
|
||||
error_msg += f"Params: {params}\n"
|
||||
|
||||
@@ -20,6 +20,7 @@ from typing import ClassVar, Dict, List, Any, Optional, TYPE_CHECKING, TypeVar
|
||||
from pydantic import BaseModel, ValidationError
|
||||
|
||||
from ...common.type_safety import get_pydantic_model_fields
|
||||
from ...logging import get_logger
|
||||
from ..handler import BaseNodeHandler
|
||||
from ..task_result import TaskResult
|
||||
from ..handler import ValidationResult # Import ValidationResult
|
||||
@@ -30,6 +31,7 @@ if TYPE_CHECKING:
|
||||
from .client import ApiClient
|
||||
|
||||
T = TypeVar("T", bound=BaseModel)
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class BaseApiHandler(BaseNodeHandler[T]):
|
||||
@@ -67,6 +69,16 @@ class BaseApiHandler(BaseNodeHandler[T]):
|
||||
"""设置 Collection(由 DataSource 调用)"""
|
||||
self._collection = collection
|
||||
|
||||
def ensure_api_client(self) -> 'ApiClient':
|
||||
if self.api_client is None:
|
||||
raise RuntimeError("API client not set. Call set_api_client() before invoking API handler methods.")
|
||||
return self.api_client
|
||||
|
||||
def ensure_collection(self) -> 'DataCollection':
|
||||
if self._collection is None:
|
||||
raise RuntimeError("Collection not set. Call set_collection() before invoking handler methods.")
|
||||
return self._collection
|
||||
|
||||
@property
|
||||
def node_type(self) -> str:
|
||||
"""返回节点类型"""
|
||||
@@ -109,8 +121,7 @@ class BaseApiHandler(BaseNodeHandler[T]):
|
||||
def _create_basic_node(self, data: Dict[str, Any], depend_ids: Optional[List[str]] = None) -> "SyncNode":
|
||||
"""创建或复用基础节点(默认使用 data['id'] 或 data['_id'])"""
|
||||
data_id = str(data.get("id") or data.get("_id") or "")
|
||||
if self._collection is None:
|
||||
raise RuntimeError("Collection not set. Call set_collection() before creating nodes.")
|
||||
self.ensure_collection()
|
||||
|
||||
if data_id:
|
||||
existing_node = self._collection.get_by_data_id(self.node_type, data_id)
|
||||
@@ -329,10 +340,10 @@ class BaseApiHandler(BaseNodeHandler[T]):
|
||||
|
||||
if diff_fields:
|
||||
node_info = f"[{node_id}]" if node_id else ""
|
||||
print(f" [DIFF] {self.node_type}{node_info} schema={schema.__name__}")
|
||||
logger.info(" [DIFF] %s%s schema=%s", self.node_type, node_info, schema.__name__)
|
||||
for k, v in diff_fields.items():
|
||||
orig_v = origin_dict.get(k)
|
||||
print(f" • {k}: {orig_v!r} → {v!r}")
|
||||
logger.info(" • %s: %r → %r", k, orig_v, v)
|
||||
|
||||
return diff_fields if diff_fields else None
|
||||
|
||||
|
||||
@@ -37,9 +37,10 @@ from ..engine import (
|
||||
StateMachineRuntime,
|
||||
e40_sync_execute,
|
||||
)
|
||||
from ..logging import get_logger
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class BaseDataSource(ABC):
|
||||
@@ -460,7 +461,7 @@ class BaseDataSource(ABC):
|
||||
# Handler 加载数据并创建节点
|
||||
nodes = await handler.load()
|
||||
|
||||
print(f"Loaded nodes for {node_type}: {len(nodes)}")
|
||||
logger.info("Loaded nodes for %s: %d", node_type, len(nodes))
|
||||
|
||||
# 将节点写入 Collection:优先按 data_id 命中已有节点并更新
|
||||
await self._upsert_loaded_nodes(handler, node_type, nodes)
|
||||
@@ -468,7 +469,7 @@ class BaseDataSource(ABC):
|
||||
# 统计加载的节点数
|
||||
self._stats[node_type]["loaded"] += len(nodes)
|
||||
except Exception as exc:
|
||||
print(f"❌ [{node_type}] 加载失败: {exc}")
|
||||
logger.error("❌ [%s] 加载失败: %s", node_type, exc)
|
||||
continue
|
||||
|
||||
# ========== 同步执行 ==========
|
||||
@@ -639,7 +640,7 @@ class BaseDataSource(ABC):
|
||||
f"in_progress={create_in_progress}, failed={create_failed}, skipped={create_skipped}"
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"❌ [{handler.node_type}] create_all 异常: {e}")
|
||||
logger.error("❌ [%s] create_all 异常: %s", handler.node_type, e)
|
||||
for node in create_nodes:
|
||||
ok = self._apply_sync_execute_state(
|
||||
node,
|
||||
@@ -656,7 +657,7 @@ class BaseDataSource(ABC):
|
||||
for result in update_results:
|
||||
await self._handle_task_result(handler, result, async_tasks)
|
||||
except Exception as e:
|
||||
print(f"❌ [{handler.node_type}] update_all 异常: {e}")
|
||||
logger.error("❌ [%s] update_all 异常: %s", handler.node_type, e)
|
||||
for node in update_nodes:
|
||||
ok = self._apply_sync_execute_state(
|
||||
node,
|
||||
|
||||
@@ -19,7 +19,8 @@ class ConstructionTaskSyncStrategy(DefaultSyncStrategy[ConstructionResponse]):
|
||||
update_direction=UpdateDirection.PUSH,
|
||||
)
|
||||
|
||||
def preprocess_compare_payload(self, payload: dict) -> dict:
|
||||
def normalize_compare_payload(self, data: dict | None, data_id_map=None) -> dict:
|
||||
payload = super().normalize_compare_payload(data, data_id_map)
|
||||
plan_nodes = payload.get("plan_node")
|
||||
if not isinstance(plan_nodes, list):
|
||||
return payload
|
||||
|
||||
@@ -19,7 +19,8 @@ class ContractSettlementSyncStrategy(DefaultSyncStrategy[ContractSettlementRespo
|
||||
update_direction=UpdateDirection.PUSH,
|
||||
)
|
||||
|
||||
def preprocess_compare_payload(self, payload: dict) -> dict:
|
||||
def normalize_compare_payload(self, data: dict | None, data_id_map=None) -> dict:
|
||||
payload = super().normalize_compare_payload(data, data_id_map)
|
||||
plan_nodes = payload.get("plan_node")
|
||||
if not isinstance(plan_nodes, list):
|
||||
return payload
|
||||
|
||||
@@ -19,7 +19,8 @@ class MaterialSyncStrategy(DefaultSyncStrategy[MaterialResponse]):
|
||||
update_direction=UpdateDirection.PUSH,
|
||||
)
|
||||
|
||||
def preprocess_compare_payload(self, payload: dict) -> dict:
|
||||
def normalize_compare_payload(self, data: dict | None, data_id_map=None) -> dict:
|
||||
payload = super().normalize_compare_payload(data, data_id_map)
|
||||
plan_nodes = payload.get("plan_node")
|
||||
if not isinstance(plan_nodes, list):
|
||||
return payload
|
||||
|
||||
@@ -20,16 +20,13 @@ class PreparationSyncStrategy(DefaultSyncStrategy[PreparationDetail]):
|
||||
update_direction=UpdateDirection.PUSH,
|
||||
)
|
||||
|
||||
def _needs_update(self, source_node, target_node, resolved_data=None, data_id_map=None) -> bool:
|
||||
if resolved_data is None:
|
||||
raise ValueError(f"[{self.node_type}] resolved_data is required for differential sync.")
|
||||
|
||||
target_data = target_node.get_data()
|
||||
def should_update_pair(self, source_node, target_node, *, source_data, target_data, data_id_map=None) -> bool:
|
||||
if target_data is None:
|
||||
return False
|
||||
|
||||
# 这里只按 PostPreparation 可写字段判定是否需要 update,避免 response-only 字段差异触发空更新。
|
||||
update_fields = set(PostPreparation.model_fields.keys())
|
||||
source_payload = {key: value for key, value in resolved_data.items() if key in update_fields}
|
||||
source_payload = {key: value for key, value in source_data.items() if key in update_fields}
|
||||
target_payload = {key: value for key, value in target_data.items() if key in update_fields}
|
||||
|
||||
normalized_source = normalized_data_for_compare(source_payload, data_id_map or {})
|
||||
|
||||
@@ -20,16 +20,13 @@ class ProductionSyncStrategy(DefaultSyncStrategy[ProductionDetail]):
|
||||
update_direction=UpdateDirection.PUSH,
|
||||
)
|
||||
|
||||
def _needs_update(self, source_node, target_node, resolved_data=None, data_id_map=None) -> bool:
|
||||
if resolved_data is None:
|
||||
raise ValueError(f"[{self.node_type}] resolved_data is required for differential sync.")
|
||||
|
||||
target_data = target_node.get_data()
|
||||
def should_update_pair(self, source_node, target_node, *, source_data, target_data, data_id_map=None) -> bool:
|
||||
if target_data is None:
|
||||
return False
|
||||
|
||||
# 这里只按 PostProduction 可写字段判定是否需要 update,避免 response-only 字段差异触发空更新。
|
||||
update_fields = set(PostProduction.model_fields.keys())
|
||||
source_payload = {key: value for key, value in resolved_data.items() if key in update_fields}
|
||||
source_payload = {key: value for key, value in source_data.items() if key in update_fields}
|
||||
target_payload = {key: value for key, value in target_data.items() if key in update_fields}
|
||||
|
||||
normalized_source = normalized_data_for_compare(source_payload, data_id_map or {})
|
||||
|
||||
@@ -107,6 +107,8 @@ class ProjectApiHandler(BaseApiHandler):
|
||||
project_id: 可选,指定项目ID列表则只加载这些项目(优先级高于构造函数的filter_project_id)
|
||||
"""
|
||||
try:
|
||||
self.ensure_api_client()
|
||||
self.ensure_collection()
|
||||
# 清空旧缓存
|
||||
self.clear_cache()
|
||||
|
||||
|
||||
@@ -21,6 +21,8 @@ class ProjectSyncSchema(ProjectResponseBase):
|
||||
completed_investment: float | None = Field(None, description="已完成投资")
|
||||
total_contract_quantity: float | None = Field(None, description="合同总量")
|
||||
completed_settlement_quantity: float | None = Field(None, description="已结算量")
|
||||
ic_plan_first_production_date: str | None = Field(None, description="集团计划首批投产日期")
|
||||
ic_plan_full_production_date: str | None = Field(None, description="集团计划全容量投产日期")
|
||||
|
||||
|
||||
class ProjectSyncNode(SyncNode[ProjectSyncSchema]):
|
||||
|
||||
@@ -1,8 +1,14 @@
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ...sync_system.strategy import DefaultSyncStrategy
|
||||
from ...sync_system.config import StrategyConfig, OrphanAction, UpdateDirection
|
||||
from schemas.project.project_base import ProjectResponseBase
|
||||
|
||||
|
||||
class ProjectDomainOption(BaseModel):
|
||||
pull_group_plan_production_date: bool = False
|
||||
|
||||
|
||||
class ProjectSyncStrategy(DefaultSyncStrategy[ProjectResponseBase]):
|
||||
"""
|
||||
Project 同步策略
|
||||
@@ -17,6 +23,12 @@ class ProjectSyncStrategy(DefaultSyncStrategy[ProjectResponseBase]):
|
||||
|
||||
# 类变量:schema 固定为 ProjectResponseBase
|
||||
schema = ProjectResponseBase
|
||||
domain_option_model = ProjectDomainOption
|
||||
|
||||
_PULL_ONLY_FIELDS = [
|
||||
"ic_plan_first_production_date",
|
||||
"ic_plan_full_production_date",
|
||||
]
|
||||
|
||||
# 类变量:默认配置
|
||||
default_config = StrategyConfig(
|
||||
@@ -26,4 +38,60 @@ class ProjectSyncStrategy(DefaultSyncStrategy[ProjectResponseBase]):
|
||||
local_orphan_action=OrphanAction.CREATE_REMOTE,
|
||||
remote_orphan_action=OrphanAction.NONE,
|
||||
update_direction=UpdateDirection.PUSH,
|
||||
)
|
||||
)
|
||||
|
||||
@property
|
||||
def domain_option(self) -> ProjectDomainOption:
|
||||
return ProjectDomainOption.model_validate(self.config.domain_option)
|
||||
|
||||
def get_node_update_payload(self, node):
|
||||
payload = dict(node.get_data() or {})
|
||||
context = getattr(node, "context", None)
|
||||
if not isinstance(context, dict):
|
||||
return payload
|
||||
|
||||
all_info = context.get("all_info")
|
||||
if not isinstance(all_info, dict):
|
||||
return payload
|
||||
|
||||
for section_name, section_data in all_info.items():
|
||||
if not section_name.endswith("_extension") or not isinstance(section_data, dict):
|
||||
continue
|
||||
for field_name in self._PULL_ONLY_FIELDS:
|
||||
if payload.get(field_name) not in (None, ""):
|
||||
continue
|
||||
if field_name in section_data:
|
||||
payload[field_name] = section_data.get(field_name)
|
||||
|
||||
return payload
|
||||
|
||||
async def update_pair(self, local_node, remote_node, data_id_map=None):
|
||||
updated_by_id = {}
|
||||
|
||||
if self.domain_option.pull_group_plan_production_date:
|
||||
pull_update = await self.prepare_update_for_direction(
|
||||
local_node,
|
||||
remote_node,
|
||||
UpdateDirection.PULL,
|
||||
data_id_map,
|
||||
include_fields=self._PULL_ONLY_FIELDS,
|
||||
)
|
||||
if pull_update is not None:
|
||||
updated_by_id[pull_update.node_id] = pull_update
|
||||
|
||||
generic_exclude_fields = self._PULL_ONLY_FIELDS
|
||||
else:
|
||||
generic_exclude_fields = None
|
||||
|
||||
if self.config.update_direction != UpdateDirection.NONE:
|
||||
generic_update = await self.prepare_update_for_direction(
|
||||
local_node,
|
||||
remote_node,
|
||||
self.config.update_direction,
|
||||
data_id_map,
|
||||
exclude_fields=generic_exclude_fields,
|
||||
)
|
||||
if generic_update is not None:
|
||||
updated_by_id[generic_update.node_id] = generic_update
|
||||
|
||||
return list(updated_by_id.values())
|
||||
@@ -1,8 +1,16 @@
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from ...sync_system.strategy import DefaultSyncStrategy
|
||||
from ...sync_system.config import StrategyConfig, OrphanAction, UpdateDirection
|
||||
from ...sync_system.strategy_ops.compare_ops import build_data_id_normalization_map
|
||||
from ...sync_system.strategy_ops.update_ops import emit_schema_diff_report, _build_schema_diff_validator
|
||||
from schemas.project_extensions.project_detail import ProjectDetailKey, ProjectDetailProject
|
||||
|
||||
|
||||
class ProjectDetailDomainOption(BaseModel):
|
||||
pull_group_production_plans: bool = Field(True, description="固定将集团保供/爬坡/挑战产能计划按 pull 处理")
|
||||
|
||||
|
||||
class ProjectDetailSyncStrategy(DefaultSyncStrategy[ProjectDetailProject]):
|
||||
"""
|
||||
ProjectDetail 同步策略(项目容量信息)。
|
||||
@@ -13,6 +21,12 @@ class ProjectDetailSyncStrategy(DefaultSyncStrategy[ProjectDetailProject]):
|
||||
"""
|
||||
|
||||
schema = ProjectDetailProject
|
||||
domain_option_model = ProjectDetailDomainOption
|
||||
_GROUP_PRODUCTION_PLAN_KEYS = {
|
||||
"ensure_production",
|
||||
"climb_production",
|
||||
"challenge_production",
|
||||
}
|
||||
|
||||
default_config = StrategyConfig(
|
||||
auto_bind=True,
|
||||
@@ -21,16 +35,54 @@ class ProjectDetailSyncStrategy(DefaultSyncStrategy[ProjectDetailProject]):
|
||||
local_orphan_action=OrphanAction.NONE,
|
||||
remote_orphan_action=OrphanAction.NONE,
|
||||
update_direction=UpdateDirection.PUSH,
|
||||
field_direction_key="key",
|
||||
field_direction_overrides={
|
||||
"ensure_production": UpdateDirection.PULL,
|
||||
"climb_production": UpdateDirection.PULL,
|
||||
"challenge_production": UpdateDirection.PULL,
|
||||
},
|
||||
)
|
||||
|
||||
def preprocess_compare_payload(self, payload: dict) -> dict:
|
||||
normalized = dict(payload or {})
|
||||
async def update(self):
|
||||
self.ensure_runtime()
|
||||
validator = _build_schema_diff_validator(self)
|
||||
self._active_schema_diff_validator = validator
|
||||
data_id_map = await build_data_id_normalization_map(
|
||||
node_type=self.node_type,
|
||||
local_collection=self.local_collection,
|
||||
remote_collection=self.remote_collection,
|
||||
binding_manager=self.binding_manager,
|
||||
)
|
||||
|
||||
pull_pairs = []
|
||||
generic_pairs = []
|
||||
for pair in await self.collect_update_pairs():
|
||||
if self._should_pull_pair(pair.local_node, pair.remote_node):
|
||||
pull_pairs.append(pair)
|
||||
else:
|
||||
generic_pairs.append(pair)
|
||||
|
||||
updated_by_id = {}
|
||||
try:
|
||||
for pair in pull_pairs:
|
||||
updated_node = await self.prepare_update_for_direction(
|
||||
pair.local_node,
|
||||
pair.remote_node,
|
||||
UpdateDirection.PULL,
|
||||
data_id_map,
|
||||
)
|
||||
if updated_node is not None:
|
||||
updated_by_id[updated_node.node_id] = updated_node
|
||||
|
||||
for pair in generic_pairs:
|
||||
for updated_node in await self.update_pair(pair.local_node, pair.remote_node, data_id_map):
|
||||
updated_by_id[updated_node.node_id] = updated_node
|
||||
finally:
|
||||
self._active_schema_diff_validator = None
|
||||
|
||||
emit_schema_diff_report(self, validator)
|
||||
return list(updated_by_id.values())
|
||||
|
||||
@property
|
||||
def domain_option(self) -> ProjectDetailDomainOption:
|
||||
return ProjectDetailDomainOption.model_validate(self.config.domain_option)
|
||||
|
||||
def normalize_compare_payload(self, data: dict | None, data_id_map=None) -> dict:
|
||||
normalized = super().normalize_compare_payload(data, data_id_map)
|
||||
key = normalized.get("key")
|
||||
if isinstance(key, ProjectDetailKey):
|
||||
key = key.value
|
||||
@@ -76,3 +128,20 @@ class ProjectDetailSyncStrategy(DefaultSyncStrategy[ProjectDetailProject]):
|
||||
items.append(normalized_item)
|
||||
|
||||
return items
|
||||
|
||||
@staticmethod
|
||||
def _extract_pair_key(local_node, remote_node) -> str | None:
|
||||
local_data = local_node.get_data() or {}
|
||||
remote_data = remote_node.get_data() or {}
|
||||
|
||||
key = local_data.get("key", remote_data.get("key"))
|
||||
if isinstance(key, ProjectDetailKey):
|
||||
return key.value
|
||||
if isinstance(key, str):
|
||||
return key
|
||||
return None
|
||||
|
||||
def _should_pull_pair(self, local_node, remote_node) -> bool:
|
||||
if not self.domain_option.pull_group_production_plans:
|
||||
return False
|
||||
return self._extract_pair_key(local_node, remote_node) in self._GROUP_PRODUCTION_PLAN_KEYS
|
||||
|
||||
@@ -24,19 +24,16 @@ class SupplierSyncStrategy(DefaultSyncStrategy[SupplierDetailResponse]):
|
||||
update_direction=UpdateDirection.PULL,
|
||||
)
|
||||
|
||||
def _needs_update(
|
||||
def should_update_pair(
|
||||
self,
|
||||
source_node,
|
||||
target_node,
|
||||
resolved_data=None,
|
||||
*,
|
||||
source_data,
|
||||
target_data,
|
||||
data_id_map=None,
|
||||
) -> bool:
|
||||
"""Supplier 更新忽略 id 字段差异。"""
|
||||
if resolved_data is None:
|
||||
raise ValueError(f"[{self.node_type}] resolved_data is required for differential sync.")
|
||||
|
||||
source_data = resolved_data
|
||||
target_data = target_node.get_data()
|
||||
if source_data is None or target_data is None:
|
||||
return False
|
||||
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
|
||||
APP_LOGGER_NAME = "sync_state_machine"
|
||||
|
||||
|
||||
class _NodeLogFilter(logging.Filter):
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
message = record.getMessage()
|
||||
if "node=" in message:
|
||||
return False
|
||||
if "节点 " in message and "状态" in message:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def get_app_logger() -> logging.Logger:
|
||||
return logging.getLogger(APP_LOGGER_NAME)
|
||||
|
||||
|
||||
def get_logger(name: Optional[str] = None) -> logging.Logger:
|
||||
if not name or name == "__main__":
|
||||
return get_app_logger()
|
||||
if name == APP_LOGGER_NAME or name.startswith(f"{APP_LOGGER_NAME}."):
|
||||
return logging.getLogger(name)
|
||||
return logging.getLogger(f"{APP_LOGGER_NAME}.{name}")
|
||||
|
||||
|
||||
def clear_app_logger_handlers(logger: Optional[logging.Logger] = None) -> None:
|
||||
target = logger or get_app_logger()
|
||||
handlers = list(target.handlers)
|
||||
for handler in handlers:
|
||||
target.removeHandler(handler)
|
||||
try:
|
||||
handler.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def init_app_logger(
|
||||
*,
|
||||
log_file_path: Optional[str] = None,
|
||||
level: int = logging.INFO,
|
||||
mirror_console: bool = True,
|
||||
replace_handlers: bool = True,
|
||||
suppress_node_logs: bool = True,
|
||||
) -> logging.Logger:
|
||||
logger = get_app_logger()
|
||||
logger.setLevel(level)
|
||||
logger.propagate = False
|
||||
|
||||
if logger.handlers and not replace_handlers:
|
||||
return logger
|
||||
|
||||
if replace_handlers:
|
||||
clear_app_logger_handlers(logger)
|
||||
|
||||
formatter = logging.Formatter("%(message)s")
|
||||
node_filter = _NodeLogFilter() if suppress_node_logs else None
|
||||
|
||||
if mirror_console:
|
||||
console_handler = logging.StreamHandler()
|
||||
console_handler.setLevel(level)
|
||||
console_handler.setFormatter(formatter)
|
||||
if node_filter is not None:
|
||||
console_handler.addFilter(node_filter)
|
||||
logger.addHandler(console_handler)
|
||||
|
||||
if log_file_path:
|
||||
path = Path(log_file_path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
file_handler = logging.FileHandler(path, mode="w", encoding="utf-8")
|
||||
file_handler.setLevel(level)
|
||||
file_handler.setFormatter(formatter)
|
||||
if node_filter is not None:
|
||||
file_handler.addFilter(node_filter)
|
||||
logger.addHandler(file_handler)
|
||||
|
||||
return logger
|
||||
|
||||
|
||||
def configure_app_logging(config, *, replace_handlers: bool = False, mirror_console: bool = True) -> logging.Logger:
|
||||
if not getattr(config, "initialize", True):
|
||||
return get_app_logger()
|
||||
|
||||
if not getattr(config, "file_path", None):
|
||||
from .config.logging_config import resolve_log_file_path
|
||||
|
||||
config.file_path = resolve_log_file_path(config)
|
||||
|
||||
return init_app_logger(
|
||||
log_file_path=config.file_path,
|
||||
level=config.level,
|
||||
mirror_console=mirror_console,
|
||||
replace_handlers=replace_handlers,
|
||||
suppress_node_logs=config.suppress_node_logs,
|
||||
)
|
||||
|
||||
|
||||
def ensure_app_logging(config, *, mirror_console: bool = True) -> logging.Logger:
|
||||
logger = get_app_logger()
|
||||
if not getattr(config, "initialize", True):
|
||||
return logger
|
||||
if logger.handlers:
|
||||
return logger
|
||||
return configure_app_logging(config, replace_handlers=False, mirror_console=mirror_console)
|
||||
@@ -45,12 +45,14 @@ from ..config import (
|
||||
is_multi_project_payload,
|
||||
load_overrides_from_file,
|
||||
)
|
||||
from ..config.strategy_config import resolve_domain_option_config
|
||||
from ..sync_system.strategy import BaseSyncStrategy
|
||||
from ..logging import configure_app_logging, ensure_app_logging, get_logger
|
||||
from .run_logger import init_pipeline_logger
|
||||
from .full_sync_pipeline import FullSyncPipeline
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
def _project_root() -> Path:
|
||||
@@ -63,29 +65,6 @@ def _prepare_remote_jsonl_dir(remote_dir: Path, *, project_root: Optional[Path]
|
||||
return prepare_remote_jsonl_dir(remote_dir, project_root=(project_root or _project_root()))
|
||||
|
||||
|
||||
def _ensure_pipeline_logger(
|
||||
*,
|
||||
initialize_logger: bool,
|
||||
logger_file_path: Optional[str],
|
||||
logger_level: int,
|
||||
suppress_node_logs: bool,
|
||||
) -> None:
|
||||
if not initialize_logger:
|
||||
return
|
||||
|
||||
app_logger = logging.getLogger("sync_state_machine")
|
||||
if app_logger.handlers:
|
||||
return
|
||||
|
||||
init_pipeline_logger(
|
||||
log_file_path=logger_file_path,
|
||||
level=logger_level,
|
||||
mirror_console=True,
|
||||
replace_handlers=False,
|
||||
suppress_node_logs=suppress_node_logs,
|
||||
)
|
||||
|
||||
|
||||
def _build_handler_config(
|
||||
node_type: str,
|
||||
ds_config: DataSourceConfig,
|
||||
@@ -198,16 +177,28 @@ def _build_collections_and_bindings(
|
||||
|
||||
def _resolve_strategy_map(config: PipelineRunConfig, node_types: List[str]) -> Dict[str, StrategyRuntimeConfig]:
|
||||
if config.strategies:
|
||||
return {item.node_type: item for item in config.strategies}
|
||||
resolved_from_config = {item.node_type: item for item in config.strategies}
|
||||
for node_type, runtime_cfg in resolved_from_config.items():
|
||||
strategy_class = DomainRegistry.get_strategy_class(node_type)
|
||||
runtime_cfg.config.domain_option = resolve_domain_option_config(
|
||||
getattr(strategy_class, "domain_option_model", None) if strategy_class is not None else None,
|
||||
runtime_cfg.config.domain_option,
|
||||
)
|
||||
return resolved_from_config
|
||||
|
||||
resolved: Dict[str, StrategyRuntimeConfig] = {}
|
||||
for node_type in node_types:
|
||||
strategy_class = DomainRegistry.get_strategy_class(node_type)
|
||||
if strategy_class is None:
|
||||
continue
|
||||
strategy_config = strategy_class.default_config.model_copy(deep=True)
|
||||
strategy_config.domain_option = resolve_domain_option_config(
|
||||
getattr(strategy_class, "domain_option_model", None),
|
||||
strategy_config.domain_option,
|
||||
)
|
||||
resolved[node_type] = StrategyRuntimeConfig(
|
||||
node_type=node_type,
|
||||
config=strategy_class.default_config.model_copy(deep=True),
|
||||
config=strategy_config,
|
||||
)
|
||||
return resolved
|
||||
|
||||
@@ -251,20 +242,18 @@ def _build_strategies(
|
||||
raise RuntimeError(f"Missing strategy for node_type: {node_type}")
|
||||
strategy = strategy_class(node_type, local_collection, remote_collection, binding_manager)
|
||||
|
||||
if both_jsonl:
|
||||
strategy.update_config(
|
||||
local_orphan_action=OrphanAction.CREATE_REMOTE,
|
||||
remote_orphan_action=OrphanAction.NONE,
|
||||
update_direction=UpdateDirection.PUSH,
|
||||
)
|
||||
if node_type == "contract":
|
||||
strategy.update_config(depend_fields={"project_id": "project"})
|
||||
|
||||
runtime_cfg = strategy_map.get(node_type)
|
||||
if runtime_cfg is not None:
|
||||
strategy.set_config(runtime_cfg.config.model_copy(deep=True))
|
||||
strategy.skip_sync = bool(runtime_cfg.skip_sync)
|
||||
strategy.skip_post_check = bool(runtime_cfg.skip_post_check)
|
||||
strategy.config = runtime_cfg.config.model_copy(deep=True)
|
||||
strategy.config.log_logic_warnings(node_type=strategy.node_type, logger=logger)
|
||||
|
||||
if both_jsonl:
|
||||
strategy.config.local_orphan_action = OrphanAction.CREATE_REMOTE
|
||||
strategy.config.remote_orphan_action = OrphanAction.NONE
|
||||
strategy.config.update_direction = UpdateDirection.PUSH
|
||||
if node_type == "contract":
|
||||
strategy.config.depend_fields = {"project_id": "project"}
|
||||
strategy.config.log_logic_warnings(node_type=strategy.node_type, logger=logger)
|
||||
|
||||
strategies.append(strategy)
|
||||
return strategies
|
||||
@@ -280,15 +269,7 @@ async def create_pipeline_from_config(
|
||||
local_datasource_override=None,
|
||||
remote_datasource_override=None,
|
||||
) -> FullSyncPipeline:
|
||||
if config.logging.initialize and not config.logging.file_path:
|
||||
config.logging.file_path = resolve_log_file_path(config.logging)
|
||||
|
||||
_ensure_pipeline_logger(
|
||||
initialize_logger=config.logging.initialize,
|
||||
logger_file_path=config.logging.file_path,
|
||||
logger_level=config.logging.level,
|
||||
suppress_node_logs=config.logging.suppress_node_logs,
|
||||
)
|
||||
ensure_app_logging(config.logging)
|
||||
|
||||
all_types = list(config.node_types)
|
||||
strategy_map = _resolve_strategy_map(config, all_types)
|
||||
@@ -376,15 +357,7 @@ async def run_pipeline_from_config(
|
||||
local_datasource_override=None,
|
||||
remote_datasource_override=None,
|
||||
) -> Dict[str, Any]:
|
||||
if config.logging.initialize and not config.logging.file_path:
|
||||
config.logging.file_path = resolve_log_file_path(config.logging)
|
||||
|
||||
_ensure_pipeline_logger(
|
||||
initialize_logger=config.logging.initialize,
|
||||
logger_file_path=config.logging.file_path,
|
||||
logger_level=config.logging.level,
|
||||
suppress_node_logs=config.logging.suppress_node_logs,
|
||||
)
|
||||
ensure_app_logging(config.logging)
|
||||
|
||||
resolved_cfg = config_snapshot(config)
|
||||
resolved_cfg_text = json.dumps(resolved_cfg, ensure_ascii=False, indent=2)
|
||||
@@ -394,28 +367,21 @@ async def run_pipeline_from_config(
|
||||
config.local_datasource.type,
|
||||
config.remote_datasource.type,
|
||||
)
|
||||
if config.logging.file_path:
|
||||
try:
|
||||
with open(config.logging.file_path, "a", encoding="utf-8") as fp:
|
||||
fp.write("Resolved Full Config:\n")
|
||||
fp.write(resolved_cfg_text)
|
||||
fp.write("\n")
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to write resolved config into log file: %s", exc)
|
||||
logger.info("Resolved Full Config:\n%s", resolved_cfg_text)
|
||||
|
||||
pipeline: Optional[FullSyncPipeline] = None
|
||||
try:
|
||||
if print_summary:
|
||||
print("\n" + "=" * 80)
|
||||
print(f"🚀 Creating {title}")
|
||||
print(f"📝 Log file: {config.logging.file_path or '-'}")
|
||||
print(
|
||||
f"🔧 Resolved Full Config Summary: node_types={len(config.node_types)} "
|
||||
f"local={config.local_datasource.type} "
|
||||
f"remote={config.remote_datasource.type}"
|
||||
logger.info("\n%s", "=" * 80)
|
||||
logger.info("🚀 Creating %s", title)
|
||||
logger.info("📝 Log file: %s", config.logging.file_path or "-")
|
||||
logger.info(
|
||||
"🔧 Resolved Full Config Summary: node_types=%d local=%s remote=%s",
|
||||
len(config.node_types),
|
||||
config.local_datasource.type,
|
||||
config.remote_datasource.type,
|
||||
)
|
||||
print(f"🔧 Resolved Full Config:\n{resolved_cfg_text}")
|
||||
print("=" * 80)
|
||||
logger.info("=" * 80)
|
||||
|
||||
pipeline = await create_pipeline_from_config(
|
||||
config,
|
||||
@@ -428,8 +394,8 @@ async def run_pipeline_from_config(
|
||||
stats = await pipeline.run()
|
||||
|
||||
if print_summary:
|
||||
print(f"\n✅ Pipeline completed: stats={stats}")
|
||||
print(f"🗃️ Persistence DB: {config.persist.db_path}")
|
||||
logger.info("✅ Pipeline completed: stats=%s", stats)
|
||||
logger.info("🗃️ Persistence DB: %s", config.persist.db_path)
|
||||
|
||||
return stats
|
||||
finally:
|
||||
@@ -497,12 +463,13 @@ async def run_profile_from_file(
|
||||
multi_cfg = MultiProjectRunConfig.model_validate(raw)
|
||||
pipeline = MultiProjectPipeline(project_root=project_root, config_path=cfg_path, config=multi_cfg)
|
||||
if print_summary:
|
||||
print("\n" + "=" * 80)
|
||||
print(f"🚀 Creating multi-project pipeline from: {cfg_path}")
|
||||
print("=" * 80)
|
||||
logger.info("\n%s", "=" * 80)
|
||||
logger.info("🚀 Creating multi-project pipeline from: %s", cfg_path)
|
||||
logger.info("=" * 80)
|
||||
return await pipeline.run()
|
||||
|
||||
config = build_config(project_root=project_root, overrides=raw)
|
||||
configure_app_logging(config.logging, replace_handlers=False)
|
||||
mode = f"{config.local_datasource.type}->{config.remote_datasource.type}"
|
||||
return await run_pipeline_from_config(
|
||||
config,
|
||||
|
||||
@@ -20,7 +20,7 @@ from ..sync_system.strategy import BaseSyncStrategy
|
||||
from ..sync_system.config import OrphanAction, UpdateDirection
|
||||
from ..engine import StateMachineConfig, StateMachineRuntime
|
||||
from ..engine import e41_post_create_ready
|
||||
from ..sync_system.strategy_ops import finalize_peer_creating_sources
|
||||
from ..sync_system.strategy_ops import finalize_peer_creating_sources, run_phase2_cleanup
|
||||
from ..sync_system.strategy_ops.post_check_ops import evaluate_consistency_for_node_type
|
||||
from .summary_report import print_pipeline_summary
|
||||
|
||||
@@ -282,7 +282,7 @@ class FullSyncPipeline:
|
||||
# 加载后 reset 清理:清理 CREATE 失败的僵尸绑定
|
||||
if not self.strategies:
|
||||
raise RuntimeError("no strategies configured: cannot run bootstrap event E01")
|
||||
total_cleaned = await BaseSyncStrategy.run_reset(
|
||||
total_cleaned = await run_phase2_cleanup(
|
||||
node_types=self.node_types,
|
||||
local_collection=self.local_collection,
|
||||
remote_collection=self.remote_collection,
|
||||
@@ -397,11 +397,11 @@ class FullSyncPipeline:
|
||||
|
||||
result = await evaluate_consistency_for_node_type(
|
||||
node_type=node_type,
|
||||
strategy=strategy,
|
||||
local_collection=self.local_collection,
|
||||
remote_collection=self.remote_collection,
|
||||
binding_manager=self.binding_manager,
|
||||
logger=self._logger,
|
||||
normalize_compare_payload=strategy.normalize_compare_payload,
|
||||
depend_fields=depend_fields,
|
||||
compare_to_remote=compare_to_remote,
|
||||
ignore_fields=ignore_fields,
|
||||
|
||||
@@ -1,18 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class _NodeLogFilter(logging.Filter):
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
message = record.getMessage()
|
||||
if "node=" in message:
|
||||
return False
|
||||
if "节点 " in message and "状态" in message:
|
||||
return False
|
||||
return True
|
||||
from ..logging import clear_app_logger_handlers, init_app_logger
|
||||
|
||||
|
||||
def init_pipeline_logger(
|
||||
@@ -23,43 +14,14 @@ def init_pipeline_logger(
|
||||
replace_handlers: bool = True,
|
||||
suppress_node_logs: bool = True,
|
||||
) -> logging.Logger:
|
||||
logger = logging.getLogger("sync_state_machine")
|
||||
logger.setLevel(level)
|
||||
logger.propagate = False
|
||||
|
||||
if replace_handlers:
|
||||
clear_pipeline_logger_handlers(logger)
|
||||
|
||||
formatter = logging.Formatter("%(message)s")
|
||||
node_filter = _NodeLogFilter() if suppress_node_logs else None
|
||||
|
||||
if mirror_console:
|
||||
console_handler = logging.StreamHandler()
|
||||
console_handler.setLevel(level)
|
||||
console_handler.setFormatter(formatter)
|
||||
if node_filter is not None:
|
||||
console_handler.addFilter(node_filter)
|
||||
logger.addHandler(console_handler)
|
||||
|
||||
if log_file_path:
|
||||
path = Path(log_file_path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
file_handler = logging.FileHandler(path, mode="w", encoding="utf-8")
|
||||
file_handler.setLevel(level)
|
||||
file_handler.setFormatter(formatter)
|
||||
if node_filter is not None:
|
||||
file_handler.addFilter(node_filter)
|
||||
logger.addHandler(file_handler)
|
||||
|
||||
return logger
|
||||
return init_app_logger(
|
||||
log_file_path=log_file_path,
|
||||
level=level,
|
||||
mirror_console=mirror_console,
|
||||
replace_handlers=replace_handlers,
|
||||
suppress_node_logs=suppress_node_logs,
|
||||
)
|
||||
|
||||
|
||||
def clear_pipeline_logger_handlers(logger: Optional[logging.Logger] = None) -> None:
|
||||
target = logger or logging.getLogger("sync_state_machine")
|
||||
handlers = list(target.handlers)
|
||||
for handler in handlers:
|
||||
target.removeHandler(handler)
|
||||
try:
|
||||
handler.close()
|
||||
except Exception:
|
||||
pass
|
||||
clear_app_logger_handlers(logger)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import copy
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import TypeVar, Generic, Type, Dict, List, Optional, Any
|
||||
from pydantic import BaseModel
|
||||
import logging
|
||||
@@ -7,31 +7,35 @@ from pathlib import Path
|
||||
from ..common.collection import DataCollection
|
||||
from ..common.binding import BindingManager
|
||||
from ..common.sync_node import SyncNode
|
||||
from .config import StrategyConfig, OrphanAction, UpdateDirection, ConfigPresets
|
||||
from .config import StrategyConfig, OrphanAction, UpdateDirection
|
||||
from ..config.strategy_config import resolve_domain_option_config
|
||||
from .strategy_ops import (
|
||||
get_phase1_reset_defaults,
|
||||
run_bind,
|
||||
run_create,
|
||||
run_delete,
|
||||
run_phase2_cleanup,
|
||||
)
|
||||
from .strategy_ops.update_ops import (
|
||||
collect_bound_node_pairs,
|
||||
default_get_node_update_payload,
|
||||
default_needs_update,
|
||||
default_update_pair,
|
||||
prepare_directional_update,
|
||||
run_update,
|
||||
)
|
||||
from ..engine import (
|
||||
StateMachineConfig,
|
||||
StateMachineRuntime,
|
||||
)
|
||||
from ..validation import SchemaDiffValidator
|
||||
from .strategy_ops.compare_ops import normalized_data_for_compare
|
||||
|
||||
T = TypeVar("T", bound=BaseModel)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class BaseSyncStrategy(Generic[T]):
|
||||
class BaseSyncStrategy(ABC, Generic[T]):
|
||||
"""
|
||||
同步策略基类。
|
||||
纯逻辑层,不负责 IO 执行。
|
||||
同步策略抽象接口。
|
||||
|
||||
子类可以通过覆盖 default_config 来设置默认配置:
|
||||
子类可以通过覆盖以下类变量提供静态配置:
|
||||
```python
|
||||
class ProjectStrategy(DefaultSyncStrategy):
|
||||
default_config = StrategyConfig(
|
||||
@@ -47,41 +51,122 @@ class BaseSyncStrategy(Generic[T]):
|
||||
适用于只需要加载数据供其他类型引用的场景(如 supplier)
|
||||
skip_post_check: bool - 如果为 True,则跳过该类型最终 post-check 的 reload 与一致性校验
|
||||
"""
|
||||
# 类变量:默认配置(子类可覆盖)
|
||||
default_config: StrategyConfig = StrategyConfig()
|
||||
# 类变量:schema 类型(子类必须设置)
|
||||
schema: Type[T] = None # type: ignore
|
||||
# 类变量:是否默认跳过同步(子类可覆盖)
|
||||
default_skip_sync: bool = False
|
||||
# 类变量:是否默认跳过 post-check(子类可覆盖)
|
||||
default_skip_post_check: bool = False
|
||||
|
||||
default_config: StrategyConfig
|
||||
domain_option_model: Type[BaseModel] | None
|
||||
schema: Type[T]
|
||||
node_type: str
|
||||
config: StrategyConfig
|
||||
sm_runtime: Optional[StateMachineRuntime]
|
||||
|
||||
@abstractmethod
|
||||
def __init__(
|
||||
self,
|
||||
node_type: str,
|
||||
local_collection: DataCollection,
|
||||
remote_collection: DataCollection,
|
||||
self,
|
||||
node_type: str,
|
||||
local_collection: DataCollection,
|
||||
remote_collection: DataCollection,
|
||||
binding_manager: BindingManager,
|
||||
):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def bind(self, **kwargs):
|
||||
"""进行绑定逻辑判定,设置 node.binding_status 和 node.action"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def create(self) -> List[SyncNode]:
|
||||
"""准备 CREATE 操作"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def update(self) -> List[SyncNode]:
|
||||
"""准备 UPDATE 操作"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def delete(self) -> List[SyncNode]:
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def skip_sync(self) -> bool:
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def skip_post_check(self) -> bool:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def normalize_compare_payload(
|
||||
self,
|
||||
data: Optional[Dict[str, Any]],
|
||||
data_id_map: Optional[Dict[str, str]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
pass
|
||||
|
||||
|
||||
class DefaultSyncStrategy(BaseSyncStrategy[T]):
|
||||
"""
|
||||
基于 STATE_DEFINITIONS.md 实现的标准化同步策略。
|
||||
采用四阶段管线:核心状态 -> 依赖检查 -> 自动匹配 -> 动作映射。
|
||||
"""
|
||||
|
||||
default_config: StrategyConfig = StrategyConfig()
|
||||
domain_option_model: Type[BaseModel] | None = None
|
||||
schema: Type[T] = None # type: ignore
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
node_type: str,
|
||||
local_collection: DataCollection,
|
||||
remote_collection: DataCollection,
|
||||
binding_manager: BindingManager,
|
||||
):
|
||||
if self.schema is None:
|
||||
raise ValueError(f"{self.__class__.__name__} must define schema as a class variable")
|
||||
|
||||
|
||||
self.node_type = node_type
|
||||
self.local_collection = local_collection
|
||||
self.remote_collection = remote_collection
|
||||
self.binding_manager = binding_manager
|
||||
# 复制类级别的默认配置到实例
|
||||
self.config = self.default_config.model_copy(deep=True)
|
||||
# 是否跳过同步(可运行时修改)
|
||||
self.skip_sync = self.default_skip_sync
|
||||
# 是否跳过 post-check(可运行时修改)
|
||||
self.skip_post_check = self.default_skip_post_check
|
||||
# 统一由 Pipeline 注入 runtime;未注入时在执行阶段显式报错
|
||||
self.sm_runtime: Optional[StateMachineRuntime] = None
|
||||
self._schema_diff_validator: Optional[SchemaDiffValidator] = None
|
||||
|
||||
# 配置逻辑校验
|
||||
self._validate_config()
|
||||
self.config.domain_option = resolve_domain_option_config(
|
||||
self.domain_option_model,
|
||||
self.config.domain_option,
|
||||
)
|
||||
|
||||
self.config.log_logic_warnings(node_type=self.node_type, logger=logger)
|
||||
|
||||
@classmethod
|
||||
def get_default_domain_option(cls) -> Dict[str, Any]:
|
||||
return resolve_domain_option_config(cls.domain_option_model, None)
|
||||
|
||||
def _validate_domain_option(self, domain_option: Optional[Dict[str, Any]]) -> BaseModel | Dict[str, Any]:
|
||||
if self.domain_option_model is None:
|
||||
return dict(domain_option or {})
|
||||
return self.domain_option_model.model_validate(domain_option or {})
|
||||
|
||||
@property
|
||||
def domain_option(self) -> BaseModel | Dict[str, Any]:
|
||||
return self._validate_domain_option(self.config.domain_option)
|
||||
|
||||
@property
|
||||
def skip_sync(self) -> bool:
|
||||
return bool(self.config.skip_sync)
|
||||
|
||||
@skip_sync.setter
|
||||
def skip_sync(self, value: bool) -> None:
|
||||
self.config.skip_sync = bool(value)
|
||||
|
||||
@property
|
||||
def skip_post_check(self) -> bool:
|
||||
return bool(self.config.skip_post_check)
|
||||
|
||||
@skip_post_check.setter
|
||||
def skip_post_check(self, value: bool) -> None:
|
||||
self.config.skip_post_check = bool(value)
|
||||
|
||||
def ensure_runtime(self) -> StateMachineRuntime:
|
||||
runtime = self.sm_runtime
|
||||
@@ -93,128 +178,6 @@ class BaseSyncStrategy(Generic[T]):
|
||||
self.remote_collection.set_state_machine_runtime(runtime)
|
||||
return runtime
|
||||
|
||||
def _validate_config(self) -> None:
|
||||
"""
|
||||
校验配置的逻辑一致性。
|
||||
|
||||
规则:
|
||||
- PUSH 模式:local_orphan_action 应为 CREATE_REMOTE,remote_orphan_action 应为 NONE
|
||||
- PULL 模式:local_orphan_action 应为 NONE,remote_orphan_action 应为 CREATE_LOCAL
|
||||
- NONE 模式:两者都应为 NONE
|
||||
"""
|
||||
direction = self.config.update_direction
|
||||
local_action = self.config.local_orphan_action
|
||||
remote_action = self.config.remote_orphan_action
|
||||
|
||||
# PUSH 模式检查
|
||||
if direction == UpdateDirection.PUSH:
|
||||
if local_action == OrphanAction.CREATE_LOCAL:
|
||||
logger.warning(
|
||||
f"[{self.node_type}] Config inconsistency: "
|
||||
f"update_direction=PUSH but local_orphan_action=CREATE_LOCAL (本地创建)。"
|
||||
f"应使用 CREATE_REMOTE (推送到远程)"
|
||||
)
|
||||
if remote_action == OrphanAction.CREATE_LOCAL:
|
||||
logger.warning(
|
||||
f"[{self.node_type}] Config inconsistency: "
|
||||
f"update_direction=PUSH but remote_orphan_action=CREATE_LOCAL (本地创建)。"
|
||||
f"PUSH 模式应设为 NONE 或 DELETE_REMOTE"
|
||||
)
|
||||
|
||||
# PULL 模式检查
|
||||
elif direction == UpdateDirection.PULL:
|
||||
if local_action == OrphanAction.CREATE_REMOTE:
|
||||
logger.warning(
|
||||
f"[{self.node_type}] Config inconsistency: "
|
||||
f"update_direction=PULL but local_orphan_action=CREATE_REMOTE (推送到远程)。"
|
||||
f"PULL 模式应设为 NONE 或 DELETE_LOCAL"
|
||||
)
|
||||
if remote_action in {OrphanAction.CREATE_REMOTE, OrphanAction.DELETE_LOCAL}:
|
||||
logger.warning(
|
||||
f"[{self.node_type}] Config inconsistency: "
|
||||
f"update_direction=PULL but remote_orphan_action={remote_action}。"
|
||||
f"PULL 模式建议使用 CREATE_LOCAL 或 NONE"
|
||||
)
|
||||
|
||||
# NONE 模式检查
|
||||
elif direction == UpdateDirection.NONE:
|
||||
if local_action != OrphanAction.NONE:
|
||||
logger.warning(
|
||||
f"[{self.node_type}] Config inconsistency: "
|
||||
f"update_direction=NONE but local_orphan_action={local_action}。"
|
||||
f"应设为 NONE"
|
||||
)
|
||||
if remote_action != OrphanAction.NONE:
|
||||
logger.warning(
|
||||
f"[{self.node_type}] Config inconsistency: "
|
||||
f"update_direction=NONE but remote_orphan_action={remote_action}。"
|
||||
f"应设为 NONE"
|
||||
)
|
||||
|
||||
def set_config(self, config: StrategyConfig):
|
||||
"""完全替换配置对象"""
|
||||
self.config = config
|
||||
self._schema_diff_validator = None
|
||||
self._validate_config()
|
||||
|
||||
def update_config(self, **kwargs):
|
||||
"""
|
||||
临时修改部分配置项。
|
||||
|
||||
Example:
|
||||
strategy.update_config(
|
||||
local_orphan_action=OrphanAction.NONE,
|
||||
update_direction=UpdateDirection.PULL
|
||||
)
|
||||
"""
|
||||
valid_fields = type(self.config).model_fields
|
||||
for key, value in kwargs.items():
|
||||
if key in valid_fields:
|
||||
setattr(self.config, key, value)
|
||||
else:
|
||||
logger.warning(f"[{self.node_type}] Unknown config key: {key}")
|
||||
self._schema_diff_validator = None
|
||||
# 更新配置后也要校验
|
||||
self._validate_config()
|
||||
|
||||
def apply_preset(self, preset_name: str):
|
||||
"""
|
||||
应用配置预设。
|
||||
|
||||
Args:
|
||||
preset_name: 预设名称,可选:first_sync, daily_sync, repair_sync, pull_only, push_only
|
||||
|
||||
Example:
|
||||
strategy.apply_preset("first_sync")
|
||||
"""
|
||||
preset_config = ConfigPresets.get_preset(preset_name)
|
||||
self.config = preset_config
|
||||
self._schema_diff_validator = None
|
||||
self._validate_config()
|
||||
logger.info(f"[{self.node_type}] Applied preset: {preset_name}")
|
||||
|
||||
def get_schema_diff_validator(self) -> SchemaDiffValidator:
|
||||
if self._schema_diff_validator is None:
|
||||
self._schema_diff_validator = SchemaDiffValidator(
|
||||
schema_name=self.schema.__name__,
|
||||
ignore_fields=self.config.compare_ignore_fields,
|
||||
ignore_list_item_fields=self.config.post_check_ignore_list_item_fields,
|
||||
)
|
||||
return self._schema_diff_validator
|
||||
|
||||
def reset_schema_diff_validator(self) -> None:
|
||||
self.get_schema_diff_validator().reset()
|
||||
|
||||
def emit_schema_diff_report(self) -> None:
|
||||
validator = self.get_schema_diff_validator()
|
||||
if not validator.has_records():
|
||||
return
|
||||
for line in validator.format_summary_lines():
|
||||
logger.info(f"[{self.node_type}] validate {line}")
|
||||
|
||||
def preprocess_compare_payload(self, payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
return payload
|
||||
|
||||
def normalize_compare_payload(
|
||||
self,
|
||||
data: Optional[Dict[str, Any]],
|
||||
@@ -226,103 +189,12 @@ class BaseSyncStrategy(Generic[T]):
|
||||
ignore_fields=set(self.config.compare_ignore_fields),
|
||||
ignore_list_item_fields=self.config.post_check_ignore_list_item_fields,
|
||||
)
|
||||
return self.preprocess_compare_payload(copy.deepcopy(normalized))
|
||||
|
||||
async def bind(self, **kwargs):
|
||||
"""进行绑定逻辑判定,设置 node.binding_status 和 node.action"""
|
||||
raise NotImplementedError
|
||||
|
||||
async def create(self) -> List[SyncNode]:
|
||||
"""准备 CREATE 操作"""
|
||||
raise NotImplementedError
|
||||
|
||||
async def update(self) -> List[SyncNode]:
|
||||
"""准备 UPDATE 操作"""
|
||||
raise NotImplementedError
|
||||
|
||||
def _needs_update(
|
||||
self,
|
||||
source_node: SyncNode,
|
||||
target_node: SyncNode,
|
||||
resolved_data: Optional[Dict[str, Any]] = None,
|
||||
data_id_map: Optional[Dict[str, str]] = None,
|
||||
) -> bool:
|
||||
from .strategy_ops.update_ops import default_needs_update
|
||||
|
||||
return default_needs_update(self, source_node, target_node, resolved_data, data_id_map)
|
||||
|
||||
@classmethod
|
||||
def get_phase1_reset_defaults(cls) -> Dict[str, Any]:
|
||||
return get_phase1_reset_defaults()
|
||||
|
||||
@classmethod
|
||||
async def run_reset(
|
||||
cls,
|
||||
*,
|
||||
node_types: List[str],
|
||||
local_collection: DataCollection,
|
||||
remote_collection: DataCollection,
|
||||
binding_manager: BindingManager,
|
||||
runtime: StateMachineRuntime,
|
||||
) -> int:
|
||||
"""
|
||||
执行加载后的重置清理(僵尸 CREATE 节点清理)。
|
||||
|
||||
说明:
|
||||
- `DataCollection.load_from_persistence()` 仅恢复持久化状态;
|
||||
- 这里触发 E01:create_zombie -> S15(删除),其余 -> S00。
|
||||
"""
|
||||
return await run_phase2_cleanup(
|
||||
node_types=node_types,
|
||||
local_collection=local_collection,
|
||||
remote_collection=remote_collection,
|
||||
binding_manager=binding_manager,
|
||||
runtime=runtime,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
async def run_phase2_reset(
|
||||
cls,
|
||||
*,
|
||||
node_types: List[str],
|
||||
local_collection: DataCollection,
|
||||
remote_collection: DataCollection,
|
||||
binding_manager: BindingManager,
|
||||
runtime: StateMachineRuntime,
|
||||
) -> int:
|
||||
"""兼容旧命名,等价于 `run_reset`。"""
|
||||
return await cls.run_reset(
|
||||
node_types=node_types,
|
||||
local_collection=local_collection,
|
||||
remote_collection=remote_collection,
|
||||
binding_manager=binding_manager,
|
||||
runtime=runtime,
|
||||
)
|
||||
|
||||
|
||||
class DefaultSyncStrategy(BaseSyncStrategy[T]):
|
||||
"""
|
||||
基于 STATE_DEFINITIONS.md 实现的标准化同步策略。
|
||||
采用四阶段管线:核心状态 -> 依赖检查 -> 自动匹配 -> 动作映射。
|
||||
"""
|
||||
return normalized
|
||||
|
||||
async def bind(self, **kwargs):
|
||||
self.ensure_runtime()
|
||||
await run_bind(self, **kwargs)
|
||||
|
||||
|
||||
|
||||
def _get_id_field_hints(self) -> Dict[str, str]:
|
||||
"""
|
||||
获取 ID 字段的 node_type 提示。
|
||||
|
||||
默认实现:从 depend_fields 中提取。
|
||||
子类可以重写此方法提供更精确的映射。
|
||||
|
||||
Returns:
|
||||
字段名到 node_type 的映射,例如 {"project_id": "project"}
|
||||
"""
|
||||
return dict(self.config.depend_fields) if self.config.depend_fields else {}
|
||||
async def create(self) -> List[SyncNode]:
|
||||
self.ensure_runtime()
|
||||
return await run_create(self)
|
||||
@@ -331,6 +203,51 @@ class DefaultSyncStrategy(BaseSyncStrategy[T]):
|
||||
self.ensure_runtime()
|
||||
return await run_update(self)
|
||||
|
||||
def get_node_update_payload(self, node: SyncNode) -> Dict[str, Any]:
|
||||
return default_get_node_update_payload(node)
|
||||
|
||||
def should_update_pair(
|
||||
self,
|
||||
source_node: SyncNode,
|
||||
target_node: SyncNode,
|
||||
*,
|
||||
source_data: Dict[str, Any],
|
||||
target_data: Dict[str, Any],
|
||||
data_id_map: Optional[Dict[str, str]] = None,
|
||||
) -> bool:
|
||||
return default_needs_update(self, source_data, target_data, data_id_map)
|
||||
|
||||
async def collect_update_pairs(self):
|
||||
return await collect_bound_node_pairs(self)
|
||||
|
||||
async def prepare_update_for_direction(
|
||||
self,
|
||||
local_node: SyncNode,
|
||||
remote_node: SyncNode,
|
||||
direction: UpdateDirection,
|
||||
data_id_map: Optional[Dict[str, str]] = None,
|
||||
*,
|
||||
include_fields: Optional[List[str]] = None,
|
||||
exclude_fields: Optional[List[str]] = None,
|
||||
) -> Optional[SyncNode]:
|
||||
return await prepare_directional_update(
|
||||
self,
|
||||
local_node=local_node,
|
||||
remote_node=remote_node,
|
||||
direction=direction,
|
||||
data_id_map=data_id_map,
|
||||
include_fields=include_fields,
|
||||
exclude_fields=exclude_fields,
|
||||
)
|
||||
|
||||
async def update_pair(
|
||||
self,
|
||||
local_node: SyncNode,
|
||||
remote_node: SyncNode,
|
||||
data_id_map: Optional[Dict[str, str]] = None,
|
||||
) -> List[SyncNode]:
|
||||
return await default_update_pair(self, local_node, remote_node, data_id_map)
|
||||
|
||||
async def delete(self) -> List[SyncNode]:
|
||||
self.ensure_runtime()
|
||||
return await run_delete(self)
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
from .create_ops import add_create_action, run_create, finalize_peer_creating_sources
|
||||
from .update_ops import add_update_action, run_update
|
||||
from .update_ops import (
|
||||
BoundNodePair,
|
||||
add_update_action,
|
||||
collect_bound_node_pairs,
|
||||
prepare_directional_update,
|
||||
run_update,
|
||||
)
|
||||
from .delete_ops import run_delete
|
||||
from .bind_ops import (
|
||||
run_bind,
|
||||
@@ -10,13 +16,16 @@ from .bind_ops import (
|
||||
check_dependency_satisfied,
|
||||
collect_dependency_errors,
|
||||
)
|
||||
from .reset_ops import get_phase1_reset_defaults, run_phase2_cleanup
|
||||
from .reset_ops import run_phase2_cleanup
|
||||
|
||||
__all__ = [
|
||||
"add_create_action",
|
||||
"run_create",
|
||||
"finalize_peer_creating_sources",
|
||||
"BoundNodePair",
|
||||
"add_update_action",
|
||||
"collect_bound_node_pairs",
|
||||
"prepare_directional_update",
|
||||
"run_update",
|
||||
"run_delete",
|
||||
"run_bind",
|
||||
@@ -26,6 +35,5 @@ __all__ = [
|
||||
"phase_auto_binding",
|
||||
"check_dependency_satisfied",
|
||||
"collect_dependency_errors",
|
||||
"get_phase1_reset_defaults",
|
||||
"run_phase2_cleanup",
|
||||
]
|
||||
|
||||
@@ -16,6 +16,10 @@ if TYPE_CHECKING:
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_id_field_hints(strategy) -> Dict[str, str]:
|
||||
return dict(strategy.config.depend_fields) if strategy.config.depend_fields else {}
|
||||
|
||||
|
||||
def _stable_pick_bind_pair(local_candidates: List["SyncNode"], remote_candidates: List["SyncNode"]):
|
||||
local_node = sorted(local_candidates, key=lambda item: ((item.data_id or ""), item.node_id))[0]
|
||||
remote_node = sorted(remote_candidates, key=lambda item: ((item.data_id or ""), item.node_id))[0]
|
||||
@@ -399,7 +403,7 @@ async def phase_auto_binding(strategy, local_nodes: List["SyncNode"], remote_nod
|
||||
resolved_key_dict, resolve_error = await resolve_id_fields_in_key_dict(
|
||||
key_dict,
|
||||
id_resolver,
|
||||
strategy._get_id_field_hints(),
|
||||
get_id_field_hints(strategy),
|
||||
)
|
||||
if not resolved_key_dict:
|
||||
if node.binding_status == BindingStatus.MISSING:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List
|
||||
from typing import Any, Callable, Dict, List
|
||||
|
||||
from ...sync_system.resolve_ids import IDResolver
|
||||
from ...validation import SchemaDiffValidator
|
||||
@@ -27,11 +27,11 @@ def _append_post_check_error(node, message: str) -> None:
|
||||
async def evaluate_consistency_for_node_type(
|
||||
*,
|
||||
node_type: str,
|
||||
strategy,
|
||||
local_collection,
|
||||
remote_collection,
|
||||
binding_manager,
|
||||
logger,
|
||||
normalize_compare_payload: Callable[[Dict[str, Any] | None, Dict[str, str] | None], Dict[str, Any]],
|
||||
depend_fields: Dict[str, str] | None = None,
|
||||
compare_to_remote: bool = True,
|
||||
ignore_fields: List[str] | None = None,
|
||||
@@ -44,6 +44,7 @@ async def evaluate_consistency_for_node_type(
|
||||
schema_name=node_type,
|
||||
ignore_fields=ignore_fields,
|
||||
)
|
||||
mismatches: list[str] = []
|
||||
data_id_map = await build_data_id_normalization_map(
|
||||
node_type=node_type,
|
||||
local_collection=local_collection,
|
||||
@@ -93,8 +94,8 @@ async def evaluate_consistency_for_node_type(
|
||||
if resolved_remote is not None:
|
||||
remote_data = resolved_remote
|
||||
|
||||
local_payload = strategy.normalize_compare_payload(local_data, data_id_map)
|
||||
remote_payload = strategy.normalize_compare_payload(remote_data, data_id_map)
|
||||
local_payload = normalize_compare_payload(local_data, data_id_map)
|
||||
remote_payload = normalize_compare_payload(remote_data, data_id_map)
|
||||
diff_map = validator.compare_payloads(
|
||||
local_payload,
|
||||
remote_payload,
|
||||
@@ -129,6 +130,7 @@ async def evaluate_consistency_for_node_type(
|
||||
"fields": report["fields"],
|
||||
"schema_name": report["schema_name"],
|
||||
"ignore_fields": report["ignore_fields"],
|
||||
"mismatches": mismatches,
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
@@ -14,13 +14,6 @@ if TYPE_CHECKING:
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_phase1_reset_defaults() -> Dict[str, Any]:
|
||||
return {
|
||||
"binding_status": BindingStatus.UNCHECKED,
|
||||
"error": None,
|
||||
}
|
||||
|
||||
|
||||
async def _resolve_local_id_for_unbind(
|
||||
*,
|
||||
node_type: str,
|
||||
|
||||
@@ -1,17 +1,19 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
from dataclasses import dataclass
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Callable, List, Optional
|
||||
from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Sequence
|
||||
|
||||
from ...common.types import SyncAction
|
||||
from ...sync_system.config import UpdateDirection
|
||||
from ...sync_system.resolve_ids import IDResolver
|
||||
from ...engine import decision_note, e30_update_prepare
|
||||
from ...validation import SchemaDiffValidator
|
||||
from .compare_ops import (
|
||||
build_data_id_normalization_map,
|
||||
normalized_data_for_compare,
|
||||
collect_payload_diffs,
|
||||
normalized_data_for_compare,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -21,6 +23,36 @@ if TYPE_CHECKING:
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _build_schema_diff_validator(strategy) -> SchemaDiffValidator:
|
||||
schema = getattr(strategy, "schema", None)
|
||||
schema_name = getattr(schema, "__name__", None) or str(getattr(strategy, "node_type", "unknown"))
|
||||
return SchemaDiffValidator(
|
||||
schema_name=schema_name,
|
||||
ignore_fields=strategy.config.compare_ignore_fields,
|
||||
ignore_list_item_fields=strategy.config.post_check_ignore_list_item_fields,
|
||||
)
|
||||
|
||||
|
||||
def _get_active_schema_diff_validator(strategy) -> SchemaDiffValidator:
|
||||
validator = getattr(strategy, "_active_schema_diff_validator", None)
|
||||
if validator is not None:
|
||||
return validator
|
||||
return _build_schema_diff_validator(strategy)
|
||||
|
||||
|
||||
def emit_schema_diff_report(strategy, validator: SchemaDiffValidator) -> None:
|
||||
if not validator.has_records():
|
||||
return
|
||||
for line in validator.format_summary_lines():
|
||||
logger.info(f"[{strategy.node_type}] validate {line}")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BoundNodePair:
|
||||
local_node: "SyncNode"
|
||||
remote_node: "SyncNode"
|
||||
|
||||
|
||||
def _extract_snapshot_steps(snapshot) -> List:
|
||||
if not isinstance(snapshot, dict):
|
||||
return []
|
||||
@@ -41,16 +73,10 @@ def _safe_repr(value, *, max_len: int = 120) -> str:
|
||||
|
||||
def default_needs_update(
|
||||
strategy,
|
||||
source_node,
|
||||
target_node,
|
||||
resolved_data=None,
|
||||
source_data,
|
||||
target_data,
|
||||
data_id_map=None,
|
||||
) -> bool:
|
||||
if resolved_data is None:
|
||||
raise ValueError(f"[{strategy.node_type}] resolved_data is required for differential sync.")
|
||||
|
||||
source_data = resolved_data
|
||||
target_data = target_node.get_data()
|
||||
if source_data is None or target_data is None:
|
||||
return False
|
||||
|
||||
@@ -67,10 +93,10 @@ def default_needs_update(
|
||||
ignore_fields=ignore_fields,
|
||||
ignore_list_item_fields=strategy.config.post_check_ignore_list_item_fields,
|
||||
)
|
||||
diff_map = strategy.get_schema_diff_validator().compare_payloads(
|
||||
diff_map = _get_active_schema_diff_validator(strategy).compare_payloads(
|
||||
normalized_source,
|
||||
normalized_target,
|
||||
sample_id=str(target_node.data_id or target_node.node_id),
|
||||
sample_id=str(source_data.get("id") or target_data.get("id") or "unknown"),
|
||||
schema_refs=[strategy.schema.__name__],
|
||||
)
|
||||
is_different = bool(diff_map)
|
||||
@@ -82,8 +108,8 @@ def default_needs_update(
|
||||
return is_different
|
||||
|
||||
|
||||
def resolve_needs_update_check(strategy) -> Callable:
|
||||
return strategy._needs_update
|
||||
def default_get_node_update_payload(node: "SyncNode") -> dict[str, Any]:
|
||||
return node.get_data() or {}
|
||||
|
||||
|
||||
def _collect_diff_descriptions(
|
||||
@@ -125,235 +151,268 @@ def _collect_diff_descriptions(
|
||||
return descriptions
|
||||
|
||||
|
||||
async def collect_bound_node_pairs(strategy) -> List[BoundNodePair]:
|
||||
local_nodes = strategy.local_collection.filter_by_state_ids(
|
||||
node_type=strategy.node_type,
|
||||
state_ids=["S01"],
|
||||
)
|
||||
if not local_nodes:
|
||||
return []
|
||||
|
||||
pairs: List[BoundNodePair] = []
|
||||
for local_node in local_nodes:
|
||||
remote_node_id = await strategy.binding_manager.get_remote_id(strategy.node_type, local_node.node_id)
|
||||
if not remote_node_id:
|
||||
continue
|
||||
|
||||
remote_node = strategy.remote_collection.get_by_node_id(remote_node_id)
|
||||
if remote_node is None:
|
||||
continue
|
||||
|
||||
pairs.append(BoundNodePair(local_node=local_node, remote_node=remote_node))
|
||||
|
||||
return pairs
|
||||
|
||||
|
||||
async def default_update_pair(
|
||||
strategy,
|
||||
local_node: "SyncNode",
|
||||
remote_node: "SyncNode",
|
||||
data_id_map: Optional[dict[str, str]] = None,
|
||||
) -> List["SyncNode"]:
|
||||
direction = strategy.config.update_direction
|
||||
if direction == UpdateDirection.NONE:
|
||||
return []
|
||||
|
||||
updated_node = await prepare_directional_update(
|
||||
strategy,
|
||||
local_node=local_node,
|
||||
remote_node=remote_node,
|
||||
direction=direction,
|
||||
data_id_map=data_id_map,
|
||||
)
|
||||
return [updated_node] if updated_node is not None else []
|
||||
|
||||
|
||||
def _sync_pending_approval_snapshot(source_node: "SyncNode", target_node: "SyncNode") -> bool:
|
||||
source_snapshot = source_node.context.get("approval_snapshot") if isinstance(source_node.context, dict) else None
|
||||
target_snapshot = target_node.context.get("approval_snapshot") if isinstance(target_node.context, dict) else None
|
||||
if isinstance(source_snapshot, dict):
|
||||
steps_changed = _steps_payload_changed(source_snapshot, target_snapshot)
|
||||
pending_snapshot = copy.deepcopy(source_snapshot)
|
||||
pending_snapshot["steps_changed"] = steps_changed
|
||||
target_node.context["pending_approval_snapshot"] = pending_snapshot
|
||||
return steps_changed
|
||||
|
||||
target_node.context.pop("pending_approval_snapshot", None)
|
||||
return False
|
||||
|
||||
|
||||
def _select_fields(
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
include_fields: Optional[Sequence[str]] = None,
|
||||
exclude_fields: Optional[Sequence[str]] = None,
|
||||
) -> dict[str, Any]:
|
||||
if include_fields:
|
||||
include_set = set(include_fields)
|
||||
return {field_name: value for field_name, value in payload.items() if field_name in include_set}
|
||||
if exclude_fields:
|
||||
exclude_set = set(exclude_fields)
|
||||
return {field_name: value for field_name, value in payload.items() if field_name not in exclude_set}
|
||||
return dict(payload)
|
||||
|
||||
|
||||
def build_merged_update_payload(
|
||||
*,
|
||||
source_data: dict[str, Any],
|
||||
target_data: dict[str, Any],
|
||||
include_fields: Optional[Sequence[str]] = None,
|
||||
exclude_fields: Optional[Sequence[str]] = None,
|
||||
) -> dict[str, Any]:
|
||||
if include_fields:
|
||||
merged_payload = dict(target_data)
|
||||
for field_name, value in _select_fields(source_data, include_fields=include_fields).items():
|
||||
merged_payload[field_name] = value
|
||||
return merged_payload
|
||||
|
||||
if exclude_fields:
|
||||
exclude_set = set(exclude_fields)
|
||||
merged_payload = dict(source_data)
|
||||
for field_name, value in target_data.items():
|
||||
if field_name in exclude_set:
|
||||
merged_payload[field_name] = value
|
||||
return merged_payload
|
||||
|
||||
return dict(source_data)
|
||||
|
||||
|
||||
async def _resolve_update_source_payload(
|
||||
strategy,
|
||||
*,
|
||||
source_node: "SyncNode",
|
||||
source_is_local: bool,
|
||||
) -> tuple[Optional[dict[str, Any]], Optional[str]]:
|
||||
source_payload = strategy.get_node_update_payload(source_node)
|
||||
if not source_payload:
|
||||
return None, "Node data is missing or invalid"
|
||||
|
||||
id_resolver = IDResolver(strategy.local_collection, strategy.remote_collection, strategy.binding_manager)
|
||||
return await id_resolver.resolve_and_validate(
|
||||
data=source_payload,
|
||||
node_type=strategy.node_type,
|
||||
depend_fields=dict(strategy.config.depend_fields) if strategy.config.depend_fields else {},
|
||||
to_remote=source_is_local,
|
||||
)
|
||||
|
||||
|
||||
async def prepare_directional_update(
|
||||
strategy,
|
||||
*,
|
||||
local_node: "SyncNode",
|
||||
remote_node: "SyncNode",
|
||||
direction: UpdateDirection,
|
||||
data_id_map: Optional[dict] = None,
|
||||
include_fields: Optional[Sequence[str]] = None,
|
||||
exclude_fields: Optional[Sequence[str]] = None,
|
||||
) -> Optional["SyncNode"]:
|
||||
if direction == UpdateDirection.NONE:
|
||||
return None
|
||||
|
||||
source_is_local = direction == UpdateDirection.PUSH
|
||||
source_node = local_node if source_is_local else remote_node
|
||||
target_node = remote_node if source_is_local else local_node
|
||||
|
||||
if target_node.action == SyncAction.CREATE:
|
||||
return None
|
||||
|
||||
target_has_data_id = bool(target_node.data_id)
|
||||
resolved_source_data = None
|
||||
resolve_error = None
|
||||
if target_has_data_id:
|
||||
resolved_source_data, resolve_error = await _resolve_update_source_payload(
|
||||
strategy,
|
||||
source_node=source_node,
|
||||
source_is_local=source_is_local,
|
||||
)
|
||||
|
||||
update_id_resolve_ok = resolved_source_data is not None
|
||||
target_payload = strategy.get_node_update_payload(target_node)
|
||||
prepared_payload: dict[str, Any] | None = None
|
||||
has_diff = False
|
||||
diff_descriptions: List[str] = []
|
||||
|
||||
steps_changed = _sync_pending_approval_snapshot(source_node, target_node)
|
||||
|
||||
if update_id_resolve_ok and resolved_source_data is not None:
|
||||
prepared_payload = build_merged_update_payload(
|
||||
source_data=resolved_source_data,
|
||||
target_data=target_payload,
|
||||
include_fields=include_fields,
|
||||
exclude_fields=exclude_fields,
|
||||
)
|
||||
has_diff = bool(
|
||||
strategy.should_update_pair(
|
||||
source_node,
|
||||
target_node,
|
||||
source_data=prepared_payload,
|
||||
target_data=target_payload,
|
||||
data_id_map=data_id_map,
|
||||
)
|
||||
)
|
||||
if has_diff:
|
||||
diff_descriptions = _collect_diff_descriptions(
|
||||
prepared_payload,
|
||||
target_payload,
|
||||
data_id_map=data_id_map or {},
|
||||
ignore_fields=set(strategy.config.compare_ignore_fields),
|
||||
ignore_list_item_fields=dict(strategy.config.post_check_ignore_list_item_fields),
|
||||
)
|
||||
|
||||
if steps_changed:
|
||||
has_diff = True
|
||||
diff_descriptions.append("steps: approval flow changed")
|
||||
|
||||
if target_has_data_id and update_id_resolve_ok and not has_diff:
|
||||
return None
|
||||
|
||||
decision = e30_update_prepare(
|
||||
strategy.ensure_runtime(),
|
||||
source_node=source_node,
|
||||
target_node=target_node,
|
||||
update_enabled=True,
|
||||
target_has_data_id=target_has_data_id,
|
||||
update_id_resolve_ok=update_id_resolve_ok,
|
||||
has_diff=has_diff,
|
||||
id_resolve_detail=resolve_error,
|
||||
)
|
||||
reason = decision_note(decision)
|
||||
if decision.to_state == "S08":
|
||||
target_node.error = reason
|
||||
logger.debug(f"[{strategy.node_type}] 更新准备阻断: target={target_node.node_id}, reason={reason}")
|
||||
return None
|
||||
if decision.to_state != "S07":
|
||||
raise RuntimeError(f"[{strategy.node_type}] unexpected update_prepare state: {decision.to_state}")
|
||||
if prepared_payload is None:
|
||||
raise RuntimeError(f"[{strategy.node_type}] E30(S07) requires resolved update data")
|
||||
|
||||
target_node.set_data(prepared_payload)
|
||||
target_node.error = None
|
||||
diff_text = "; ".join(diff_descriptions) if diff_descriptions else "(未生成差异明细)"
|
||||
target_node.append_log(f"检测到差异: 从节点 {source_node.node_id} 更新数据 | fields={diff_text}")
|
||||
logger.info(f"[{strategy.node_type}] 更新差异字段: target={target_node.node_id}, {diff_text}")
|
||||
logger.debug(f"[{strategy.node_type}] 更新准备完成: target={target_node.node_id}, reason={reason}")
|
||||
return target_node
|
||||
|
||||
|
||||
async def add_update_action(
|
||||
strategy,
|
||||
*,
|
||||
from_collection: "DataCollection",
|
||||
to_collection: "DataCollection",
|
||||
needs_update_check,
|
||||
source_is_local: bool,
|
||||
data_id_map: Optional[dict] = None,
|
||||
node_filter: Optional[Callable[["SyncNode"], bool]] = None,
|
||||
include_fields: Optional[Iterable[str]] = None,
|
||||
exclude_fields: Optional[Iterable[str]] = None,
|
||||
) -> List["SyncNode"]:
|
||||
nodes_to_update = from_collection.filter_by_state_ids(
|
||||
node_type=strategy.node_type,
|
||||
state_ids=["S01"],
|
||||
)
|
||||
if not nodes_to_update:
|
||||
return []
|
||||
del from_collection
|
||||
del to_collection
|
||||
|
||||
if node_filter is not None:
|
||||
nodes_to_update = [n for n in nodes_to_update if node_filter(n)]
|
||||
if not nodes_to_update:
|
||||
return []
|
||||
|
||||
id_resolver = IDResolver(strategy.local_collection, strategy.remote_collection, strategy.binding_manager)
|
||||
updated_nodes: List["SyncNode"] = []
|
||||
total_nodes = len(nodes_to_update)
|
||||
|
||||
for idx, src_node in enumerate(nodes_to_update, start=1):
|
||||
if idx % 100 == 0 or idx == total_nodes:
|
||||
logger.debug(f"[{strategy.node_type}] 更新检查 {idx}/{total_nodes}")
|
||||
|
||||
src_data = src_node.get_data()
|
||||
if not src_data:
|
||||
continue
|
||||
|
||||
if source_is_local:
|
||||
target_node_id = await strategy.binding_manager.get_remote_id(strategy.node_type, src_node.node_id)
|
||||
else:
|
||||
target_node_id = await strategy.binding_manager.get_local_id(strategy.node_type, src_node.node_id)
|
||||
if not target_node_id:
|
||||
continue
|
||||
|
||||
target_node = to_collection.get_by_node_id(target_node_id)
|
||||
if not target_node or target_node.action == SyncAction.CREATE:
|
||||
continue
|
||||
|
||||
target_has_data_id = bool(target_node.data_id)
|
||||
data = None
|
||||
resolve_error = None
|
||||
if target_has_data_id:
|
||||
data, resolve_error = await id_resolver.resolve_and_validate(
|
||||
data=src_data,
|
||||
node_type=strategy.node_type,
|
||||
depend_fields=dict(strategy.config.depend_fields) if strategy.config.depend_fields else {},
|
||||
to_remote=source_is_local,
|
||||
)
|
||||
|
||||
update_id_resolve_ok = data is not None
|
||||
has_diff = False
|
||||
diff_descriptions: List[str] = []
|
||||
source_snapshot = src_node.context.get("approval_snapshot") if isinstance(src_node.context, dict) else None
|
||||
target_snapshot = target_node.context.get("approval_snapshot") if isinstance(target_node.context, dict) else None
|
||||
steps_changed = False
|
||||
if isinstance(source_snapshot, dict):
|
||||
steps_changed = _steps_payload_changed(source_snapshot, target_snapshot)
|
||||
pending_snapshot = copy.deepcopy(source_snapshot)
|
||||
pending_snapshot["steps_changed"] = steps_changed
|
||||
target_node.context["pending_approval_snapshot"] = pending_snapshot
|
||||
else:
|
||||
target_node.context.pop("pending_approval_snapshot", None)
|
||||
|
||||
if update_id_resolve_ok:
|
||||
try:
|
||||
should_update = needs_update_check(src_node, target_node, resolved_data=data, data_id_map=data_id_map)
|
||||
except TypeError:
|
||||
try:
|
||||
should_update = needs_update_check(src_node, target_node)
|
||||
except TypeError:
|
||||
should_update = needs_update_check(src_node)
|
||||
has_diff = bool(should_update)
|
||||
if has_diff:
|
||||
diff_descriptions = _collect_diff_descriptions(
|
||||
data,
|
||||
target_node.get_data(),
|
||||
data_id_map=data_id_map or {},
|
||||
ignore_fields=set(strategy.config.compare_ignore_fields),
|
||||
ignore_list_item_fields=dict(strategy.config.post_check_ignore_list_item_fields),
|
||||
)
|
||||
|
||||
if steps_changed:
|
||||
has_diff = True
|
||||
diff_descriptions.append("steps: approval flow changed")
|
||||
|
||||
if target_has_data_id and update_id_resolve_ok and not has_diff:
|
||||
continue
|
||||
|
||||
decision = e30_update_prepare(
|
||||
strategy.ensure_runtime(),
|
||||
source_node=src_node,
|
||||
target_node=target_node,
|
||||
update_enabled=True,
|
||||
target_has_data_id=target_has_data_id,
|
||||
update_id_resolve_ok=update_id_resolve_ok,
|
||||
has_diff=has_diff,
|
||||
id_resolve_detail=resolve_error,
|
||||
direction = UpdateDirection.PUSH if source_is_local else UpdateDirection.PULL
|
||||
for pair in await collect_bound_node_pairs(strategy):
|
||||
updated_node = await prepare_directional_update(
|
||||
strategy,
|
||||
local_node=pair.local_node,
|
||||
remote_node=pair.remote_node,
|
||||
direction=direction,
|
||||
data_id_map=data_id_map,
|
||||
include_fields=list(include_fields) if include_fields is not None else None,
|
||||
exclude_fields=list(exclude_fields) if exclude_fields is not None else None,
|
||||
)
|
||||
reason = decision_note(decision)
|
||||
if decision.to_state == "S08":
|
||||
target_node.error = reason
|
||||
logger.debug(f"[{strategy.node_type}] 更新准备阻断: target={target_node.node_id}, reason={reason}")
|
||||
continue
|
||||
if decision.to_state != "S07":
|
||||
raise RuntimeError(f"[{strategy.node_type}] unexpected update_prepare state: {decision.to_state}")
|
||||
if data is None:
|
||||
raise RuntimeError(f"[{strategy.node_type}] E30(S07) requires resolved update data")
|
||||
|
||||
target_node.set_data(data)
|
||||
target_node.error = None
|
||||
diff_text = "; ".join(diff_descriptions) if diff_descriptions else "(未生成差异明细)"
|
||||
target_node.append_log(f"检测到差异: 从节点 {src_node.node_id} 更新数据 | fields={diff_text}")
|
||||
logger.info(f"[{strategy.node_type}] 更新差异字段: target={target_node.node_id}, {diff_text}")
|
||||
logger.debug(f"[{strategy.node_type}] 更新准备完成: target={target_node.node_id}, reason={reason}")
|
||||
updated_nodes.append(target_node)
|
||||
if updated_node is not None:
|
||||
updated_nodes.append(updated_node)
|
||||
|
||||
return updated_nodes
|
||||
|
||||
|
||||
async def run_update(strategy) -> List["SyncNode"]:
|
||||
updated_nodes: List["SyncNode"] = []
|
||||
strategy.reset_schema_diff_validator()
|
||||
needs_update_check = resolve_needs_update_check(strategy)
|
||||
updated_by_id: dict[str, "SyncNode"] = {}
|
||||
validator = _build_schema_diff_validator(strategy)
|
||||
strategy._active_schema_diff_validator = validator
|
||||
data_id_map = await build_data_id_normalization_map(
|
||||
node_type=strategy.node_type,
|
||||
local_collection=strategy.local_collection,
|
||||
remote_collection=strategy.remote_collection,
|
||||
binding_manager=strategy.binding_manager,
|
||||
)
|
||||
try:
|
||||
for pair in await collect_bound_node_pairs(strategy):
|
||||
nodes = await strategy.update_pair(pair.local_node, pair.remote_node, data_id_map)
|
||||
for node in nodes:
|
||||
updated_by_id[node.node_id] = node
|
||||
finally:
|
||||
strategy._active_schema_diff_validator = None
|
||||
|
||||
field_key = strategy.config.field_direction_key
|
||||
overrides: dict = strategy.config.field_direction_overrides
|
||||
|
||||
if field_key and overrides:
|
||||
# 按方向归组:对每种出现的方向各调一次 add_update_action,传入过滤器。
|
||||
# 覆盖表中未命中的节点沿用 update_direction 作为默认方向。
|
||||
default_direction = strategy.config.update_direction
|
||||
active_directions = set(overrides.values()) | {default_direction}
|
||||
|
||||
for direction in (UpdateDirection.PUSH, UpdateDirection.PULL):
|
||||
if direction not in active_directions:
|
||||
continue
|
||||
|
||||
captured_direction = direction # 避免闭包捕获循环变量
|
||||
captured_overrides = overrides
|
||||
captured_default = default_direction
|
||||
captured_field_key = field_key
|
||||
|
||||
def make_filter(d=captured_direction, fk=captured_field_key, ov=captured_overrides, dflt=captured_default):
|
||||
def _filter(node: "SyncNode") -> bool:
|
||||
node_data = node.get_data() or {}
|
||||
key_val = node_data.get(fk)
|
||||
effective = ov.get(key_val, dflt)
|
||||
return effective == d
|
||||
return _filter
|
||||
|
||||
node_filter = make_filter()
|
||||
|
||||
if direction == UpdateDirection.PUSH:
|
||||
nodes = await add_update_action(
|
||||
strategy,
|
||||
from_collection=strategy.local_collection,
|
||||
to_collection=strategy.remote_collection,
|
||||
needs_update_check=needs_update_check,
|
||||
source_is_local=True,
|
||||
data_id_map=data_id_map,
|
||||
node_filter=node_filter,
|
||||
)
|
||||
updated_nodes.extend(nodes)
|
||||
logger.info(
|
||||
f"[{strategy.node_type}] Updated {len(nodes)} nodes in remote collection"
|
||||
f" (Push, field_direction_key={field_key})"
|
||||
)
|
||||
|
||||
elif direction == UpdateDirection.PULL:
|
||||
nodes = await add_update_action(
|
||||
strategy,
|
||||
from_collection=strategy.remote_collection,
|
||||
to_collection=strategy.local_collection,
|
||||
needs_update_check=needs_update_check,
|
||||
source_is_local=False,
|
||||
data_id_map=data_id_map,
|
||||
node_filter=node_filter,
|
||||
)
|
||||
updated_nodes.extend(nodes)
|
||||
logger.info(
|
||||
f"[{strategy.node_type}] Updated {len(nodes)} nodes in local collection"
|
||||
f" (Pull, field_direction_key={field_key})"
|
||||
)
|
||||
|
||||
strategy.emit_schema_diff_report()
|
||||
return updated_nodes
|
||||
|
||||
# ── 无字段级覆盖:原有逻辑不变 ──
|
||||
if strategy.config.update_direction == UpdateDirection.PUSH:
|
||||
nodes = await add_update_action(
|
||||
strategy,
|
||||
from_collection=strategy.local_collection,
|
||||
to_collection=strategy.remote_collection,
|
||||
needs_update_check=needs_update_check,
|
||||
source_is_local=True,
|
||||
data_id_map=data_id_map,
|
||||
)
|
||||
updated_nodes.extend(nodes)
|
||||
logger.info(f"[{strategy.node_type}] Updated {len(nodes)} nodes in remote collection (Push)")
|
||||
|
||||
elif strategy.config.update_direction == UpdateDirection.PULL:
|
||||
nodes = await add_update_action(
|
||||
strategy,
|
||||
from_collection=strategy.remote_collection,
|
||||
to_collection=strategy.local_collection,
|
||||
needs_update_check=needs_update_check,
|
||||
source_is_local=False,
|
||||
data_id_map=data_id_map,
|
||||
)
|
||||
updated_nodes.extend(nodes)
|
||||
logger.info(f"[{strategy.node_type}] Updated {len(nodes)} nodes in local collection (Pull)")
|
||||
|
||||
elif strategy.config.update_direction == UpdateDirection.BIDIRECTIONAL:
|
||||
raise NotImplementedError(
|
||||
f"[{strategy.node_type}] update_direction=BIDIRECTIONAL requires custom implementation"
|
||||
)
|
||||
|
||||
strategy.emit_schema_diff_report()
|
||||
return updated_nodes
|
||||
emit_schema_diff_report(strategy, validator)
|
||||
return list(updated_by_id.values())
|
||||
|
||||
Reference in New Issue
Block a user