strepsiades/ecm_sync_system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

优化sync_system代码质量

Browse Source
This commit is contained in:
strepsiades
2026-04-01 11:52:57 +08:00
parent eb02fd32a5
commit 8f4727a772
46 changed files with 1431 additions and 890 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sync_state_machine/sync_system/strategy.py
+164 -247
View File
@@ -1,4 +1,4 @@
import copy
from abc import ABC, abstractmethod
from typing import TypeVar, Generic, Type, Dict, List, Optional, Any
from pydantic import BaseModel
import logging
@@ -7,31 +7,35 @@ from pathlib import Path
from ..common.collection import DataCollection
from ..common.binding import BindingManager
from ..common.sync_node import SyncNode
from .config import StrategyConfig, OrphanAction, UpdateDirection, ConfigPresets
from .config import StrategyConfig, OrphanAction, UpdateDirection
from ..config.strategy_config import resolve_domain_option_config
from .strategy_ops import (
get_phase1_reset_defaults,
run_bind,
run_create,
run_delete,
run_phase2_cleanup,
)
from .strategy_ops.update_ops import (
collect_bound_node_pairs,
default_get_node_update_payload,
default_needs_update,
default_update_pair,
prepare_directional_update,
run_update,
)
from ..engine import (
StateMachineConfig,
StateMachineRuntime,
)
from ..validation import SchemaDiffValidator
from .strategy_ops.compare_ops import normalized_data_for_compare
T = TypeVar("T", bound=BaseModel)
logger = logging.getLogger(__name__)
class BaseSyncStrategy(Generic[T]):
class BaseSyncStrategy(ABC, Generic[T]):
"""
同步策略基类。
纯逻辑层，不负责 IO 执行。
同步策略抽象接口。
子类可以通过覆盖 default_config 来设置默认配置：
子类可以通过覆盖以下类变量提供静态配置：
```python
class ProjectStrategy(DefaultSyncStrategy):
default_config = StrategyConfig(
@@ -47,41 +51,122 @@ class BaseSyncStrategy(Generic[T]):
适用于只需要加载数据供其他类型引用的场景（如 supplier）
skip_post_check: bool - 如果为 True，则跳过该类型最终 post-check 的 reload 与一致性校验
"""
# 类变量：默认配置（子类可覆盖）
default_config: StrategyConfig = StrategyConfig()
# 类变量：schema 类型（子类必须设置）
schema: Type[T] = None # type: ignore
# 类变量：是否默认跳过同步（子类可覆盖）
default_skip_sync: bool = False
# 类变量：是否默认跳过 post-check（子类可覆盖）
default_skip_post_check: bool = False
default_config: StrategyConfig
domain_option_model: Type[BaseModel] | None
schema: Type[T]
node_type: str
config: StrategyConfig
sm_runtime: Optional[StateMachineRuntime]
@abstractmethod
def __init__(
self,
node_type: str,
local_collection: DataCollection,
remote_collection: DataCollection,
self,
node_type: str,
local_collection: DataCollection,
remote_collection: DataCollection,
binding_manager: BindingManager,
):
pass
@abstractmethod
async def bind(self, **kwargs):
"""进行绑定逻辑判定，设置 node.binding_status 和 node.action"""
pass
@abstractmethod
async def create(self) -> List[SyncNode]:
"""准备 CREATE 操作"""
pass
@abstractmethod
async def update(self) -> List[SyncNode]:
"""准备 UPDATE 操作"""
pass
@abstractmethod
async def delete(self) -> List[SyncNode]:
pass
@property
@abstractmethod
def skip_sync(self) -> bool:
pass
@property
@abstractmethod
def skip_post_check(self) -> bool:
pass
@abstractmethod
def normalize_compare_payload(
self,
data: Optional[Dict[str, Any]],
data_id_map: Optional[Dict[str, str]] = None,
) -> Dict[str, Any]:
pass
class DefaultSyncStrategy(BaseSyncStrategy[T]):
"""
基于 STATE_DEFINITIONS.md 实现的标准化同步策略。
采用四阶段管线：核心状态 -> 依赖检查 -> 自动匹配 -> 动作映射。
"""
default_config: StrategyConfig = StrategyConfig()
domain_option_model: Type[BaseModel] | None = None
schema: Type[T] = None # type: ignore
def __init__(
self,
node_type: str,
local_collection: DataCollection,
remote_collection: DataCollection,
binding_manager: BindingManager,
):
if self.schema is None:
raise ValueError(f"{self.__class__.__name__} must define schema as a class variable")
self.node_type = node_type
self.local_collection = local_collection
self.remote_collection = remote_collection
self.binding_manager = binding_manager
# 复制类级别的默认配置到实例
self.config = self.default_config.model_copy(deep=True)
# 是否跳过同步（可运行时修改）
self.skip_sync = self.default_skip_sync
# 是否跳过 post-check（可运行时修改）
self.skip_post_check = self.default_skip_post_check
# 统一由 Pipeline 注入 runtime；未注入时在执行阶段显式报错
self.sm_runtime: Optional[StateMachineRuntime] = None
self._schema_diff_validator: Optional[SchemaDiffValidator] = None
# 配置逻辑校验
self._validate_config()
self.config.domain_option = resolve_domain_option_config(
self.domain_option_model,
self.config.domain_option,
)
self.config.log_logic_warnings(node_type=self.node_type, logger=logger)
@classmethod
def get_default_domain_option(cls) -> Dict[str, Any]:
return resolve_domain_option_config(cls.domain_option_model, None)
def _validate_domain_option(self, domain_option: Optional[Dict[str, Any]]) -> BaseModel | Dict[str, Any]:
if self.domain_option_model is None:
return dict(domain_option or {})
return self.domain_option_model.model_validate(domain_option or {})
@property
def domain_option(self) -> BaseModel | Dict[str, Any]:
return self._validate_domain_option(self.config.domain_option)
@property
def skip_sync(self) -> bool:
return bool(self.config.skip_sync)
@skip_sync.setter
def skip_sync(self, value: bool) -> None:
self.config.skip_sync = bool(value)
@property
def skip_post_check(self) -> bool:
return bool(self.config.skip_post_check)
@skip_post_check.setter
def skip_post_check(self, value: bool) -> None:
self.config.skip_post_check = bool(value)
def ensure_runtime(self) -> StateMachineRuntime:
runtime = self.sm_runtime
@@ -93,128 +178,6 @@ class BaseSyncStrategy(Generic[T]):
self.remote_collection.set_state_machine_runtime(runtime)
return runtime
def _validate_config(self) -> None:
"""
校验配置的逻辑一致性。
规则：
- PUSH 模式：local_orphan_action 应为 CREATE_REMOTE，remote_orphan_action 应为 NONE
- PULL 模式：local_orphan_action 应为 NONE，remote_orphan_action 应为 CREATE_LOCAL
- NONE 模式：两者都应为 NONE
"""
direction = self.config.update_direction
local_action = self.config.local_orphan_action
remote_action = self.config.remote_orphan_action
# PUSH 模式检查
if direction == UpdateDirection.PUSH:
if local_action == OrphanAction.CREATE_LOCAL:
logger.warning(
f"[{self.node_type}] Config inconsistency: "
f"update_direction=PUSH but local_orphan_action=CREATE_LOCAL (本地创建)。"
f"应使用 CREATE_REMOTE (推送到远程)"
)
if remote_action == OrphanAction.CREATE_LOCAL:
logger.warning(
f"[{self.node_type}] Config inconsistency: "
f"update_direction=PUSH but remote_orphan_action=CREATE_LOCAL (本地创建)。"
f"PUSH 模式应设为 NONE 或 DELETE_REMOTE"
)
# PULL 模式检查
elif direction == UpdateDirection.PULL:
if local_action == OrphanAction.CREATE_REMOTE:
logger.warning(
f"[{self.node_type}] Config inconsistency: "
f"update_direction=PULL but local_orphan_action=CREATE_REMOTE (推送到远程)。"
f"PULL 模式应设为 NONE 或 DELETE_LOCAL"
)
if remote_action in {OrphanAction.CREATE_REMOTE, OrphanAction.DELETE_LOCAL}:
logger.warning(
f"[{self.node_type}] Config inconsistency: "
f"update_direction=PULL but remote_orphan_action={remote_action}。"
f"PULL 模式建议使用 CREATE_LOCAL 或 NONE"
)
# NONE 模式检查
elif direction == UpdateDirection.NONE:
if local_action != OrphanAction.NONE:
logger.warning(
f"[{self.node_type}] Config inconsistency: "
f"update_direction=NONE but local_orphan_action={local_action}。"
f"应设为 NONE"
)
if remote_action != OrphanAction.NONE:
logger.warning(
f"[{self.node_type}] Config inconsistency: "
f"update_direction=NONE but remote_orphan_action={remote_action}。"
f"应设为 NONE"
)
def set_config(self, config: StrategyConfig):
"""完全替换配置对象"""
self.config = config
self._schema_diff_validator = None
self._validate_config()
def update_config(self, **kwargs):
"""
临时修改部分配置项。
Example:
strategy.update_config(
local_orphan_action=OrphanAction.NONE,
update_direction=UpdateDirection.PULL
)
"""
valid_fields = type(self.config).model_fields
for key, value in kwargs.items():
if key in valid_fields:
setattr(self.config, key, value)
else:
logger.warning(f"[{self.node_type}] Unknown config key: {key}")
self._schema_diff_validator = None
# 更新配置后也要校验
self._validate_config()
def apply_preset(self, preset_name: str):
"""
应用配置预设。
Args:
preset_name: 预设名称，可选：first_sync, daily_sync, repair_sync, pull_only, push_only
Example:
strategy.apply_preset("first_sync")
"""
preset_config = ConfigPresets.get_preset(preset_name)
self.config = preset_config
self._schema_diff_validator = None
self._validate_config()
logger.info(f"[{self.node_type}] Applied preset: {preset_name}")
def get_schema_diff_validator(self) -> SchemaDiffValidator:
if self._schema_diff_validator is None:
self._schema_diff_validator = SchemaDiffValidator(
schema_name=self.schema.__name__,
ignore_fields=self.config.compare_ignore_fields,
ignore_list_item_fields=self.config.post_check_ignore_list_item_fields,
)
return self._schema_diff_validator
def reset_schema_diff_validator(self) -> None:
self.get_schema_diff_validator().reset()
def emit_schema_diff_report(self) -> None:
validator = self.get_schema_diff_validator()
if not validator.has_records():
return
for line in validator.format_summary_lines():
logger.info(f"[{self.node_type}] validate {line}")
def preprocess_compare_payload(self, payload: Dict[str, Any]) -> Dict[str, Any]:
return payload
def normalize_compare_payload(
self,
data: Optional[Dict[str, Any]],
@@ -226,103 +189,12 @@ class BaseSyncStrategy(Generic[T]):
ignore_fields=set(self.config.compare_ignore_fields),
ignore_list_item_fields=self.config.post_check_ignore_list_item_fields,
)
return self.preprocess_compare_payload(copy.deepcopy(normalized))
async def bind(self, **kwargs):
"""进行绑定逻辑判定，设置 node.binding_status 和 node.action"""
raise NotImplementedError
async def create(self) -> List[SyncNode]:
"""准备 CREATE 操作"""
raise NotImplementedError
async def update(self) -> List[SyncNode]:
"""准备 UPDATE 操作"""
raise NotImplementedError
def _needs_update(
self,
source_node: SyncNode,
target_node: SyncNode,
resolved_data: Optional[Dict[str, Any]] = None,
data_id_map: Optional[Dict[str, str]] = None,
) -> bool:
from .strategy_ops.update_ops import default_needs_update
return default_needs_update(self, source_node, target_node, resolved_data, data_id_map)
@classmethod
def get_phase1_reset_defaults(cls) -> Dict[str, Any]:
return get_phase1_reset_defaults()
@classmethod
async def run_reset(
cls,
*,
node_types: List[str],
local_collection: DataCollection,
remote_collection: DataCollection,
binding_manager: BindingManager,
runtime: StateMachineRuntime,
) -> int:
"""
执行加载后的重置清理（僵尸 CREATE 节点清理）。
说明：
- `DataCollection.load_from_persistence()` 仅恢复持久化状态；
- 这里触发 E01：create_zombie -> S15（删除），其余 -> S00。
"""
return await run_phase2_cleanup(
node_types=node_types,
local_collection=local_collection,
remote_collection=remote_collection,
binding_manager=binding_manager,
runtime=runtime,
)
@classmethod
async def run_phase2_reset(
cls,
*,
node_types: List[str],
local_collection: DataCollection,
remote_collection: DataCollection,
binding_manager: BindingManager,
runtime: StateMachineRuntime,
) -> int:
"""兼容旧命名，等价于 `run_reset`。"""
return await cls.run_reset(
node_types=node_types,
local_collection=local_collection,
remote_collection=remote_collection,
binding_manager=binding_manager,
runtime=runtime,
)
class DefaultSyncStrategy(BaseSyncStrategy[T]):
"""
基于 STATE_DEFINITIONS.md 实现的标准化同步策略。
采用四阶段管线：核心状态 -> 依赖检查 -> 自动匹配 -> 动作映射。
"""
return normalized
async def bind(self, **kwargs):
self.ensure_runtime()
await run_bind(self, **kwargs)
def _get_id_field_hints(self) -> Dict[str, str]:
"""
获取 ID 字段的 node_type 提示。
默认实现：从 depend_fields 中提取。
子类可以重写此方法提供更精确的映射。
Returns:
字段名到 node_type 的映射，例如 {"project_id": "project"}
"""
return dict(self.config.depend_fields) if self.config.depend_fields else {}
async def create(self) -> List[SyncNode]:
self.ensure_runtime()
return await run_create(self)
@@ -331,6 +203,51 @@ class DefaultSyncStrategy(BaseSyncStrategy[T]):
self.ensure_runtime()
return await run_update(self)
def get_node_update_payload(self, node: SyncNode) -> Dict[str, Any]:
return default_get_node_update_payload(node)
def should_update_pair(
self,
source_node: SyncNode,
target_node: SyncNode,
*,
source_data: Dict[str, Any],
target_data: Dict[str, Any],
data_id_map: Optional[Dict[str, str]] = None,
) -> bool:
return default_needs_update(self, source_data, target_data, data_id_map)
async def collect_update_pairs(self):
return await collect_bound_node_pairs(self)
async def prepare_update_for_direction(
self,
local_node: SyncNode,
remote_node: SyncNode,
direction: UpdateDirection,
data_id_map: Optional[Dict[str, str]] = None,
*,
include_fields: Optional[List[str]] = None,
exclude_fields: Optional[List[str]] = None,
) -> Optional[SyncNode]:
return await prepare_directional_update(
self,
local_node=local_node,
remote_node=remote_node,
direction=direction,
data_id_map=data_id_map,
include_fields=include_fields,
exclude_fields=exclude_fields,
)
async def update_pair(
self,
local_node: SyncNode,
remote_node: SyncNode,
data_id_map: Optional[Dict[str, str]] = None,
) -> List[SyncNode]:
return await default_update_pair(self, local_node, remote_node, data_id_map)
async def delete(self) -> List[SyncNode]:
self.ensure_runtime()
return await run_delete(self)
sync_state_machine/sync_system/strategy_ops/__init__.py
+11 -3
View File
@@ -1,5 +1,11 @@
from .create_ops import add_create_action, run_create, finalize_peer_creating_sources
from .update_ops import add_update_action, run_update
from .update_ops import (
BoundNodePair,
add_update_action,
collect_bound_node_pairs,
prepare_directional_update,
run_update,
)
from .delete_ops import run_delete
from .bind_ops import (
run_bind,
@@ -10,13 +16,16 @@ from .bind_ops import (
check_dependency_satisfied,
collect_dependency_errors,
)
from .reset_ops import get_phase1_reset_defaults, run_phase2_cleanup
from .reset_ops import run_phase2_cleanup
__all__ = [
"add_create_action",
"run_create",
"finalize_peer_creating_sources",
"BoundNodePair",
"add_update_action",
"collect_bound_node_pairs",
"prepare_directional_update",
"run_update",
"run_delete",
"run_bind",
@@ -26,6 +35,5 @@ __all__ = [
"phase_auto_binding",
"check_dependency_satisfied",
"collect_dependency_errors",
"get_phase1_reset_defaults",
"run_phase2_cleanup",
]
sync_state_machine/sync_system/strategy_ops/bind_ops.py
+5 -1
View File
@@ -16,6 +16,10 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
def get_id_field_hints(strategy) -> Dict[str, str]:
return dict(strategy.config.depend_fields) if strategy.config.depend_fields else {}
def _stable_pick_bind_pair(local_candidates: List["SyncNode"], remote_candidates: List["SyncNode"]):
local_node = sorted(local_candidates, key=lambda item: ((item.data_id or ""), item.node_id))[0]
remote_node = sorted(remote_candidates, key=lambda item: ((item.data_id or ""), item.node_id))[0]
@@ -399,7 +403,7 @@ async def phase_auto_binding(strategy, local_nodes: List["SyncNode"], remote_nod
resolved_key_dict, resolve_error = await resolve_id_fields_in_key_dict(
key_dict,
id_resolver,
strategy._get_id_field_hints(),
get_id_field_hints(strategy),
)
if not resolved_key_dict:
if node.binding_status == BindingStatus.MISSING:
sync_state_machine/sync_system/strategy_ops/post_check_ops.py
+6 -4
View File
@@ -1,6 +1,6 @@
from __future__ import annotations
from typing import Any, Dict, List
from typing import Any, Callable, Dict, List
from ...sync_system.resolve_ids import IDResolver
from ...validation import SchemaDiffValidator
@@ -27,11 +27,11 @@ def _append_post_check_error(node, message: str) -> None:
async def evaluate_consistency_for_node_type(
*,
node_type: str,
strategy,
local_collection,
remote_collection,
binding_manager,
logger,
normalize_compare_payload: Callable[[Dict[str, Any] | None, Dict[str, str] | None], Dict[str, Any]],
depend_fields: Dict[str, str] | None = None,
compare_to_remote: bool = True,
ignore_fields: List[str] | None = None,
@@ -44,6 +44,7 @@ async def evaluate_consistency_for_node_type(
schema_name=node_type,
ignore_fields=ignore_fields,
)
mismatches: list[str] = []
data_id_map = await build_data_id_normalization_map(
node_type=node_type,
local_collection=local_collection,
@@ -93,8 +94,8 @@ async def evaluate_consistency_for_node_type(
if resolved_remote is not None:
remote_data = resolved_remote
local_payload = strategy.normalize_compare_payload(local_data, data_id_map)
remote_payload = strategy.normalize_compare_payload(remote_data, data_id_map)
local_payload = normalize_compare_payload(local_data, data_id_map)
remote_payload = normalize_compare_payload(remote_data, data_id_map)
diff_map = validator.compare_payloads(
local_payload,
remote_payload,
@@ -129,6 +130,7 @@ async def evaluate_consistency_for_node_type(
"fields": report["fields"],
"schema_name": report["schema_name"],
"ignore_fields": report["ignore_fields"],
"mismatches": mismatches,
}
return result
sync_state_machine/sync_system/strategy_ops/reset_ops.py
-7
View File
@@ -14,13 +14,6 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
def get_phase1_reset_defaults() -> Dict[str, Any]:
return {
"binding_status": BindingStatus.UNCHECKED,
"error": None,
}
async def _resolve_local_id_for_unbind(
*,
node_type: str,
sync_state_machine/sync_system/strategy_ops/update_ops.py
+282 -223
View File
@@ -1,17 +1,19 @@
from __future__ import annotations
import copy
from dataclasses import dataclass
import logging
from typing import TYPE_CHECKING, Callable, List, Optional
from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Sequence
from ...common.types import SyncAction
from ...sync_system.config import UpdateDirection
from ...sync_system.resolve_ids import IDResolver
from ...engine import decision_note, e30_update_prepare
from ...validation import SchemaDiffValidator
from .compare_ops import (
build_data_id_normalization_map,
normalized_data_for_compare,
collect_payload_diffs,
normalized_data_for_compare,
)
if TYPE_CHECKING:
@@ -21,6 +23,36 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
def _build_schema_diff_validator(strategy) -> SchemaDiffValidator:
schema = getattr(strategy, "schema", None)
schema_name = getattr(schema, "__name__", None) or str(getattr(strategy, "node_type", "unknown"))
return SchemaDiffValidator(
schema_name=schema_name,
ignore_fields=strategy.config.compare_ignore_fields,
ignore_list_item_fields=strategy.config.post_check_ignore_list_item_fields,
)
def _get_active_schema_diff_validator(strategy) -> SchemaDiffValidator:
validator = getattr(strategy, "_active_schema_diff_validator", None)
if validator is not None:
return validator
return _build_schema_diff_validator(strategy)
def emit_schema_diff_report(strategy, validator: SchemaDiffValidator) -> None:
if not validator.has_records():
return
for line in validator.format_summary_lines():
logger.info(f"[{strategy.node_type}] validate {line}")
@dataclass(frozen=True)
class BoundNodePair:
local_node: "SyncNode"
remote_node: "SyncNode"
def _extract_snapshot_steps(snapshot) -> List:
if not isinstance(snapshot, dict):
return []
@@ -41,16 +73,10 @@ def _safe_repr(value, *, max_len: int = 120) -> str:
def default_needs_update(
strategy,
source_node,
target_node,
resolved_data=None,
source_data,
target_data,
data_id_map=None,
) -> bool:
if resolved_data is None:
raise ValueError(f"[{strategy.node_type}] resolved_data is required for differential sync.")
source_data = resolved_data
target_data = target_node.get_data()
if source_data is None or target_data is None:
return False
@@ -67,10 +93,10 @@ def default_needs_update(
ignore_fields=ignore_fields,
ignore_list_item_fields=strategy.config.post_check_ignore_list_item_fields,
)
diff_map = strategy.get_schema_diff_validator().compare_payloads(
diff_map = _get_active_schema_diff_validator(strategy).compare_payloads(
normalized_source,
normalized_target,
sample_id=str(target_node.data_id or target_node.node_id),
sample_id=str(source_data.get("id") or target_data.get("id") or "unknown"),
schema_refs=[strategy.schema.__name__],
)
is_different = bool(diff_map)
@@ -82,8 +108,8 @@ def default_needs_update(
return is_different
def resolve_needs_update_check(strategy) -> Callable:
return strategy._needs_update
def default_get_node_update_payload(node: "SyncNode") -> dict[str, Any]:
return node.get_data() or {}
def _collect_diff_descriptions(
@@ -125,235 +151,268 @@ def _collect_diff_descriptions(
return descriptions
async def collect_bound_node_pairs(strategy) -> List[BoundNodePair]:
local_nodes = strategy.local_collection.filter_by_state_ids(
node_type=strategy.node_type,
state_ids=["S01"],
)
if not local_nodes:
return []
pairs: List[BoundNodePair] = []
for local_node in local_nodes:
remote_node_id = await strategy.binding_manager.get_remote_id(strategy.node_type, local_node.node_id)
if not remote_node_id:
continue
remote_node = strategy.remote_collection.get_by_node_id(remote_node_id)
if remote_node is None:
continue
pairs.append(BoundNodePair(local_node=local_node, remote_node=remote_node))
return pairs
async def default_update_pair(
strategy,
local_node: "SyncNode",
remote_node: "SyncNode",
data_id_map: Optional[dict[str, str]] = None,
) -> List["SyncNode"]:
direction = strategy.config.update_direction
if direction == UpdateDirection.NONE:
return []
updated_node = await prepare_directional_update(
strategy,
local_node=local_node,
remote_node=remote_node,
direction=direction,
data_id_map=data_id_map,
)
return [updated_node] if updated_node is not None else []
def _sync_pending_approval_snapshot(source_node: "SyncNode", target_node: "SyncNode") -> bool:
source_snapshot = source_node.context.get("approval_snapshot") if isinstance(source_node.context, dict) else None
target_snapshot = target_node.context.get("approval_snapshot") if isinstance(target_node.context, dict) else None
if isinstance(source_snapshot, dict):
steps_changed = _steps_payload_changed(source_snapshot, target_snapshot)
pending_snapshot = copy.deepcopy(source_snapshot)
pending_snapshot["steps_changed"] = steps_changed
target_node.context["pending_approval_snapshot"] = pending_snapshot
return steps_changed
target_node.context.pop("pending_approval_snapshot", None)
return False
def _select_fields(
payload: dict[str, Any],
*,
include_fields: Optional[Sequence[str]] = None,
exclude_fields: Optional[Sequence[str]] = None,
) -> dict[str, Any]:
if include_fields:
include_set = set(include_fields)
return {field_name: value for field_name, value in payload.items() if field_name in include_set}
if exclude_fields:
exclude_set = set(exclude_fields)
return {field_name: value for field_name, value in payload.items() if field_name not in exclude_set}
return dict(payload)
def build_merged_update_payload(
*,
source_data: dict[str, Any],
target_data: dict[str, Any],
include_fields: Optional[Sequence[str]] = None,
exclude_fields: Optional[Sequence[str]] = None,
) -> dict[str, Any]:
if include_fields:
merged_payload = dict(target_data)
for field_name, value in _select_fields(source_data, include_fields=include_fields).items():
merged_payload[field_name] = value
return merged_payload
if exclude_fields:
exclude_set = set(exclude_fields)
merged_payload = dict(source_data)
for field_name, value in target_data.items():
if field_name in exclude_set:
merged_payload[field_name] = value
return merged_payload
return dict(source_data)
async def _resolve_update_source_payload(
strategy,
*,
source_node: "SyncNode",
source_is_local: bool,
) -> tuple[Optional[dict[str, Any]], Optional[str]]:
source_payload = strategy.get_node_update_payload(source_node)
if not source_payload:
return None, "Node data is missing or invalid"
id_resolver = IDResolver(strategy.local_collection, strategy.remote_collection, strategy.binding_manager)
return await id_resolver.resolve_and_validate(
data=source_payload,
node_type=strategy.node_type,
depend_fields=dict(strategy.config.depend_fields) if strategy.config.depend_fields else {},
to_remote=source_is_local,
)
async def prepare_directional_update(
strategy,
*,
local_node: "SyncNode",
remote_node: "SyncNode",
direction: UpdateDirection,
data_id_map: Optional[dict] = None,
include_fields: Optional[Sequence[str]] = None,
exclude_fields: Optional[Sequence[str]] = None,
) -> Optional["SyncNode"]:
if direction == UpdateDirection.NONE:
return None
source_is_local = direction == UpdateDirection.PUSH
source_node = local_node if source_is_local else remote_node
target_node = remote_node if source_is_local else local_node
if target_node.action == SyncAction.CREATE:
return None
target_has_data_id = bool(target_node.data_id)
resolved_source_data = None
resolve_error = None
if target_has_data_id:
resolved_source_data, resolve_error = await _resolve_update_source_payload(
strategy,
source_node=source_node,
source_is_local=source_is_local,
)
update_id_resolve_ok = resolved_source_data is not None
target_payload = strategy.get_node_update_payload(target_node)
prepared_payload: dict[str, Any] | None = None
has_diff = False
diff_descriptions: List[str] = []
steps_changed = _sync_pending_approval_snapshot(source_node, target_node)
if update_id_resolve_ok and resolved_source_data is not None:
prepared_payload = build_merged_update_payload(
source_data=resolved_source_data,
target_data=target_payload,
include_fields=include_fields,
exclude_fields=exclude_fields,
)
has_diff = bool(
strategy.should_update_pair(
source_node,
target_node,
source_data=prepared_payload,
target_data=target_payload,
data_id_map=data_id_map,
)
)
if has_diff:
diff_descriptions = _collect_diff_descriptions(
prepared_payload,
target_payload,
data_id_map=data_id_map or {},
ignore_fields=set(strategy.config.compare_ignore_fields),
ignore_list_item_fields=dict(strategy.config.post_check_ignore_list_item_fields),
)
if steps_changed:
has_diff = True
diff_descriptions.append("steps: approval flow changed")
if target_has_data_id and update_id_resolve_ok and not has_diff:
return None
decision = e30_update_prepare(
strategy.ensure_runtime(),
source_node=source_node,
target_node=target_node,
update_enabled=True,
target_has_data_id=target_has_data_id,
update_id_resolve_ok=update_id_resolve_ok,
has_diff=has_diff,
id_resolve_detail=resolve_error,
)
reason = decision_note(decision)
if decision.to_state == "S08":
target_node.error = reason
logger.debug(f"[{strategy.node_type}] 更新准备阻断: target={target_node.node_id}, reason={reason}")
return None
if decision.to_state != "S07":
raise RuntimeError(f"[{strategy.node_type}] unexpected update_prepare state: {decision.to_state}")
if prepared_payload is None:
raise RuntimeError(f"[{strategy.node_type}] E30(S07) requires resolved update data")
target_node.set_data(prepared_payload)
target_node.error = None
diff_text = "; ".join(diff_descriptions) if diff_descriptions else "(未生成差异明细)"
target_node.append_log(f"检测到差异: 从节点 {source_node.node_id} 更新数据 | fields={diff_text}")
logger.info(f"[{strategy.node_type}] 更新差异字段: target={target_node.node_id}, {diff_text}")
logger.debug(f"[{strategy.node_type}] 更新准备完成: target={target_node.node_id}, reason={reason}")
return target_node
async def add_update_action(
strategy,
*,
from_collection: "DataCollection",
to_collection: "DataCollection",
needs_update_check,
source_is_local: bool,
data_id_map: Optional[dict] = None,
node_filter: Optional[Callable[["SyncNode"], bool]] = None,
include_fields: Optional[Iterable[str]] = None,
exclude_fields: Optional[Iterable[str]] = None,
) -> List["SyncNode"]:
nodes_to_update = from_collection.filter_by_state_ids(
node_type=strategy.node_type,
state_ids=["S01"],
)
if not nodes_to_update:
return []
del from_collection
del to_collection
if node_filter is not None:
nodes_to_update = [n for n in nodes_to_update if node_filter(n)]
if not nodes_to_update:
return []
id_resolver = IDResolver(strategy.local_collection, strategy.remote_collection, strategy.binding_manager)
updated_nodes: List["SyncNode"] = []
total_nodes = len(nodes_to_update)
for idx, src_node in enumerate(nodes_to_update, start=1):
if idx % 100 == 0 or idx == total_nodes:
logger.debug(f"[{strategy.node_type}] 更新检查 {idx}/{total_nodes}")
src_data = src_node.get_data()
if not src_data:
continue
if source_is_local:
target_node_id = await strategy.binding_manager.get_remote_id(strategy.node_type, src_node.node_id)
else:
target_node_id = await strategy.binding_manager.get_local_id(strategy.node_type, src_node.node_id)
if not target_node_id:
continue
target_node = to_collection.get_by_node_id(target_node_id)
if not target_node or target_node.action == SyncAction.CREATE:
continue
target_has_data_id = bool(target_node.data_id)
data = None
resolve_error = None
if target_has_data_id:
data, resolve_error = await id_resolver.resolve_and_validate(
data=src_data,
node_type=strategy.node_type,
depend_fields=dict(strategy.config.depend_fields) if strategy.config.depend_fields else {},
to_remote=source_is_local,
)
update_id_resolve_ok = data is not None
has_diff = False
diff_descriptions: List[str] = []
source_snapshot = src_node.context.get("approval_snapshot") if isinstance(src_node.context, dict) else None
target_snapshot = target_node.context.get("approval_snapshot") if isinstance(target_node.context, dict) else None
steps_changed = False
if isinstance(source_snapshot, dict):
steps_changed = _steps_payload_changed(source_snapshot, target_snapshot)
pending_snapshot = copy.deepcopy(source_snapshot)
pending_snapshot["steps_changed"] = steps_changed
target_node.context["pending_approval_snapshot"] = pending_snapshot
else:
target_node.context.pop("pending_approval_snapshot", None)
if update_id_resolve_ok:
try:
should_update = needs_update_check(src_node, target_node, resolved_data=data, data_id_map=data_id_map)
except TypeError:
try:
should_update = needs_update_check(src_node, target_node)
except TypeError:
should_update = needs_update_check(src_node)
has_diff = bool(should_update)
if has_diff:
diff_descriptions = _collect_diff_descriptions(
data,
target_node.get_data(),
data_id_map=data_id_map or {},
ignore_fields=set(strategy.config.compare_ignore_fields),
ignore_list_item_fields=dict(strategy.config.post_check_ignore_list_item_fields),
)
if steps_changed:
has_diff = True
diff_descriptions.append("steps: approval flow changed")
if target_has_data_id and update_id_resolve_ok and not has_diff:
continue
decision = e30_update_prepare(
strategy.ensure_runtime(),
source_node=src_node,
target_node=target_node,
update_enabled=True,
target_has_data_id=target_has_data_id,
update_id_resolve_ok=update_id_resolve_ok,
has_diff=has_diff,
id_resolve_detail=resolve_error,
direction = UpdateDirection.PUSH if source_is_local else UpdateDirection.PULL
for pair in await collect_bound_node_pairs(strategy):
updated_node = await prepare_directional_update(
strategy,
local_node=pair.local_node,
remote_node=pair.remote_node,
direction=direction,
data_id_map=data_id_map,
include_fields=list(include_fields) if include_fields is not None else None,
exclude_fields=list(exclude_fields) if exclude_fields is not None else None,
)
reason = decision_note(decision)
if decision.to_state == "S08":
target_node.error = reason
logger.debug(f"[{strategy.node_type}] 更新准备阻断: target={target_node.node_id}, reason={reason}")
continue
if decision.to_state != "S07":
raise RuntimeError(f"[{strategy.node_type}] unexpected update_prepare state: {decision.to_state}")
if data is None:
raise RuntimeError(f"[{strategy.node_type}] E30(S07) requires resolved update data")
target_node.set_data(data)
target_node.error = None
diff_text = "; ".join(diff_descriptions) if diff_descriptions else "(未生成差异明细)"
target_node.append_log(f"检测到差异: 从节点 {src_node.node_id} 更新数据 | fields={diff_text}")
logger.info(f"[{strategy.node_type}] 更新差异字段: target={target_node.node_id}, {diff_text}")
logger.debug(f"[{strategy.node_type}] 更新准备完成: target={target_node.node_id}, reason={reason}")
updated_nodes.append(target_node)
if updated_node is not None:
updated_nodes.append(updated_node)
return updated_nodes
async def run_update(strategy) -> List["SyncNode"]:
updated_nodes: List["SyncNode"] = []
strategy.reset_schema_diff_validator()
needs_update_check = resolve_needs_update_check(strategy)
updated_by_id: dict[str, "SyncNode"] = {}
validator = _build_schema_diff_validator(strategy)
strategy._active_schema_diff_validator = validator
data_id_map = await build_data_id_normalization_map(
node_type=strategy.node_type,
local_collection=strategy.local_collection,
remote_collection=strategy.remote_collection,
binding_manager=strategy.binding_manager,
)
try:
for pair in await collect_bound_node_pairs(strategy):
nodes = await strategy.update_pair(pair.local_node, pair.remote_node, data_id_map)
for node in nodes:
updated_by_id[node.node_id] = node
finally:
strategy._active_schema_diff_validator = None
field_key = strategy.config.field_direction_key
overrides: dict = strategy.config.field_direction_overrides
if field_key and overrides:
# 按方向归组：对每种出现的方向各调一次 add_update_action，传入过滤器。
# 覆盖表中未命中的节点沿用 update_direction 作为默认方向。
default_direction = strategy.config.update_direction
active_directions = set(overrides.values()) | {default_direction}
for direction in (UpdateDirection.PUSH, UpdateDirection.PULL):
if direction not in active_directions:
continue
captured_direction = direction # 避免闭包捕获循环变量
captured_overrides = overrides
captured_default = default_direction
captured_field_key = field_key
def make_filter(d=captured_direction, fk=captured_field_key, ov=captured_overrides, dflt=captured_default):
def _filter(node: "SyncNode") -> bool:
node_data = node.get_data() or {}
key_val = node_data.get(fk)
effective = ov.get(key_val, dflt)
return effective == d
return _filter
node_filter = make_filter()
if direction == UpdateDirection.PUSH:
nodes = await add_update_action(
strategy,
from_collection=strategy.local_collection,
to_collection=strategy.remote_collection,
needs_update_check=needs_update_check,
source_is_local=True,
data_id_map=data_id_map,
node_filter=node_filter,
)
updated_nodes.extend(nodes)
logger.info(
f"[{strategy.node_type}] Updated {len(nodes)} nodes in remote collection"
f" (Push, field_direction_key={field_key})"
)
elif direction == UpdateDirection.PULL:
nodes = await add_update_action(
strategy,
from_collection=strategy.remote_collection,
to_collection=strategy.local_collection,
needs_update_check=needs_update_check,
source_is_local=False,
data_id_map=data_id_map,
node_filter=node_filter,
)
updated_nodes.extend(nodes)
logger.info(
f"[{strategy.node_type}] Updated {len(nodes)} nodes in local collection"
f" (Pull, field_direction_key={field_key})"
)
strategy.emit_schema_diff_report()
return updated_nodes
# ── 无字段级覆盖：原有逻辑不变 ──
if strategy.config.update_direction == UpdateDirection.PUSH:
nodes = await add_update_action(
strategy,
from_collection=strategy.local_collection,
to_collection=strategy.remote_collection,
needs_update_check=needs_update_check,
source_is_local=True,
data_id_map=data_id_map,
)
updated_nodes.extend(nodes)
logger.info(f"[{strategy.node_type}] Updated {len(nodes)} nodes in remote collection (Push)")
elif strategy.config.update_direction == UpdateDirection.PULL:
nodes = await add_update_action(
strategy,
from_collection=strategy.remote_collection,
to_collection=strategy.local_collection,
needs_update_check=needs_update_check,
source_is_local=False,
data_id_map=data_id_map,
)
updated_nodes.extend(nodes)
logger.info(f"[{strategy.node_type}] Updated {len(nodes)} nodes in local collection (Pull)")
elif strategy.config.update_direction == UpdateDirection.BIDIRECTIONAL:
raise NotImplementedError(
f"[{strategy.node_type}] update_direction=BIDIRECTIONAL requires custom implementation"
)
strategy.emit_schema_diff_report()
return updated_nodes
emit_schema_diff_report(strategy, validator)
return list(updated_by_id.values())