strepsiades/ecm_sync_system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

整体同步代码

Browse Source
This commit is contained in:
strepsiades
2026-03-26 14:10:47 +08:00
parent 4b81b83fd8
commit f50dc1aed0
9 changed files with 188 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sync_state_machine/config/strategy_config.py
+5
View File
@@ -51,6 +51,11 @@ class StrategyConfig(BaseModel):
# 只影响最终一致性评估，不影响 create/update 的差异检测。
post_check_ignore_fields: List[str] = Field(default_factory=list)
# post-check 一致性比较时，忽略列表型字段内各元素的特定子键。
# 格式: {字段名: [子键, ...]}，如 {"plan_node": ["is_audit"]}。
# 用于排除后端硬写死、永远与本地不一致的字段，避免误报。
post_check_ignore_list_item_fields: Dict[str, List[str]] = Field(default_factory=dict)
# schema diff / validate 时忽略的顶层字段。
compare_ignore_fields: List[str] = Field(default_factory=list)
sync_state_machine/pipeline/full_sync_pipeline.py
+8 -1
View File
@@ -382,11 +382,18 @@ class FullSyncPipeline:
async def _evaluate_consistency_for_node_type(self, node_type: str) -> Dict[str, Any]:
strategy = next((item for item in self.strategies if item.node_type == node_type), None)
if strategy is None:
raise RuntimeError(f"Missing strategy for node_type={node_type}")
depend_fields = dict(strategy.config.depend_fields) if strategy and strategy.config.depend_fields else {}
compare_to_remote = True
if strategy and strategy.config.update_direction == UpdateDirection.PULL:
compare_to_remote = False
ignore_fields = list(strategy.config.post_check_ignore_fields) if strategy else []
ignore_fields = list(
dict.fromkeys([
*strategy.config.compare_ignore_fields,
*strategy.config.post_check_ignore_fields,
])
)
result = await evaluate_consistency_for_node_type(
node_type=node_type,
sync_state_machine/sync_system/strategy.py
+19 -18
View File
@@ -1,4 +1,5 @@
from typing import TypeVar, Generic, Type, Dict, List, Optional, Any
import copy
from typing import TypeVar, Generic, Type, Dict, List, Optional, Any
from pydantic import BaseModel
import logging
from pathlib import Path
@@ -187,28 +188,12 @@ class BaseSyncStrategy(Generic[T]):
self._validate_config()
logger.info(f"[{self.node_type}] Applied preset: {preset_name}")
def preprocess_compare_payload(self, payload: Dict[str, Any]) -> Dict[str, Any]:
return payload
def normalize_compare_payload(
self,
data: Optional[Dict[str, Any]],
data_id_map: Optional[Dict[str, str]] = None,
*,
ignore_fields: Optional[set[str]] = None,
) -> Dict[str, Any]:
prepared = self.preprocess_compare_payload(dict(data or {}))
return normalized_data_for_compare(
prepared,
data_id_map,
ignore_fields=ignore_fields,
)
def get_schema_diff_validator(self) -> SchemaDiffValidator:
if self._schema_diff_validator is None:
self._schema_diff_validator = SchemaDiffValidator(
schema_name=self.schema.__name__,
ignore_fields=self.config.compare_ignore_fields,
ignore_list_item_fields=self.config.post_check_ignore_list_item_fields,
)
return self._schema_diff_validator
@@ -222,6 +207,22 @@ class BaseSyncStrategy(Generic[T]):
for line in validator.format_summary_lines():
logger.info(f"[{self.node_type}] validate {line}")
def preprocess_compare_payload(self, payload: Dict[str, Any]) -> Dict[str, Any]:
return payload
def normalize_compare_payload(
self,
data: Optional[Dict[str, Any]],
data_id_map: Optional[Dict[str, str]] = None,
) -> Dict[str, Any]:
normalized = normalized_data_for_compare(
data,
data_id_map,
ignore_fields=set(self.config.compare_ignore_fields),
ignore_list_item_fields=self.config.post_check_ignore_list_item_fields,
)
return self.preprocess_compare_payload(copy.deepcopy(normalized))
async def bind(self, **kwargs):
"""进行绑定逻辑判定，设置 node.binding_status 和 node.action"""
raise NotImplementedError
sync_state_machine/sync_system/strategy_ops/compare_ops.py
+29 -4
View File
@@ -2,7 +2,7 @@ from __future__ import annotations
import copy
import inspect
from typing import Any, Dict, Optional
from typing import Any, Dict, Mapping, Optional
async def build_data_id_normalization_map(
@@ -39,18 +39,42 @@ def is_id_like_field(field_name: Optional[str]) -> bool:
return lowered.endswith("_id") or lowered.endswith("_ids")
def _strip_ignored_list_item_fields(
field_name: Optional[str],
value: Any,
ignore_list_item_fields: Mapping[str, list[str]],
) -> Any:
if not isinstance(value, list):
return value
ignored_keys = set(ignore_list_item_fields.get(field_name or "", []))
if not ignored_keys:
return value
normalized_items = []
for item in value:
if isinstance(item, dict):
normalized_items.append({key: nested for key, nested in item.items() if key not in ignored_keys})
else:
normalized_items.append(item)
return normalized_items
def normalize_compare_value(
field_name: Optional[str],
value: Any,
data_id_map: Dict[str, str],
ignore_list_item_fields: Optional[Mapping[str, list[str]]] = None,
) -> Any:
ignored_list_fields = ignore_list_item_fields or {}
if isinstance(value, dict):
return {
key: normalize_compare_value(key, nested, data_id_map)
key: normalize_compare_value(key, nested, data_id_map, ignored_list_fields)
for key, nested in value.items()
}
if isinstance(value, list):
return [normalize_compare_value(field_name, item, data_id_map) for item in value]
value = _strip_ignored_list_item_fields(field_name, value, ignored_list_fields)
return [normalize_compare_value(field_name, item, data_id_map, ignored_list_fields) for item in value]
if value == "":
value = None
@@ -64,6 +88,7 @@ def normalized_data_for_compare(
data: Optional[Dict[str, Any]],
data_id_map: Optional[Dict[str, str]] = None,
ignore_fields: Optional[set[str]] = None,
ignore_list_item_fields: Optional[Mapping[str, list[str]]] = None,
) -> Dict[str, Any]:
payload = copy.deepcopy(data or {})
payload.pop("id", None)
@@ -71,7 +96,7 @@ def normalized_data_for_compare(
ignored = ignore_fields or set()
mapping = data_id_map or {}
return {
key: normalize_compare_value(key, value, mapping)
key: normalize_compare_value(key, value, mapping, ignore_list_item_fields)
for key, value in payload.items()
if key not in ignored
}
sync_state_machine/sync_system/strategy_ops/update_ops.py
+12 -6
View File
@@ -55,15 +55,17 @@ def default_needs_update(
return False
ignore_fields = set(strategy.config.compare_ignore_fields)
normalized_source = strategy.normalize_compare_payload(
normalized_source = normalized_data_for_compare(
source_data,
data_id_map,
ignore_fields=ignore_fields,
ignore_list_item_fields=strategy.config.post_check_ignore_list_item_fields,
)
normalized_target = strategy.normalize_compare_payload(
normalized_target = normalized_data_for_compare(
target_data,
data_id_map,
ignore_fields=ignore_fields,
ignore_list_item_fields=strategy.config.post_check_ignore_list_item_fields,
)
diff_map = strategy.get_schema_diff_validator().compare_payloads(
normalized_source,
@@ -85,26 +87,28 @@ def resolve_needs_update_check(strategy) -> Callable:
def _collect_diff_descriptions(
strategy,
source_data,
target_data,
*,
data_id_map,
ignore_fields: set[str],
ignore_list_item_fields: dict[str, list[str]],
max_items: int = 8,
) -> List[str]:
if source_data is None or target_data is None:
return []
normalized_source = strategy.normalize_compare_payload(
normalized_source = normalized_data_for_compare(
source_data,
data_id_map,
ignore_fields=ignore_fields,
ignore_list_item_fields=ignore_list_item_fields,
)
normalized_target = strategy.normalize_compare_payload(
normalized_target = normalized_data_for_compare(
target_data,
data_id_map,
ignore_fields=ignore_fields,
ignore_list_item_fields=ignore_list_item_fields,
)
diff_map = collect_payload_diffs(
normalized_source,
@@ -202,11 +206,11 @@ async def add_update_action(
has_diff = bool(should_update)
if has_diff:
diff_descriptions = _collect_diff_descriptions(
strategy,
data,
target_node.get_data(),
data_id_map=data_id_map or {},
ignore_fields=set(strategy.config.compare_ignore_fields),
ignore_list_item_fields=dict(strategy.config.post_check_ignore_list_item_fields),
)
if steps_changed:
@@ -318,6 +322,7 @@ async def run_update(strategy) -> List["SyncNode"]:
f" (Pull, field_direction_key={field_key})"
)
strategy.emit_schema_diff_report()
return updated_nodes
# ── 无字段级覆盖：原有逻辑不变 ──
@@ -350,4 +355,5 @@ async def run_update(strategy) -> List["SyncNode"]:
f"[{strategy.node_type}] update_direction=BIDIRECTIONAL requires custom implementation"
)
strategy.emit_schema_diff_report()
return updated_nodes
sync_state_machine/validation/schema_diff.py
+5 -1
View File
@@ -1,7 +1,7 @@
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any, Iterable
from typing import Any, Iterable, Mapping
from ..sync_system.strategy_ops.compare_ops import normalized_data_for_compare
@@ -28,10 +28,12 @@ class SchemaDiffValidator:
*,
schema_name: str,
ignore_fields: Iterable[str] | None = None,
ignore_list_item_fields: Mapping[str, list[str]] | None = None,
sample_limit: int = 3,
) -> None:
self.schema_name = schema_name
self.ignore_fields = {str(field) for field in (ignore_fields or []) if str(field)}
self.ignore_list_item_fields = dict(ignore_list_item_fields or {})
self.sample_limit = max(1, sample_limit)
self.reset()
@@ -54,10 +56,12 @@ class SchemaDiffValidator:
local_normalized = normalized_data_for_compare(
dict(local_payload or {}),
ignore_fields=self.ignore_fields,
ignore_list_item_fields=self.ignore_list_item_fields,
)
remote_normalized = normalized_data_for_compare(
dict(remote_payload or {}),
ignore_fields=self.ignore_fields,
ignore_list_item_fields=self.ignore_list_item_fields,
)
field_names = sorted(set(local_normalized) | set(remote_normalized))