strepsiades/ecm_sync_system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

多pipeline同步支持

Browse Source
This commit is contained in:
strepsiades
2026-03-23 21:02:52 +08:00
parent 0812489f9d
commit b51ed0175b
49 changed files with 2110 additions and 792 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sync_state_machine/sync_system/strategy.py
+18 -1
View File
@@ -20,6 +20,7 @@ from ..engine import (
StateMachineRuntime,
)
from ..validation import SchemaDiffValidator
from .strategy_ops.compare_ops import normalized_data_for_compare
T = TypeVar("T", bound=BaseModel)
logger = logging.getLogger(__name__)
@@ -186,12 +187,28 @@ class BaseSyncStrategy(Generic[T]):
self._validate_config()
logger.info(f"[{self.node_type}] Applied preset: {preset_name}")
def preprocess_compare_payload(self, payload: Dict[str, Any]) -> Dict[str, Any]:
return payload
def normalize_compare_payload(
self,
data: Optional[Dict[str, Any]],
data_id_map: Optional[Dict[str, str]] = None,
*,
ignore_fields: Optional[set[str]] = None,
) -> Dict[str, Any]:
prepared = self.preprocess_compare_payload(dict(data or {}))
return normalized_data_for_compare(
prepared,
data_id_map,
ignore_fields=ignore_fields,
)
def get_schema_diff_validator(self) -> SchemaDiffValidator:
if self._schema_diff_validator is None:
self._schema_diff_validator = SchemaDiffValidator(
schema_name=self.schema.__name__,
ignore_fields=self.config.compare_ignore_fields,
ignore_list_item_fields=self.config.post_check_ignore_list_item_fields,
)
return self._schema_diff_validator
sync_state_machine/sync_system/strategy_ops/compare_ops.py
+4 -29
View File
@@ -2,7 +2,7 @@ from __future__ import annotations
import copy
import inspect
from typing import Any, Dict, Mapping, Optional
from typing import Any, Dict, Optional
async def build_data_id_normalization_map(
@@ -39,42 +39,18 @@ def is_id_like_field(field_name: Optional[str]) -> bool:
return lowered.endswith("_id") or lowered.endswith("_ids")
def _strip_ignored_list_item_fields(
field_name: Optional[str],
value: Any,
ignore_list_item_fields: Mapping[str, list[str]],
) -> Any:
if not isinstance(value, list):
return value
ignored_keys = set(ignore_list_item_fields.get(field_name or "", []))
if not ignored_keys:
return value
normalized_items = []
for item in value:
if isinstance(item, dict):
normalized_items.append({key: nested for key, nested in item.items() if key not in ignored_keys})
else:
normalized_items.append(item)
return normalized_items
def normalize_compare_value(
field_name: Optional[str],
value: Any,
data_id_map: Dict[str, str],
ignore_list_item_fields: Optional[Mapping[str, list[str]]] = None,
) -> Any:
ignored_list_fields = ignore_list_item_fields or {}
if isinstance(value, dict):
return {
key: normalize_compare_value(key, nested, data_id_map, ignored_list_fields)
key: normalize_compare_value(key, nested, data_id_map)
for key, nested in value.items()
}
if isinstance(value, list):
value = _strip_ignored_list_item_fields(field_name, value, ignored_list_fields)
return [normalize_compare_value(field_name, item, data_id_map, ignored_list_fields) for item in value]
return [normalize_compare_value(field_name, item, data_id_map) for item in value]
if value == "":
value = None
@@ -88,7 +64,6 @@ def normalized_data_for_compare(
data: Optional[Dict[str, Any]],
data_id_map: Optional[Dict[str, str]] = None,
ignore_fields: Optional[set[str]] = None,
ignore_list_item_fields: Optional[Mapping[str, list[str]]] = None,
) -> Dict[str, Any]:
payload = copy.deepcopy(data or {})
payload.pop("id", None)
@@ -96,7 +71,7 @@ def normalized_data_for_compare(
ignored = ignore_fields or set()
mapping = data_id_map or {}
return {
key: normalize_compare_value(key, value, mapping, ignore_list_item_fields)
key: normalize_compare_value(key, value, mapping)
for key, value in payload.items()
if key not in ignored
}
sync_state_machine/sync_system/strategy_ops/post_check_ops.py
+9 -39
View File
@@ -4,34 +4,7 @@ from typing import Any, Dict, List
from ...sync_system.resolve_ids import IDResolver
from ...validation import SchemaDiffValidator
from .compare_ops import (
build_data_id_normalization_map,
normalized_data_for_compare,
)
def _strip_list_item_fields(payload: Dict, ignore_map: Dict[str, List[str]]) -> Dict:
"""从 payload 中列表型字段的每个元素里，剔除指定子键。
例如 ignore_map={"plan_node": ["is_audit"]} 会把 payload["plan_node"][*]["is_audit"] 全部去掉。
payload 本身不会被修改（浅拷贝）。
"""
if not ignore_map:
return payload
result = dict(payload)
for field_name, sub_keys in ignore_map.items():
if field_name not in result:
continue
items = result[field_name]
if not isinstance(items, list):
continue
stripped = []
for item in items:
if isinstance(item, dict):
item = {k: v for k, v in item.items() if k not in sub_keys}
stripped.append(item)
result[field_name] = stripped
return result
from .compare_ops import build_data_id_normalization_map
def _compact_repr(value: Any, *, max_len: int = 80) -> str:
@@ -54,20 +27,23 @@ def _append_post_check_error(node, message: str) -> None:
async def evaluate_consistency_for_node_type(
*,
node_type: str,
strategy,
local_collection,
remote_collection,
binding_manager,
logger,
depend_fields: Dict[str, str] | None = None,
compare_to_remote: bool = True,
ignore_list_item_fields: Dict[str, List[str]] | None = None,
post_check_fields: List[str] | None = None,
ignore_fields: List[str] | None = None,
) -> Dict[str, Any]:
records = await binding_manager.get_all_records(node_type)
checked_pairs = 0
depend_fields = dict(depend_fields or {})
id_resolver = IDResolver(local_collection, remote_collection, binding_manager)
validator = SchemaDiffValidator(schema_name=node_type)
validator = SchemaDiffValidator(
schema_name=node_type,
ignore_fields=ignore_fields,
)
data_id_map = await build_data_id_normalization_map(
node_type=node_type,
local_collection=local_collection,
@@ -117,14 +93,8 @@ async def evaluate_consistency_for_node_type(
if resolved_remote is not None:
remote_data = resolved_remote
local_payload = normalized_data_for_compare(local_data, data_id_map)
remote_payload = normalized_data_for_compare(remote_data, data_id_map)
if post_check_fields:
local_payload = {k: v for k, v in local_payload.items() if k in post_check_fields}
remote_payload = {k: v for k, v in remote_payload.items() if k in post_check_fields}
if ignore_list_item_fields:
local_payload = _strip_list_item_fields(local_payload, ignore_list_item_fields)
remote_payload = _strip_list_item_fields(remote_payload, ignore_list_item_fields)
local_payload = strategy.normalize_compare_payload(local_data, data_id_map)
remote_payload = strategy.normalize_compare_payload(remote_data, data_id_map)
diff_map = validator.compare_payloads(
local_payload,
remote_payload,
sync_state_machine/sync_system/strategy_ops/update_ops.py
+6 -10
View File
@@ -55,17 +55,15 @@ def default_needs_update(
return False
ignore_fields = set(strategy.config.compare_ignore_fields)
normalized_source = normalized_data_for_compare(
normalized_source = strategy.normalize_compare_payload(
source_data,
data_id_map,
ignore_fields=ignore_fields,
ignore_list_item_fields=strategy.config.post_check_ignore_list_item_fields,
)
normalized_target = normalized_data_for_compare(
normalized_target = strategy.normalize_compare_payload(
target_data,
data_id_map,
ignore_fields=ignore_fields,
ignore_list_item_fields=strategy.config.post_check_ignore_list_item_fields,
)
diff_map = strategy.get_schema_diff_validator().compare_payloads(
normalized_source,
@@ -87,28 +85,26 @@ def resolve_needs_update_check(strategy) -> Callable:
def _collect_diff_descriptions(
strategy,
source_data,
target_data,
*,
data_id_map,
ignore_fields: set[str],
ignore_list_item_fields: dict[str, list[str]],
max_items: int = 8,
) -> List[str]:
if source_data is None or target_data is None:
return []
normalized_source = normalized_data_for_compare(
normalized_source = strategy.normalize_compare_payload(
source_data,
data_id_map,
ignore_fields=ignore_fields,
ignore_list_item_fields=ignore_list_item_fields,
)
normalized_target = normalized_data_for_compare(
normalized_target = strategy.normalize_compare_payload(
target_data,
data_id_map,
ignore_fields=ignore_fields,
ignore_list_item_fields=ignore_list_item_fields,
)
diff_map = collect_payload_diffs(
normalized_source,
@@ -206,11 +202,11 @@ async def add_update_action(
has_diff = bool(should_update)
if has_diff:
diff_descriptions = _collect_diff_descriptions(
strategy,
data,
target_node.get_data(),
data_id_map=data_id_map or {},
ignore_fields=set(strategy.config.compare_ignore_fields),
ignore_list_item_fields=dict(strategy.config.post_check_ignore_list_item_fields),
)
if steps_changed: