strepsiades/ecm_sync_system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增强validator和打印区别

Browse Source
This commit is contained in:
strepsiades
2026-03-23 14:39:29 +08:00
parent 4c380f1157
commit 0812489f9d
13 changed files with 496 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sync_state_machine/sync_system/strategy_ops/compare_ops.py
+41 -5
View File
@@ -2,7 +2,7 @@ from __future__ import annotations
import copy
import inspect
from typing import Any, Dict, Optional
from typing import Any, Dict, Mapping, Optional
async def build_data_id_normalization_map(
@@ -39,14 +39,42 @@ def is_id_like_field(field_name: Optional[str]) -> bool:
return lowered.endswith("_id") or lowered.endswith("_ids")
def normalize_compare_value(field_name: Optional[str], value: Any, data_id_map: Dict[str, str]) -> Any:
def _strip_ignored_list_item_fields(
field_name: Optional[str],
value: Any,
ignore_list_item_fields: Mapping[str, list[str]],
) -> Any:
if not isinstance(value, list):
return value
ignored_keys = set(ignore_list_item_fields.get(field_name or "", []))
if not ignored_keys:
return value
normalized_items = []
for item in value:
if isinstance(item, dict):
normalized_items.append({key: nested for key, nested in item.items() if key not in ignored_keys})
else:
normalized_items.append(item)
return normalized_items
def normalize_compare_value(
field_name: Optional[str],
value: Any,
data_id_map: Dict[str, str],
ignore_list_item_fields: Optional[Mapping[str, list[str]]] = None,
) -> Any:
ignored_list_fields = ignore_list_item_fields or {}
if isinstance(value, dict):
return {
key: normalize_compare_value(key, nested, data_id_map)
key: normalize_compare_value(key, nested, data_id_map, ignored_list_fields)
for key, nested in value.items()
}
if isinstance(value, list):
return [normalize_compare_value(field_name, item, data_id_map) for item in value]
value = _strip_ignored_list_item_fields(field_name, value, ignored_list_fields)
return [normalize_compare_value(field_name, item, data_id_map, ignored_list_fields) for item in value]
if value == "":
value = None
@@ -59,14 +87,18 @@ def normalize_compare_value(field_name: Optional[str], value: Any, data_id_map:
def normalized_data_for_compare(
data: Optional[Dict[str, Any]],
data_id_map: Optional[Dict[str, str]] = None,
ignore_fields: Optional[set[str]] = None,
ignore_list_item_fields: Optional[Mapping[str, list[str]]] = None,
) -> Dict[str, Any]:
payload = copy.deepcopy(data or {})
payload.pop("id", None)
payload.pop("_id", None)
ignored = ignore_fields or set()
mapping = data_id_map or {}
return {
key: normalize_compare_value(key, value, mapping)
key: normalize_compare_value(key, value, mapping, ignore_list_item_fields)
for key, value in payload.items()
if key not in ignored
}
@@ -75,10 +107,14 @@ def collect_payload_diffs(
target_payload: Dict[str, Any],
*,
max_items: int = 6,
ignore_fields: Optional[set[str]] = None,
) -> Dict[str, Dict[str, Any]]:
out: Dict[str, Dict[str, Any]] = {}
ignored = ignore_fields or set()
keys = list(dict.fromkeys(list(source_payload.keys()) + list(target_payload.keys())))
for key in keys:
if key in ignored:
continue
source_value = source_payload.get(key)
target_value = target_payload.get(key)
if source_value == target_value:
sync_state_machine/sync_system/strategy_ops/post_check_ops.py
+31 -26
View File
@@ -3,10 +3,10 @@ from __future__ import annotations
from typing import Any, Dict, List
from ...sync_system.resolve_ids import IDResolver
from ...validation import SchemaDiffValidator
from .compare_ops import (
build_data_id_normalization_map,
normalized_data_for_compare,
collect_payload_diffs,
)
@@ -64,10 +64,10 @@ async def evaluate_consistency_for_node_type(
post_check_fields: List[str] | None = None,
) -> Dict[str, Any]:
records = await binding_manager.get_all_records(node_type)
mismatches: List[str] = []
checked_pairs = 0
depend_fields = dict(depend_fields or {})
id_resolver = IDResolver(local_collection, remote_collection, binding_manager)
validator = SchemaDiffValidator(schema_name=node_type)
data_id_map = await build_data_id_normalization_map(
node_type=node_type,
local_collection=local_collection,
@@ -125,35 +125,40 @@ async def evaluate_consistency_for_node_type(
if ignore_list_item_fields:
local_payload = _strip_list_item_fields(local_payload, ignore_list_item_fields)
remote_payload = _strip_list_item_fields(remote_payload, ignore_list_item_fields)
if local_payload != remote_payload:
diff_map = collect_payload_diffs(local_payload, remote_payload)
diff_parts = []
diff_map = validator.compare_payloads(
local_payload,
remote_payload,
sample_id=f"{local_id}->{remote_id}",
schema_refs=[node_type],
)
if diff_map:
detail_parts = []
for field_name, values in diff_map.items():
local_text = _compact_repr(values.get("source"))
remote_text = _compact_repr(values.get("target"))
diff_parts.append(f"{field_name}{{local:{local_text},remote:{remote_text}}}")
detail = "; ".join(diff_parts) if diff_parts else "payload differs"
mismatch_text = f"pair_diff(local_id={local_id}, remote_id={remote_id}, {detail})"
mismatches.append(mismatch_text)
local_text = _compact_repr(values.get("local"))
remote_text = _compact_repr(values.get("remote"))
detail_parts.append(f"{field_name}{{local:{local_text},remote:{remote_text}}}")
detail = "; ".join(detail_parts) if detail_parts else "payload differs"
_append_post_check_error(local_node, f"post_check diff: {detail}")
_append_post_check_error(remote_node, f"post_check diff: {detail}")
result = {
"ok": len(mismatches) == 0,
"checked_pairs": checked_pairs,
"mismatch_count": len(mismatches),
"samples": mismatches[:10],
report = validator.build_report() if checked_pairs else {
"schema_name": node_type,
"pairs_compared": 0,
"mismatched_pairs": 0,
"ignore_fields": [],
"fields": [],
}
if result["ok"]:
logger.info(f"✅ Consistency check passed: node_type={node_type}, checked_pairs={checked_pairs}")
else:
logger.error(
f"❌ Consistency check failed: node_type={node_type}, checked_pairs={checked_pairs}, mismatches={len(mismatches)}"
)
logger.error(" ↳ consistent语义: 比较绑定对(local_node.data vs remote_node.data)，非 original_data")
for item in result["samples"]:
logger.error(f" - {item}")
# 将字段统计补回到结果里，供 summary 输出。
report["fields"] = report.get("fields", [])
result = {
"ok": report["mismatched_pairs"] == 0,
"checked_pairs": checked_pairs,
"mismatch_count": report["mismatched_pairs"],
"fields": report["fields"],
"schema_name": report["schema_name"],
"ignore_fields": report["ignore_fields"],
}
return result
sync_state_machine/sync_system/strategy_ops/update_ops.py
+55 -27
View File
@@ -39,24 +39,6 @@ def _safe_repr(value, *, max_len: int = 120) -> str:
return text[: max_len - 3] + "..."
def _collect_diff_descriptions(source_data, target_data, *, max_items: int = 8) -> List[str]:
if source_data is None or target_data is None:
return []
descriptions: List[str] = []
for key, source_value in source_data.items():
if key in {"id", "_id"}:
continue
target_value = target_data.get(key)
if source_value == target_value:
continue
descriptions.append(f"{key}: {_safe_repr(target_value)} -> {_safe_repr(source_value)}")
if len(descriptions) >= max_items:
break
return descriptions
def default_needs_update(
strategy,
source_node,
@@ -72,13 +54,29 @@ def default_needs_update(
if source_data is None or target_data is None:
return False
normalized_source = normalized_data_for_compare(source_data, data_id_map)
normalized_target = normalized_data_for_compare(target_data, data_id_map)
diff_map = collect_payload_diffs(normalized_source, normalized_target, max_items=8)
ignore_fields = set(strategy.config.compare_ignore_fields)
normalized_source = normalized_data_for_compare(
source_data,
data_id_map,
ignore_fields=ignore_fields,
ignore_list_item_fields=strategy.config.post_check_ignore_list_item_fields,
)
normalized_target = normalized_data_for_compare(
target_data,
data_id_map,
ignore_fields=ignore_fields,
ignore_list_item_fields=strategy.config.post_check_ignore_list_item_fields,
)
diff_map = strategy.get_schema_diff_validator().compare_payloads(
normalized_source,
normalized_target,
sample_id=str(target_node.data_id or target_node.node_id),
schema_refs=[strategy.schema.__name__],
)
is_different = bool(diff_map)
if is_different and diff_map:
previews = [f"{key}: {values.get('source')} != {values.get('target')}" for key, values in list(diff_map.items())[:5]]
previews = [f"{key}: {values.get('local')} != {values.get('remote')}" for key, values in list(diff_map.items())[:5]]
logger.debug(f"[{strategy.node_type}] Node diff detected: {', '.join(previews)}")
return is_different
@@ -88,13 +86,36 @@ def resolve_needs_update_check(strategy) -> Callable:
return strategy._needs_update
def _collect_diff_descriptions(source_data, target_data, *, data_id_map, max_items: int = 8) -> List[str]:
def _collect_diff_descriptions(
source_data,
target_data,
*,
data_id_map,
ignore_fields: set[str],
ignore_list_item_fields: dict[str, list[str]],
max_items: int = 8,
) -> List[str]:
if source_data is None or target_data is None:
return []
normalized_source = normalized_data_for_compare(source_data, data_id_map)
normalized_target = normalized_data_for_compare(target_data, data_id_map)
diff_map = collect_payload_diffs(normalized_source, normalized_target, max_items=max_items)
normalized_source = normalized_data_for_compare(
source_data,
data_id_map,
ignore_fields=ignore_fields,
ignore_list_item_fields=ignore_list_item_fields,
)
normalized_target = normalized_data_for_compare(
target_data,
data_id_map,
ignore_fields=ignore_fields,
ignore_list_item_fields=ignore_list_item_fields,
)
diff_map = collect_payload_diffs(
normalized_source,
normalized_target,
max_items=max_items,
ignore_fields=ignore_fields,
)
descriptions: List[str] = []
for key, values in diff_map.items():
@@ -184,7 +205,13 @@ async def add_update_action(
should_update = needs_update_check(src_node)
has_diff = bool(should_update)
if has_diff:
diff_descriptions = _collect_diff_descriptions(data, target_node.get_data(), data_id_map=data_id_map or {})
diff_descriptions = _collect_diff_descriptions(
data,
target_node.get_data(),
data_id_map=data_id_map or {},
ignore_fields=set(strategy.config.compare_ignore_fields),
ignore_list_item_fields=dict(strategy.config.post_check_ignore_list_item_fields),
)
if steps_changed:
has_diff = True
@@ -226,6 +253,7 @@ async def add_update_action(
async def run_update(strategy) -> List["SyncNode"]:
updated_nodes: List["SyncNode"] = []
strategy.reset_schema_diff_validator()
needs_update_check = resolve_needs_update_check(strategy)
data_id_map = await build_data_id_normalization_map(
node_type=strategy.node_type,