增强validator和打印区别
This commit is contained in:
@@ -52,6 +52,9 @@ class StrategyConfig(BaseModel):
|
||||
# 用于排除后端硬写死、永远与本地不一致的字段,避免误报。
|
||||
post_check_ignore_list_item_fields: Dict[str, List[str]] = Field(default_factory=dict)
|
||||
|
||||
# schema diff / validate 时忽略的顶层字段。
|
||||
compare_ignore_fields: List[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class StrategyRuntimeConfig(BaseModel):
|
||||
"""Pipeline 层策略运行配置(按 node_type)。"""
|
||||
|
||||
@@ -20,7 +20,7 @@ from typing import ClassVar, Dict, List, Any, Optional, TYPE_CHECKING, TypeVar
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ...common.type_safety import get_pydantic_model_fields
|
||||
from ..handler import NodeHandler
|
||||
from ..handler import BaseNodeHandler
|
||||
from ..task_result import TaskResult
|
||||
from ..handler import ValidationResult # Import ValidationResult
|
||||
|
||||
@@ -32,7 +32,7 @@ if TYPE_CHECKING:
|
||||
T = TypeVar("T", bound=BaseModel)
|
||||
|
||||
|
||||
class BaseApiHandler(NodeHandler[T]):
|
||||
class BaseApiHandler(BaseNodeHandler[T]):
|
||||
"""API Handler 基类"""
|
||||
|
||||
# 子类需要设置这些属性
|
||||
@@ -40,15 +40,19 @@ class BaseApiHandler(NodeHandler[T]):
|
||||
_node_class: type['SyncNode']
|
||||
_schema: Any
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self, **handler_config: Any):
|
||||
# 子类应该在调用 super().__init__() 之前或之后设置 _node_type, _node_class, _schema
|
||||
super().__init__()
|
||||
self.api_client: 'ApiClient' = None # type: ignore # 由 DataSource 注入
|
||||
self.poll_mode: str = "async" # async | sync
|
||||
self._collection: 'DataCollection' = None # type: ignore # 由 DataSource 注入
|
||||
self.handler_config: Dict[str, Any] = {}
|
||||
self.handler_config = dict(handler_config)
|
||||
# 子类在 __init__ 中设置此列表以声明 create/update 中涉及的 Pydantic schema 类;
|
||||
# post-check 将只比较这些 schema 覆盖的字段,过滤后端只读的派生字段。
|
||||
self.update_schemas: List[type] = []
|
||||
|
||||
def set_collection(self, collection: 'DataCollection') -> None:
|
||||
self._collection = collection
|
||||
|
||||
def get_update_fields(self, *, exclude: frozenset = frozenset({'id'})) -> List[str]:
|
||||
"""从 update_schemas 中提取所有可写字段名,排除标识键(默认排除 id)。"""
|
||||
@@ -66,17 +70,6 @@ class BaseApiHandler(NodeHandler[T]):
|
||||
"""设置 Collection(由 DataSource 调用)"""
|
||||
self._collection = collection
|
||||
|
||||
def set_handler_config(self, config: Dict[str, Any]) -> None:
|
||||
self.handler_config = dict(config)
|
||||
|
||||
def get_data_id_filter(self) -> List[str]:
|
||||
value = self.handler_config.get("data_id_filter", [])
|
||||
if isinstance(value, str):
|
||||
return [value] if value else []
|
||||
if isinstance(value, list):
|
||||
return [str(item) for item in value if str(item)]
|
||||
return []
|
||||
|
||||
@property
|
||||
def node_type(self) -> str:
|
||||
"""返回节点类型"""
|
||||
|
||||
@@ -337,11 +337,11 @@ class BaseNodeHandler(ABC, Generic[T]):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
node_type: str,
|
||||
node_class: Type["SyncNode[T]"],
|
||||
schema: Type[T]
|
||||
node_type: str | None = None,
|
||||
node_class: Type["SyncNode[T]"] | None = None,
|
||||
schema: Type[T] | None = None,
|
||||
):
|
||||
self._node_type = node_type
|
||||
self._node_type = node_type or ""
|
||||
self._node_class = node_class
|
||||
self._schema = schema
|
||||
self._collection: Optional["DataCollection"] = None
|
||||
@@ -368,6 +368,18 @@ class BaseNodeHandler(ABC, Generic[T]):
|
||||
return [str(item) for item in value if str(item)]
|
||||
return []
|
||||
|
||||
def should_skip_by_data_id_filter(self, item: Dict[str, Any], item_id: str) -> bool:
|
||||
data_id_filter = set(self.get_data_id_filter())
|
||||
if not data_id_filter:
|
||||
return False
|
||||
if self.node_type == "project":
|
||||
return item_id not in data_id_filter
|
||||
|
||||
project_id = item.get("project_id")
|
||||
if project_id:
|
||||
return str(project_id) not in data_id_filter
|
||||
return False
|
||||
|
||||
def set_api_client(self, client: "ApiClient") -> None:
|
||||
"""默认忽略 API 客户端注入。"""
|
||||
return
|
||||
|
||||
@@ -51,18 +51,6 @@ class BaseJsonlHandler(BaseNodeHandler):
|
||||
super().__init__(node_type, node_class, schema)
|
||||
self.datasource = datasource
|
||||
|
||||
def _should_skip_item_by_project_filter(self, item: Dict[str, Any], item_id: str) -> bool:
|
||||
data_id_filter = set(self.get_data_id_filter())
|
||||
if not data_id_filter:
|
||||
return False
|
||||
if self.node_type == "project":
|
||||
return item_id not in data_id_filter
|
||||
|
||||
project_id = item.get("project_id")
|
||||
if project_id:
|
||||
return str(project_id) not in data_id_filter
|
||||
return False
|
||||
|
||||
async def load(self) -> List['SyncNode']:
|
||||
"""
|
||||
查找并加载对应的 JSONL 文件并创建节点
|
||||
@@ -95,7 +83,7 @@ class BaseJsonlHandler(BaseNodeHandler):
|
||||
item = json.loads(line)
|
||||
item_id = self.extract_id(item)
|
||||
|
||||
if self._should_skip_item_by_project_filter(item, item_id):
|
||||
if self.should_skip_by_data_id_filter(item, item_id):
|
||||
continue
|
||||
|
||||
# 创建节点
|
||||
|
||||
@@ -71,8 +71,8 @@ class ProjectApiHandler(BaseApiHandler):
|
||||
# 类级别缓存:存储 all_info 数据
|
||||
_all_info_cache: Dict[str, Dict[str, Any]] = {}
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
def __init__(self, **handler_config: Any):
|
||||
super().__init__(**handler_config)
|
||||
self._node_type = "project"
|
||||
self._node_class = ProjectSyncNode
|
||||
self._schema = ProjectResponseBase
|
||||
@@ -148,10 +148,14 @@ class ProjectApiHandler(BaseApiHandler):
|
||||
|
||||
# 处理所有项目
|
||||
for project_data in projects:
|
||||
project_item = project_data.model_dump()
|
||||
if self.should_skip_by_data_id_filter(project_item, project_data.id):
|
||||
continue
|
||||
|
||||
# 预加载并缓存 all_info,同时存储到 node.context
|
||||
all_info = await self.get_all_info(self.api_client, project_data.id)
|
||||
normalized_project_data = _merge_project_all_info_fields(
|
||||
project_data.model_dump(),
|
||||
project_item,
|
||||
all_info,
|
||||
)
|
||||
node = self._create_node(normalized_project_data)
|
||||
|
||||
@@ -28,6 +28,44 @@ async def print_pipeline_summary(*, logger, node_types, binding_manager, local_d
|
||||
return text[:width]
|
||||
return text[: width - 1] + "…"
|
||||
|
||||
def print_consistency_summary() -> None:
|
||||
line_width = 96
|
||||
|
||||
def _format_sample(sample: dict) -> str:
|
||||
local_text = _clip(repr(sample.get("local")), 72)
|
||||
remote_text = _clip(repr(sample.get("remote")), 72)
|
||||
return f"{local_text}/{remote_text}"
|
||||
|
||||
logger.info(f"\n{'=' * line_width}")
|
||||
logger.info("🔎 Consistency Summary")
|
||||
logger.info(f"{'=' * line_width}")
|
||||
|
||||
for node_type in node_types:
|
||||
item = consistency_by_type.get(node_type, {}) or {}
|
||||
checked_pairs = int(item.get("checked_pairs", 0))
|
||||
mismatch_count = int(item.get("mismatch_count", 0))
|
||||
logger.info(f"{node_type} (checked_pairs={checked_pairs}, mismatches={mismatch_count})")
|
||||
|
||||
fields = item.get("fields", []) or []
|
||||
differing_fields = [field for field in fields if int(field.get("mismatch_count", 0)) > 0]
|
||||
if not differing_fields:
|
||||
logger.info(" pass")
|
||||
continue
|
||||
|
||||
logger.info(" field mismatch/total diff_schemas samples")
|
||||
logger.info(" ------------------------------------------------------------------------------")
|
||||
for field_report in differing_fields:
|
||||
field_name = str(field_report.get("field_name", ""))
|
||||
field_mismatch = int(field_report.get("mismatch_count", 0))
|
||||
field_total = int(field_report.get("total_count", 0))
|
||||
schema_refs = ",".join(str(item) for item in (field_report.get("schema_refs", []) or [])) or "-"
|
||||
samples = field_report.get("samples", []) or []
|
||||
sample_text = _format_sample(samples[0]) if samples else "-"
|
||||
logger.info(
|
||||
f" {_clip(field_name, 30):<30} {field_mismatch:>4}/{field_total:<4} "
|
||||
f"{_clip(schema_refs, 28):<28} {sample_text}"
|
||||
)
|
||||
|
||||
def print_collection_summary(title: str, datasource, collection) -> None:
|
||||
action_summary = datasource.get_action_summary()
|
||||
line_width = 133
|
||||
@@ -110,20 +148,17 @@ async def print_pipeline_summary(*, logger, node_types, binding_manager, local_d
|
||||
f"{_clip(total_consistency_sft, consistency_w):<{consistency_w}}"
|
||||
)
|
||||
|
||||
print_collection_summary("Local", local_datasource, local_collection)
|
||||
print_collection_summary("Remote", remote_datasource, remote_collection)
|
||||
|
||||
loaded = int(stats.get("loaded", 0))
|
||||
synced = int(stats.get("synced", 0))
|
||||
failed = int(stats.get("failed", 0))
|
||||
post_check_passed = bool(stats.get("post_check_passed", True))
|
||||
|
||||
mismatch_node_types = [
|
||||
node_type
|
||||
for node_type, item in consistency_by_type.items()
|
||||
if int(item.get("mismatch_count", 0)) > 0
|
||||
]
|
||||
|
||||
loaded = int(stats.get("loaded", 0))
|
||||
synced = int(stats.get("synced", 0))
|
||||
failed = int(stats.get("failed", 0))
|
||||
post_check_passed = bool(stats.get("post_check_passed", True))
|
||||
|
||||
logger.info("\n" + "=" * 96)
|
||||
logger.info(
|
||||
f"📈 Pipeline Stats: loaded={loaded}, synced={synced}, failed={failed}, post_check_passed={post_check_passed}, mismatch_types={len(mismatch_node_types)}"
|
||||
@@ -132,19 +167,6 @@ async def print_pipeline_summary(*, logger, node_types, binding_manager, local_d
|
||||
logger.info(f"🔎 Mismatch Node Types: {', '.join(mismatch_node_types)}")
|
||||
logger.info("=" * 96)
|
||||
|
||||
sample_per_type = 3
|
||||
has_mismatch_samples = any(int(item.get("mismatch_count", 0)) > 0 for item in consistency_by_type.values())
|
||||
if has_mismatch_samples:
|
||||
logger.info("\n" + "=" * 96)
|
||||
logger.info("🧪 Inconsistency Samples (per type)")
|
||||
logger.info("=" * 96)
|
||||
for node_type in node_types:
|
||||
item = consistency_by_type.get(node_type, {})
|
||||
mismatch_count = int(item.get("mismatch_count", 0))
|
||||
if mismatch_count <= 0:
|
||||
continue
|
||||
checked_pairs = int(item.get("checked_pairs", 0))
|
||||
samples = item.get("samples", []) or []
|
||||
logger.info(f"- {node_type}: mismatches={mismatch_count}, checked_pairs={checked_pairs}")
|
||||
for sample in samples[:sample_per_type]:
|
||||
logger.info(f" • {sample}")
|
||||
print_consistency_summary()
|
||||
print_collection_summary("Local", local_datasource, local_collection)
|
||||
print_collection_summary("Remote", remote_datasource, remote_collection)
|
||||
|
||||
@@ -19,6 +19,7 @@ from ..engine import (
|
||||
StateMachineConfig,
|
||||
StateMachineRuntime,
|
||||
)
|
||||
from ..validation import SchemaDiffValidator
|
||||
|
||||
T = TypeVar("T", bound=BaseModel)
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -70,6 +71,7 @@ class BaseSyncStrategy(Generic[T]):
|
||||
self.skip_sync = self.default_skip_sync
|
||||
# 统一由 Pipeline 注入 runtime;未注入时在执行阶段显式报错
|
||||
self.sm_runtime: Optional[StateMachineRuntime] = None
|
||||
self._schema_diff_validator: Optional[SchemaDiffValidator] = None
|
||||
|
||||
# 配置逻辑校验
|
||||
self._validate_config()
|
||||
@@ -145,6 +147,7 @@ class BaseSyncStrategy(Generic[T]):
|
||||
def set_config(self, config: StrategyConfig):
|
||||
"""完全替换配置对象"""
|
||||
self.config = config
|
||||
self._schema_diff_validator = None
|
||||
self._validate_config()
|
||||
|
||||
def update_config(self, **kwargs):
|
||||
@@ -163,6 +166,7 @@ class BaseSyncStrategy(Generic[T]):
|
||||
setattr(self.config, key, value)
|
||||
else:
|
||||
logger.warning(f"[{self.node_type}] Unknown config key: {key}")
|
||||
self._schema_diff_validator = None
|
||||
# 更新配置后也要校验
|
||||
self._validate_config()
|
||||
|
||||
@@ -178,9 +182,29 @@ class BaseSyncStrategy(Generic[T]):
|
||||
"""
|
||||
preset_config = ConfigPresets.get_preset(preset_name)
|
||||
self.config = preset_config
|
||||
self._schema_diff_validator = None
|
||||
self._validate_config()
|
||||
logger.info(f"[{self.node_type}] Applied preset: {preset_name}")
|
||||
|
||||
def get_schema_diff_validator(self) -> SchemaDiffValidator:
|
||||
if self._schema_diff_validator is None:
|
||||
self._schema_diff_validator = SchemaDiffValidator(
|
||||
schema_name=self.schema.__name__,
|
||||
ignore_fields=self.config.compare_ignore_fields,
|
||||
ignore_list_item_fields=self.config.post_check_ignore_list_item_fields,
|
||||
)
|
||||
return self._schema_diff_validator
|
||||
|
||||
def reset_schema_diff_validator(self) -> None:
|
||||
self.get_schema_diff_validator().reset()
|
||||
|
||||
def emit_schema_diff_report(self) -> None:
|
||||
validator = self.get_schema_diff_validator()
|
||||
if not validator.has_records():
|
||||
return
|
||||
for line in validator.format_summary_lines():
|
||||
logger.info(f"[{self.node_type}] validate {line}")
|
||||
|
||||
async def bind(self, **kwargs):
|
||||
"""进行绑定逻辑判定,设置 node.binding_status 和 node.action"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -2,7 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import inspect
|
||||
from typing import Any, Dict, Optional
|
||||
from typing import Any, Dict, Mapping, Optional
|
||||
|
||||
|
||||
async def build_data_id_normalization_map(
|
||||
@@ -39,14 +39,42 @@ def is_id_like_field(field_name: Optional[str]) -> bool:
|
||||
return lowered.endswith("_id") or lowered.endswith("_ids")
|
||||
|
||||
|
||||
def normalize_compare_value(field_name: Optional[str], value: Any, data_id_map: Dict[str, str]) -> Any:
|
||||
def _strip_ignored_list_item_fields(
|
||||
field_name: Optional[str],
|
||||
value: Any,
|
||||
ignore_list_item_fields: Mapping[str, list[str]],
|
||||
) -> Any:
|
||||
if not isinstance(value, list):
|
||||
return value
|
||||
|
||||
ignored_keys = set(ignore_list_item_fields.get(field_name or "", []))
|
||||
if not ignored_keys:
|
||||
return value
|
||||
|
||||
normalized_items = []
|
||||
for item in value:
|
||||
if isinstance(item, dict):
|
||||
normalized_items.append({key: nested for key, nested in item.items() if key not in ignored_keys})
|
||||
else:
|
||||
normalized_items.append(item)
|
||||
return normalized_items
|
||||
|
||||
|
||||
def normalize_compare_value(
|
||||
field_name: Optional[str],
|
||||
value: Any,
|
||||
data_id_map: Dict[str, str],
|
||||
ignore_list_item_fields: Optional[Mapping[str, list[str]]] = None,
|
||||
) -> Any:
|
||||
ignored_list_fields = ignore_list_item_fields or {}
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
key: normalize_compare_value(key, nested, data_id_map)
|
||||
key: normalize_compare_value(key, nested, data_id_map, ignored_list_fields)
|
||||
for key, nested in value.items()
|
||||
}
|
||||
if isinstance(value, list):
|
||||
return [normalize_compare_value(field_name, item, data_id_map) for item in value]
|
||||
value = _strip_ignored_list_item_fields(field_name, value, ignored_list_fields)
|
||||
return [normalize_compare_value(field_name, item, data_id_map, ignored_list_fields) for item in value]
|
||||
|
||||
if value == "":
|
||||
value = None
|
||||
@@ -59,14 +87,18 @@ def normalize_compare_value(field_name: Optional[str], value: Any, data_id_map:
|
||||
def normalized_data_for_compare(
|
||||
data: Optional[Dict[str, Any]],
|
||||
data_id_map: Optional[Dict[str, str]] = None,
|
||||
ignore_fields: Optional[set[str]] = None,
|
||||
ignore_list_item_fields: Optional[Mapping[str, list[str]]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
payload = copy.deepcopy(data or {})
|
||||
payload.pop("id", None)
|
||||
payload.pop("_id", None)
|
||||
ignored = ignore_fields or set()
|
||||
mapping = data_id_map or {}
|
||||
return {
|
||||
key: normalize_compare_value(key, value, mapping)
|
||||
key: normalize_compare_value(key, value, mapping, ignore_list_item_fields)
|
||||
for key, value in payload.items()
|
||||
if key not in ignored
|
||||
}
|
||||
|
||||
|
||||
@@ -75,10 +107,14 @@ def collect_payload_diffs(
|
||||
target_payload: Dict[str, Any],
|
||||
*,
|
||||
max_items: int = 6,
|
||||
ignore_fields: Optional[set[str]] = None,
|
||||
) -> Dict[str, Dict[str, Any]]:
|
||||
out: Dict[str, Dict[str, Any]] = {}
|
||||
ignored = ignore_fields or set()
|
||||
keys = list(dict.fromkeys(list(source_payload.keys()) + list(target_payload.keys())))
|
||||
for key in keys:
|
||||
if key in ignored:
|
||||
continue
|
||||
source_value = source_payload.get(key)
|
||||
target_value = target_payload.get(key)
|
||||
if source_value == target_value:
|
||||
|
||||
@@ -3,10 +3,10 @@ from __future__ import annotations
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from ...sync_system.resolve_ids import IDResolver
|
||||
from ...validation import SchemaDiffValidator
|
||||
from .compare_ops import (
|
||||
build_data_id_normalization_map,
|
||||
normalized_data_for_compare,
|
||||
collect_payload_diffs,
|
||||
)
|
||||
|
||||
|
||||
@@ -64,10 +64,10 @@ async def evaluate_consistency_for_node_type(
|
||||
post_check_fields: List[str] | None = None,
|
||||
) -> Dict[str, Any]:
|
||||
records = await binding_manager.get_all_records(node_type)
|
||||
mismatches: List[str] = []
|
||||
checked_pairs = 0
|
||||
depend_fields = dict(depend_fields or {})
|
||||
id_resolver = IDResolver(local_collection, remote_collection, binding_manager)
|
||||
validator = SchemaDiffValidator(schema_name=node_type)
|
||||
data_id_map = await build_data_id_normalization_map(
|
||||
node_type=node_type,
|
||||
local_collection=local_collection,
|
||||
@@ -125,35 +125,40 @@ async def evaluate_consistency_for_node_type(
|
||||
if ignore_list_item_fields:
|
||||
local_payload = _strip_list_item_fields(local_payload, ignore_list_item_fields)
|
||||
remote_payload = _strip_list_item_fields(remote_payload, ignore_list_item_fields)
|
||||
if local_payload != remote_payload:
|
||||
diff_map = collect_payload_diffs(local_payload, remote_payload)
|
||||
diff_parts = []
|
||||
diff_map = validator.compare_payloads(
|
||||
local_payload,
|
||||
remote_payload,
|
||||
sample_id=f"{local_id}->{remote_id}",
|
||||
schema_refs=[node_type],
|
||||
)
|
||||
if diff_map:
|
||||
detail_parts = []
|
||||
for field_name, values in diff_map.items():
|
||||
local_text = _compact_repr(values.get("source"))
|
||||
remote_text = _compact_repr(values.get("target"))
|
||||
diff_parts.append(f"{field_name}{{local:{local_text},remote:{remote_text}}}")
|
||||
detail = "; ".join(diff_parts) if diff_parts else "payload differs"
|
||||
|
||||
mismatch_text = f"pair_diff(local_id={local_id}, remote_id={remote_id}, {detail})"
|
||||
mismatches.append(mismatch_text)
|
||||
local_text = _compact_repr(values.get("local"))
|
||||
remote_text = _compact_repr(values.get("remote"))
|
||||
detail_parts.append(f"{field_name}{{local:{local_text},remote:{remote_text}}}")
|
||||
detail = "; ".join(detail_parts) if detail_parts else "payload differs"
|
||||
_append_post_check_error(local_node, f"post_check diff: {detail}")
|
||||
_append_post_check_error(remote_node, f"post_check diff: {detail}")
|
||||
|
||||
result = {
|
||||
"ok": len(mismatches) == 0,
|
||||
"checked_pairs": checked_pairs,
|
||||
"mismatch_count": len(mismatches),
|
||||
"samples": mismatches[:10],
|
||||
report = validator.build_report() if checked_pairs else {
|
||||
"schema_name": node_type,
|
||||
"pairs_compared": 0,
|
||||
"mismatched_pairs": 0,
|
||||
"ignore_fields": [],
|
||||
"fields": [],
|
||||
}
|
||||
|
||||
if result["ok"]:
|
||||
logger.info(f"✅ Consistency check passed: node_type={node_type}, checked_pairs={checked_pairs}")
|
||||
else:
|
||||
logger.error(
|
||||
f"❌ Consistency check failed: node_type={node_type}, checked_pairs={checked_pairs}, mismatches={len(mismatches)}"
|
||||
)
|
||||
logger.error(" ↳ consistent语义: 比较绑定对(local_node.data vs remote_node.data),非 original_data")
|
||||
for item in result["samples"]:
|
||||
logger.error(f" - {item}")
|
||||
# 将字段统计补回到结果里,供 summary 输出。
|
||||
report["fields"] = report.get("fields", [])
|
||||
|
||||
result = {
|
||||
"ok": report["mismatched_pairs"] == 0,
|
||||
"checked_pairs": checked_pairs,
|
||||
"mismatch_count": report["mismatched_pairs"],
|
||||
"fields": report["fields"],
|
||||
"schema_name": report["schema_name"],
|
||||
"ignore_fields": report["ignore_fields"],
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
@@ -39,24 +39,6 @@ def _safe_repr(value, *, max_len: int = 120) -> str:
|
||||
return text[: max_len - 3] + "..."
|
||||
|
||||
|
||||
def _collect_diff_descriptions(source_data, target_data, *, max_items: int = 8) -> List[str]:
|
||||
if source_data is None or target_data is None:
|
||||
return []
|
||||
|
||||
descriptions: List[str] = []
|
||||
for key, source_value in source_data.items():
|
||||
if key in {"id", "_id"}:
|
||||
continue
|
||||
target_value = target_data.get(key)
|
||||
if source_value == target_value:
|
||||
continue
|
||||
descriptions.append(f"{key}: {_safe_repr(target_value)} -> {_safe_repr(source_value)}")
|
||||
if len(descriptions) >= max_items:
|
||||
break
|
||||
|
||||
return descriptions
|
||||
|
||||
|
||||
def default_needs_update(
|
||||
strategy,
|
||||
source_node,
|
||||
@@ -72,13 +54,29 @@ def default_needs_update(
|
||||
if source_data is None or target_data is None:
|
||||
return False
|
||||
|
||||
normalized_source = normalized_data_for_compare(source_data, data_id_map)
|
||||
normalized_target = normalized_data_for_compare(target_data, data_id_map)
|
||||
diff_map = collect_payload_diffs(normalized_source, normalized_target, max_items=8)
|
||||
ignore_fields = set(strategy.config.compare_ignore_fields)
|
||||
normalized_source = normalized_data_for_compare(
|
||||
source_data,
|
||||
data_id_map,
|
||||
ignore_fields=ignore_fields,
|
||||
ignore_list_item_fields=strategy.config.post_check_ignore_list_item_fields,
|
||||
)
|
||||
normalized_target = normalized_data_for_compare(
|
||||
target_data,
|
||||
data_id_map,
|
||||
ignore_fields=ignore_fields,
|
||||
ignore_list_item_fields=strategy.config.post_check_ignore_list_item_fields,
|
||||
)
|
||||
diff_map = strategy.get_schema_diff_validator().compare_payloads(
|
||||
normalized_source,
|
||||
normalized_target,
|
||||
sample_id=str(target_node.data_id or target_node.node_id),
|
||||
schema_refs=[strategy.schema.__name__],
|
||||
)
|
||||
is_different = bool(diff_map)
|
||||
|
||||
if is_different and diff_map:
|
||||
previews = [f"{key}: {values.get('source')} != {values.get('target')}" for key, values in list(diff_map.items())[:5]]
|
||||
previews = [f"{key}: {values.get('local')} != {values.get('remote')}" for key, values in list(diff_map.items())[:5]]
|
||||
logger.debug(f"[{strategy.node_type}] Node diff detected: {', '.join(previews)}")
|
||||
|
||||
return is_different
|
||||
@@ -88,13 +86,36 @@ def resolve_needs_update_check(strategy) -> Callable:
|
||||
return strategy._needs_update
|
||||
|
||||
|
||||
def _collect_diff_descriptions(source_data, target_data, *, data_id_map, max_items: int = 8) -> List[str]:
|
||||
def _collect_diff_descriptions(
|
||||
source_data,
|
||||
target_data,
|
||||
*,
|
||||
data_id_map,
|
||||
ignore_fields: set[str],
|
||||
ignore_list_item_fields: dict[str, list[str]],
|
||||
max_items: int = 8,
|
||||
) -> List[str]:
|
||||
if source_data is None or target_data is None:
|
||||
return []
|
||||
|
||||
normalized_source = normalized_data_for_compare(source_data, data_id_map)
|
||||
normalized_target = normalized_data_for_compare(target_data, data_id_map)
|
||||
diff_map = collect_payload_diffs(normalized_source, normalized_target, max_items=max_items)
|
||||
normalized_source = normalized_data_for_compare(
|
||||
source_data,
|
||||
data_id_map,
|
||||
ignore_fields=ignore_fields,
|
||||
ignore_list_item_fields=ignore_list_item_fields,
|
||||
)
|
||||
normalized_target = normalized_data_for_compare(
|
||||
target_data,
|
||||
data_id_map,
|
||||
ignore_fields=ignore_fields,
|
||||
ignore_list_item_fields=ignore_list_item_fields,
|
||||
)
|
||||
diff_map = collect_payload_diffs(
|
||||
normalized_source,
|
||||
normalized_target,
|
||||
max_items=max_items,
|
||||
ignore_fields=ignore_fields,
|
||||
)
|
||||
|
||||
descriptions: List[str] = []
|
||||
for key, values in diff_map.items():
|
||||
@@ -184,7 +205,13 @@ async def add_update_action(
|
||||
should_update = needs_update_check(src_node)
|
||||
has_diff = bool(should_update)
|
||||
if has_diff:
|
||||
diff_descriptions = _collect_diff_descriptions(data, target_node.get_data(), data_id_map=data_id_map or {})
|
||||
diff_descriptions = _collect_diff_descriptions(
|
||||
data,
|
||||
target_node.get_data(),
|
||||
data_id_map=data_id_map or {},
|
||||
ignore_fields=set(strategy.config.compare_ignore_fields),
|
||||
ignore_list_item_fields=dict(strategy.config.post_check_ignore_list_item_fields),
|
||||
)
|
||||
|
||||
if steps_changed:
|
||||
has_diff = True
|
||||
@@ -226,6 +253,7 @@ async def add_update_action(
|
||||
|
||||
async def run_update(strategy) -> List["SyncNode"]:
|
||||
updated_nodes: List["SyncNode"] = []
|
||||
strategy.reset_schema_diff_validator()
|
||||
needs_update_check = resolve_needs_update_check(strategy)
|
||||
data_id_map = await build_data_id_normalization_map(
|
||||
node_type=strategy.node_type,
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
from .schema_diff import SchemaDiffValidator
|
||||
|
||||
__all__ = ["SchemaDiffValidator"]
|
||||
@@ -0,0 +1,150 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Iterable, Mapping
|
||||
|
||||
from ..sync_system.strategy_ops.compare_ops import normalized_data_for_compare
|
||||
|
||||
|
||||
@dataclass
|
||||
class SchemaDiffSample:
|
||||
sample_id: str
|
||||
local: Any
|
||||
remote: Any
|
||||
|
||||
|
||||
@dataclass
|
||||
class SchemaFieldDiffStat:
|
||||
field_name: str
|
||||
compared_count: int = 0
|
||||
mismatch_count: int = 0
|
||||
schema_refs: set[str] = field(default_factory=set)
|
||||
samples: list[SchemaDiffSample] = field(default_factory=list)
|
||||
|
||||
|
||||
class SchemaDiffValidator:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
schema_name: str,
|
||||
ignore_fields: Iterable[str] | None = None,
|
||||
ignore_list_item_fields: Mapping[str, list[str]] | None = None,
|
||||
sample_limit: int = 3,
|
||||
) -> None:
|
||||
self.schema_name = schema_name
|
||||
self.ignore_fields = {str(field) for field in (ignore_fields or []) if str(field)}
|
||||
self.ignore_list_item_fields = dict(ignore_list_item_fields or {})
|
||||
self.sample_limit = max(1, sample_limit)
|
||||
self.reset()
|
||||
|
||||
def reset(self) -> None:
|
||||
self.pairs_compared = 0
|
||||
self.mismatched_pairs = 0
|
||||
self._field_stats: dict[str, SchemaFieldDiffStat] = {}
|
||||
|
||||
def has_records(self) -> bool:
|
||||
return self.pairs_compared > 0
|
||||
|
||||
def compare_payloads(
|
||||
self,
|
||||
local_payload: Mapping[str, Any] | None,
|
||||
remote_payload: Mapping[str, Any] | None,
|
||||
*,
|
||||
sample_id: str | None = None,
|
||||
schema_refs: Iterable[str] | None = None,
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
local_normalized = normalized_data_for_compare(
|
||||
dict(local_payload or {}),
|
||||
ignore_fields=self.ignore_fields,
|
||||
ignore_list_item_fields=self.ignore_list_item_fields,
|
||||
)
|
||||
remote_normalized = normalized_data_for_compare(
|
||||
dict(remote_payload or {}),
|
||||
ignore_fields=self.ignore_fields,
|
||||
ignore_list_item_fields=self.ignore_list_item_fields,
|
||||
)
|
||||
|
||||
field_names = sorted(set(local_normalized) | set(remote_normalized))
|
||||
diff_map: dict[str, dict[str, Any]] = {}
|
||||
refs = {self.schema_name, *(str(item) for item in (schema_refs or []) if str(item))}
|
||||
|
||||
self.pairs_compared += 1
|
||||
current_sample_id = sample_id or f"pair-{self.pairs_compared}"
|
||||
|
||||
for field_name in field_names:
|
||||
stat = self._field_stats.setdefault(field_name, SchemaFieldDiffStat(field_name=field_name))
|
||||
stat.compared_count += 1
|
||||
stat.schema_refs.update(refs)
|
||||
|
||||
local_value = local_normalized.get(field_name)
|
||||
remote_value = remote_normalized.get(field_name)
|
||||
if local_value == remote_value:
|
||||
continue
|
||||
|
||||
stat.mismatch_count += 1
|
||||
if len(stat.samples) < self.sample_limit:
|
||||
stat.samples.append(
|
||||
SchemaDiffSample(
|
||||
sample_id=current_sample_id,
|
||||
local=local_value,
|
||||
remote=remote_value,
|
||||
)
|
||||
)
|
||||
diff_map[field_name] = {"local": local_value, "remote": remote_value}
|
||||
|
||||
if diff_map:
|
||||
self.mismatched_pairs += 1
|
||||
return diff_map
|
||||
|
||||
def build_report(self) -> dict[str, Any]:
|
||||
fields = []
|
||||
for stat in sorted(
|
||||
self._field_stats.values(),
|
||||
key=lambda item: (-item.mismatch_count, item.field_name),
|
||||
):
|
||||
if stat.compared_count <= 0:
|
||||
continue
|
||||
fields.append(
|
||||
{
|
||||
"field_name": stat.field_name,
|
||||
"mismatch_count": stat.mismatch_count,
|
||||
"total_count": stat.compared_count,
|
||||
"mismatch_rate": stat.mismatch_count / stat.compared_count,
|
||||
"schema_refs": sorted(stat.schema_refs),
|
||||
"samples": [
|
||||
{
|
||||
"sample_id": sample.sample_id,
|
||||
"local": sample.local,
|
||||
"remote": sample.remote,
|
||||
}
|
||||
for sample in stat.samples
|
||||
],
|
||||
}
|
||||
)
|
||||
return {
|
||||
"schema_name": self.schema_name,
|
||||
"pairs_compared": self.pairs_compared,
|
||||
"mismatched_pairs": self.mismatched_pairs,
|
||||
"ignore_fields": sorted(self.ignore_fields),
|
||||
"fields": fields,
|
||||
}
|
||||
|
||||
def format_summary_lines(self, *, max_fields: int = 20, max_samples: int = 1) -> list[str]:
|
||||
report = self.build_report()
|
||||
lines = [
|
||||
(
|
||||
f"schema={report['schema_name']} pairs={report['pairs_compared']} "
|
||||
f"mismatched_pairs={report['mismatched_pairs']} ignored={report['ignore_fields']}"
|
||||
)
|
||||
]
|
||||
for field_report in report["fields"][:max_fields]:
|
||||
line = (
|
||||
f"field={field_report['field_name']} mismatch={field_report['mismatch_count']}/"
|
||||
f"{field_report['total_count']} refs={field_report['schema_refs']}"
|
||||
)
|
||||
samples = field_report["samples"][:max_samples]
|
||||
if samples:
|
||||
sample = samples[0]
|
||||
line += f" sample[{sample['sample_id']}]={sample['local']!r}/{sample['remote']!r}"
|
||||
lines.append(line)
|
||||
return lines
|
||||
Reference in New Issue
Block a user