增强validator和打印区别

This commit is contained in:
strepsiades
2026-03-23 14:39:29 +08:00
parent 4c380f1157
commit 0812489f9d
13 changed files with 496 additions and 117 deletions
@@ -52,6 +52,9 @@ class StrategyConfig(BaseModel):
# 用于排除后端硬写死、永远与本地不一致的字段,避免误报。
post_check_ignore_list_item_fields: Dict[str, List[str]] = Field(default_factory=dict)
# schema diff / validate 时忽略的顶层字段。
compare_ignore_fields: List[str] = Field(default_factory=list)
class StrategyRuntimeConfig(BaseModel):
"""Pipeline 层策略运行配置(按 node_type)。"""
+8 -15
View File
@@ -20,7 +20,7 @@ from typing import ClassVar, Dict, List, Any, Optional, TYPE_CHECKING, TypeVar
from pydantic import BaseModel
from ...common.type_safety import get_pydantic_model_fields
from ..handler import NodeHandler
from ..handler import BaseNodeHandler
from ..task_result import TaskResult
from ..handler import ValidationResult # Import ValidationResult
@@ -32,7 +32,7 @@ if TYPE_CHECKING:
T = TypeVar("T", bound=BaseModel)
class BaseApiHandler(NodeHandler[T]):
class BaseApiHandler(BaseNodeHandler[T]):
"""API Handler 基类"""
# 子类需要设置这些属性
@@ -40,15 +40,19 @@ class BaseApiHandler(NodeHandler[T]):
_node_class: type['SyncNode']
_schema: Any
def __init__(self):
def __init__(self, **handler_config: Any):
# 子类应该在调用 super().__init__() 之前或之后设置 _node_type, _node_class, _schema
super().__init__()
self.api_client: 'ApiClient' = None # type: ignore # 由 DataSource 注入
self.poll_mode: str = "async" # async | sync
self._collection: 'DataCollection' = None # type: ignore # 由 DataSource 注入
self.handler_config: Dict[str, Any] = {}
self.handler_config = dict(handler_config)
# 子类在 __init__ 中设置此列表以声明 create/update 中涉及的 Pydantic schema 类;
# post-check 将只比较这些 schema 覆盖的字段,过滤后端只读的派生字段。
self.update_schemas: List[type] = []
def set_collection(self, collection: 'DataCollection') -> None:
self._collection = collection
def get_update_fields(self, *, exclude: frozenset = frozenset({'id'})) -> List[str]:
"""从 update_schemas 中提取所有可写字段名,排除标识键(默认排除 id)。"""
@@ -66,17 +70,6 @@ class BaseApiHandler(NodeHandler[T]):
"""设置 Collection(由 DataSource 调用)"""
self._collection = collection
def set_handler_config(self, config: Dict[str, Any]) -> None:
self.handler_config = dict(config)
def get_data_id_filter(self) -> List[str]:
value = self.handler_config.get("data_id_filter", [])
if isinstance(value, str):
return [value] if value else []
if isinstance(value, list):
return [str(item) for item in value if str(item)]
return []
@property
def node_type(self) -> str:
"""返回节点类型"""
+16 -4
View File
@@ -337,11 +337,11 @@ class BaseNodeHandler(ABC, Generic[T]):
def __init__(
self,
node_type: str,
node_class: Type["SyncNode[T]"],
schema: Type[T]
node_type: str | None = None,
node_class: Type["SyncNode[T]"] | None = None,
schema: Type[T] | None = None,
):
self._node_type = node_type
self._node_type = node_type or ""
self._node_class = node_class
self._schema = schema
self._collection: Optional["DataCollection"] = None
@@ -368,6 +368,18 @@ class BaseNodeHandler(ABC, Generic[T]):
return [str(item) for item in value if str(item)]
return []
def should_skip_by_data_id_filter(self, item: Dict[str, Any], item_id: str) -> bool:
data_id_filter = set(self.get_data_id_filter())
if not data_id_filter:
return False
if self.node_type == "project":
return item_id not in data_id_filter
project_id = item.get("project_id")
if project_id:
return str(project_id) not in data_id_filter
return False
def set_api_client(self, client: "ApiClient") -> None:
"""默认忽略 API 客户端注入。"""
return
+1 -13
View File
@@ -51,18 +51,6 @@ class BaseJsonlHandler(BaseNodeHandler):
super().__init__(node_type, node_class, schema)
self.datasource = datasource
def _should_skip_item_by_project_filter(self, item: Dict[str, Any], item_id: str) -> bool:
data_id_filter = set(self.get_data_id_filter())
if not data_id_filter:
return False
if self.node_type == "project":
return item_id not in data_id_filter
project_id = item.get("project_id")
if project_id:
return str(project_id) not in data_id_filter
return False
async def load(self) -> List['SyncNode']:
"""
查找并加载对应的 JSONL 文件并创建节点
@@ -95,7 +83,7 @@ class BaseJsonlHandler(BaseNodeHandler):
item = json.loads(line)
item_id = self.extract_id(item)
if self._should_skip_item_by_project_filter(item, item_id):
if self.should_skip_by_data_id_filter(item, item_id):
continue
# 创建节点
@@ -71,8 +71,8 @@ class ProjectApiHandler(BaseApiHandler):
# 类级别缓存:存储 all_info 数据
_all_info_cache: Dict[str, Dict[str, Any]] = {}
def __init__(self):
super().__init__()
def __init__(self, **handler_config: Any):
super().__init__(**handler_config)
self._node_type = "project"
self._node_class = ProjectSyncNode
self._schema = ProjectResponseBase
@@ -148,10 +148,14 @@ class ProjectApiHandler(BaseApiHandler):
# 处理所有项目
for project_data in projects:
project_item = project_data.model_dump()
if self.should_skip_by_data_id_filter(project_item, project_data.id):
continue
# 预加载并缓存 all_info,同时存储到 node.context
all_info = await self.get_all_info(self.api_client, project_data.id)
normalized_project_data = _merge_project_all_info_fields(
project_data.model_dump(),
project_item,
all_info,
)
node = self._create_node(normalized_project_data)
+46 -24
View File
@@ -28,6 +28,44 @@ async def print_pipeline_summary(*, logger, node_types, binding_manager, local_d
return text[:width]
return text[: width - 1] + ""
def print_consistency_summary() -> None:
line_width = 96
def _format_sample(sample: dict) -> str:
local_text = _clip(repr(sample.get("local")), 72)
remote_text = _clip(repr(sample.get("remote")), 72)
return f"{local_text}/{remote_text}"
logger.info(f"\n{'=' * line_width}")
logger.info("🔎 Consistency Summary")
logger.info(f"{'=' * line_width}")
for node_type in node_types:
item = consistency_by_type.get(node_type, {}) or {}
checked_pairs = int(item.get("checked_pairs", 0))
mismatch_count = int(item.get("mismatch_count", 0))
logger.info(f"{node_type} (checked_pairs={checked_pairs}, mismatches={mismatch_count})")
fields = item.get("fields", []) or []
differing_fields = [field for field in fields if int(field.get("mismatch_count", 0)) > 0]
if not differing_fields:
logger.info(" pass")
continue
logger.info(" field mismatch/total diff_schemas samples")
logger.info(" ------------------------------------------------------------------------------")
for field_report in differing_fields:
field_name = str(field_report.get("field_name", ""))
field_mismatch = int(field_report.get("mismatch_count", 0))
field_total = int(field_report.get("total_count", 0))
schema_refs = ",".join(str(item) for item in (field_report.get("schema_refs", []) or [])) or "-"
samples = field_report.get("samples", []) or []
sample_text = _format_sample(samples[0]) if samples else "-"
logger.info(
f" {_clip(field_name, 30):<30} {field_mismatch:>4}/{field_total:<4} "
f"{_clip(schema_refs, 28):<28} {sample_text}"
)
def print_collection_summary(title: str, datasource, collection) -> None:
action_summary = datasource.get_action_summary()
line_width = 133
@@ -110,20 +148,17 @@ async def print_pipeline_summary(*, logger, node_types, binding_manager, local_d
f"{_clip(total_consistency_sft, consistency_w):<{consistency_w}}"
)
print_collection_summary("Local", local_datasource, local_collection)
print_collection_summary("Remote", remote_datasource, remote_collection)
loaded = int(stats.get("loaded", 0))
synced = int(stats.get("synced", 0))
failed = int(stats.get("failed", 0))
post_check_passed = bool(stats.get("post_check_passed", True))
mismatch_node_types = [
node_type
for node_type, item in consistency_by_type.items()
if int(item.get("mismatch_count", 0)) > 0
]
loaded = int(stats.get("loaded", 0))
synced = int(stats.get("synced", 0))
failed = int(stats.get("failed", 0))
post_check_passed = bool(stats.get("post_check_passed", True))
logger.info("\n" + "=" * 96)
logger.info(
f"📈 Pipeline Stats: loaded={loaded}, synced={synced}, failed={failed}, post_check_passed={post_check_passed}, mismatch_types={len(mismatch_node_types)}"
@@ -132,19 +167,6 @@ async def print_pipeline_summary(*, logger, node_types, binding_manager, local_d
logger.info(f"🔎 Mismatch Node Types: {', '.join(mismatch_node_types)}")
logger.info("=" * 96)
sample_per_type = 3
has_mismatch_samples = any(int(item.get("mismatch_count", 0)) > 0 for item in consistency_by_type.values())
if has_mismatch_samples:
logger.info("\n" + "=" * 96)
logger.info("🧪 Inconsistency Samples (per type)")
logger.info("=" * 96)
for node_type in node_types:
item = consistency_by_type.get(node_type, {})
mismatch_count = int(item.get("mismatch_count", 0))
if mismatch_count <= 0:
continue
checked_pairs = int(item.get("checked_pairs", 0))
samples = item.get("samples", []) or []
logger.info(f"- {node_type}: mismatches={mismatch_count}, checked_pairs={checked_pairs}")
for sample in samples[:sample_per_type]:
logger.info(f"{sample}")
print_consistency_summary()
print_collection_summary("Local", local_datasource, local_collection)
print_collection_summary("Remote", remote_datasource, remote_collection)
@@ -19,6 +19,7 @@ from ..engine import (
StateMachineConfig,
StateMachineRuntime,
)
from ..validation import SchemaDiffValidator
T = TypeVar("T", bound=BaseModel)
logger = logging.getLogger(__name__)
@@ -70,6 +71,7 @@ class BaseSyncStrategy(Generic[T]):
self.skip_sync = self.default_skip_sync
# 统一由 Pipeline 注入 runtime;未注入时在执行阶段显式报错
self.sm_runtime: Optional[StateMachineRuntime] = None
self._schema_diff_validator: Optional[SchemaDiffValidator] = None
# 配置逻辑校验
self._validate_config()
@@ -145,6 +147,7 @@ class BaseSyncStrategy(Generic[T]):
def set_config(self, config: StrategyConfig):
"""完全替换配置对象"""
self.config = config
self._schema_diff_validator = None
self._validate_config()
def update_config(self, **kwargs):
@@ -163,6 +166,7 @@ class BaseSyncStrategy(Generic[T]):
setattr(self.config, key, value)
else:
logger.warning(f"[{self.node_type}] Unknown config key: {key}")
self._schema_diff_validator = None
# 更新配置后也要校验
self._validate_config()
@@ -178,9 +182,29 @@ class BaseSyncStrategy(Generic[T]):
"""
preset_config = ConfigPresets.get_preset(preset_name)
self.config = preset_config
self._schema_diff_validator = None
self._validate_config()
logger.info(f"[{self.node_type}] Applied preset: {preset_name}")
def get_schema_diff_validator(self) -> SchemaDiffValidator:
if self._schema_diff_validator is None:
self._schema_diff_validator = SchemaDiffValidator(
schema_name=self.schema.__name__,
ignore_fields=self.config.compare_ignore_fields,
ignore_list_item_fields=self.config.post_check_ignore_list_item_fields,
)
return self._schema_diff_validator
def reset_schema_diff_validator(self) -> None:
self.get_schema_diff_validator().reset()
def emit_schema_diff_report(self) -> None:
validator = self.get_schema_diff_validator()
if not validator.has_records():
return
for line in validator.format_summary_lines():
logger.info(f"[{self.node_type}] validate {line}")
async def bind(self, **kwargs):
"""进行绑定逻辑判定,设置 node.binding_status 和 node.action"""
raise NotImplementedError
@@ -2,7 +2,7 @@ from __future__ import annotations
import copy
import inspect
from typing import Any, Dict, Optional
from typing import Any, Dict, Mapping, Optional
async def build_data_id_normalization_map(
@@ -39,14 +39,42 @@ def is_id_like_field(field_name: Optional[str]) -> bool:
return lowered.endswith("_id") or lowered.endswith("_ids")
def normalize_compare_value(field_name: Optional[str], value: Any, data_id_map: Dict[str, str]) -> Any:
def _strip_ignored_list_item_fields(
field_name: Optional[str],
value: Any,
ignore_list_item_fields: Mapping[str, list[str]],
) -> Any:
if not isinstance(value, list):
return value
ignored_keys = set(ignore_list_item_fields.get(field_name or "", []))
if not ignored_keys:
return value
normalized_items = []
for item in value:
if isinstance(item, dict):
normalized_items.append({key: nested for key, nested in item.items() if key not in ignored_keys})
else:
normalized_items.append(item)
return normalized_items
def normalize_compare_value(
field_name: Optional[str],
value: Any,
data_id_map: Dict[str, str],
ignore_list_item_fields: Optional[Mapping[str, list[str]]] = None,
) -> Any:
ignored_list_fields = ignore_list_item_fields or {}
if isinstance(value, dict):
return {
key: normalize_compare_value(key, nested, data_id_map)
key: normalize_compare_value(key, nested, data_id_map, ignored_list_fields)
for key, nested in value.items()
}
if isinstance(value, list):
return [normalize_compare_value(field_name, item, data_id_map) for item in value]
value = _strip_ignored_list_item_fields(field_name, value, ignored_list_fields)
return [normalize_compare_value(field_name, item, data_id_map, ignored_list_fields) for item in value]
if value == "":
value = None
@@ -59,14 +87,18 @@ def normalize_compare_value(field_name: Optional[str], value: Any, data_id_map:
def normalized_data_for_compare(
data: Optional[Dict[str, Any]],
data_id_map: Optional[Dict[str, str]] = None,
ignore_fields: Optional[set[str]] = None,
ignore_list_item_fields: Optional[Mapping[str, list[str]]] = None,
) -> Dict[str, Any]:
payload = copy.deepcopy(data or {})
payload.pop("id", None)
payload.pop("_id", None)
ignored = ignore_fields or set()
mapping = data_id_map or {}
return {
key: normalize_compare_value(key, value, mapping)
key: normalize_compare_value(key, value, mapping, ignore_list_item_fields)
for key, value in payload.items()
if key not in ignored
}
@@ -75,10 +107,14 @@ def collect_payload_diffs(
target_payload: Dict[str, Any],
*,
max_items: int = 6,
ignore_fields: Optional[set[str]] = None,
) -> Dict[str, Dict[str, Any]]:
out: Dict[str, Dict[str, Any]] = {}
ignored = ignore_fields or set()
keys = list(dict.fromkeys(list(source_payload.keys()) + list(target_payload.keys())))
for key in keys:
if key in ignored:
continue
source_value = source_payload.get(key)
target_value = target_payload.get(key)
if source_value == target_value:
@@ -3,10 +3,10 @@ from __future__ import annotations
from typing import Any, Dict, List
from ...sync_system.resolve_ids import IDResolver
from ...validation import SchemaDiffValidator
from .compare_ops import (
build_data_id_normalization_map,
normalized_data_for_compare,
collect_payload_diffs,
)
@@ -64,10 +64,10 @@ async def evaluate_consistency_for_node_type(
post_check_fields: List[str] | None = None,
) -> Dict[str, Any]:
records = await binding_manager.get_all_records(node_type)
mismatches: List[str] = []
checked_pairs = 0
depend_fields = dict(depend_fields or {})
id_resolver = IDResolver(local_collection, remote_collection, binding_manager)
validator = SchemaDiffValidator(schema_name=node_type)
data_id_map = await build_data_id_normalization_map(
node_type=node_type,
local_collection=local_collection,
@@ -125,35 +125,40 @@ async def evaluate_consistency_for_node_type(
if ignore_list_item_fields:
local_payload = _strip_list_item_fields(local_payload, ignore_list_item_fields)
remote_payload = _strip_list_item_fields(remote_payload, ignore_list_item_fields)
if local_payload != remote_payload:
diff_map = collect_payload_diffs(local_payload, remote_payload)
diff_parts = []
diff_map = validator.compare_payloads(
local_payload,
remote_payload,
sample_id=f"{local_id}->{remote_id}",
schema_refs=[node_type],
)
if diff_map:
detail_parts = []
for field_name, values in diff_map.items():
local_text = _compact_repr(values.get("source"))
remote_text = _compact_repr(values.get("target"))
diff_parts.append(f"{field_name}{{local:{local_text},remote:{remote_text}}}")
detail = "; ".join(diff_parts) if diff_parts else "payload differs"
mismatch_text = f"pair_diff(local_id={local_id}, remote_id={remote_id}, {detail})"
mismatches.append(mismatch_text)
local_text = _compact_repr(values.get("local"))
remote_text = _compact_repr(values.get("remote"))
detail_parts.append(f"{field_name}{{local:{local_text},remote:{remote_text}}}")
detail = "; ".join(detail_parts) if detail_parts else "payload differs"
_append_post_check_error(local_node, f"post_check diff: {detail}")
_append_post_check_error(remote_node, f"post_check diff: {detail}")
result = {
"ok": len(mismatches) == 0,
"checked_pairs": checked_pairs,
"mismatch_count": len(mismatches),
"samples": mismatches[:10],
report = validator.build_report() if checked_pairs else {
"schema_name": node_type,
"pairs_compared": 0,
"mismatched_pairs": 0,
"ignore_fields": [],
"fields": [],
}
if result["ok"]:
logger.info(f"✅ Consistency check passed: node_type={node_type}, checked_pairs={checked_pairs}")
else:
logger.error(
f"❌ Consistency check failed: node_type={node_type}, checked_pairs={checked_pairs}, mismatches={len(mismatches)}"
)
logger.error(" ↳ consistent语义: 比较绑定对(local_node.data vs remote_node.data),非 original_data")
for item in result["samples"]:
logger.error(f" - {item}")
# 将字段统计补回到结果里,供 summary 输出。
report["fields"] = report.get("fields", [])
result = {
"ok": report["mismatched_pairs"] == 0,
"checked_pairs": checked_pairs,
"mismatch_count": report["mismatched_pairs"],
"fields": report["fields"],
"schema_name": report["schema_name"],
"ignore_fields": report["ignore_fields"],
}
return result
@@ -39,24 +39,6 @@ def _safe_repr(value, *, max_len: int = 120) -> str:
return text[: max_len - 3] + "..."
def _collect_diff_descriptions(source_data, target_data, *, max_items: int = 8) -> List[str]:
if source_data is None or target_data is None:
return []
descriptions: List[str] = []
for key, source_value in source_data.items():
if key in {"id", "_id"}:
continue
target_value = target_data.get(key)
if source_value == target_value:
continue
descriptions.append(f"{key}: {_safe_repr(target_value)} -> {_safe_repr(source_value)}")
if len(descriptions) >= max_items:
break
return descriptions
def default_needs_update(
strategy,
source_node,
@@ -72,13 +54,29 @@ def default_needs_update(
if source_data is None or target_data is None:
return False
normalized_source = normalized_data_for_compare(source_data, data_id_map)
normalized_target = normalized_data_for_compare(target_data, data_id_map)
diff_map = collect_payload_diffs(normalized_source, normalized_target, max_items=8)
ignore_fields = set(strategy.config.compare_ignore_fields)
normalized_source = normalized_data_for_compare(
source_data,
data_id_map,
ignore_fields=ignore_fields,
ignore_list_item_fields=strategy.config.post_check_ignore_list_item_fields,
)
normalized_target = normalized_data_for_compare(
target_data,
data_id_map,
ignore_fields=ignore_fields,
ignore_list_item_fields=strategy.config.post_check_ignore_list_item_fields,
)
diff_map = strategy.get_schema_diff_validator().compare_payloads(
normalized_source,
normalized_target,
sample_id=str(target_node.data_id or target_node.node_id),
schema_refs=[strategy.schema.__name__],
)
is_different = bool(diff_map)
if is_different and diff_map:
previews = [f"{key}: {values.get('source')} != {values.get('target')}" for key, values in list(diff_map.items())[:5]]
previews = [f"{key}: {values.get('local')} != {values.get('remote')}" for key, values in list(diff_map.items())[:5]]
logger.debug(f"[{strategy.node_type}] Node diff detected: {', '.join(previews)}")
return is_different
@@ -88,13 +86,36 @@ def resolve_needs_update_check(strategy) -> Callable:
return strategy._needs_update
def _collect_diff_descriptions(source_data, target_data, *, data_id_map, max_items: int = 8) -> List[str]:
def _collect_diff_descriptions(
source_data,
target_data,
*,
data_id_map,
ignore_fields: set[str],
ignore_list_item_fields: dict[str, list[str]],
max_items: int = 8,
) -> List[str]:
if source_data is None or target_data is None:
return []
normalized_source = normalized_data_for_compare(source_data, data_id_map)
normalized_target = normalized_data_for_compare(target_data, data_id_map)
diff_map = collect_payload_diffs(normalized_source, normalized_target, max_items=max_items)
normalized_source = normalized_data_for_compare(
source_data,
data_id_map,
ignore_fields=ignore_fields,
ignore_list_item_fields=ignore_list_item_fields,
)
normalized_target = normalized_data_for_compare(
target_data,
data_id_map,
ignore_fields=ignore_fields,
ignore_list_item_fields=ignore_list_item_fields,
)
diff_map = collect_payload_diffs(
normalized_source,
normalized_target,
max_items=max_items,
ignore_fields=ignore_fields,
)
descriptions: List[str] = []
for key, values in diff_map.items():
@@ -184,7 +205,13 @@ async def add_update_action(
should_update = needs_update_check(src_node)
has_diff = bool(should_update)
if has_diff:
diff_descriptions = _collect_diff_descriptions(data, target_node.get_data(), data_id_map=data_id_map or {})
diff_descriptions = _collect_diff_descriptions(
data,
target_node.get_data(),
data_id_map=data_id_map or {},
ignore_fields=set(strategy.config.compare_ignore_fields),
ignore_list_item_fields=dict(strategy.config.post_check_ignore_list_item_fields),
)
if steps_changed:
has_diff = True
@@ -226,6 +253,7 @@ async def add_update_action(
async def run_update(strategy) -> List["SyncNode"]:
updated_nodes: List["SyncNode"] = []
strategy.reset_schema_diff_validator()
needs_update_check = resolve_needs_update_check(strategy)
data_id_map = await build_data_id_normalization_map(
node_type=strategy.node_type,
@@ -0,0 +1,3 @@
from .schema_diff import SchemaDiffValidator
__all__ = ["SchemaDiffValidator"]
@@ -0,0 +1,150 @@
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any, Iterable, Mapping
from ..sync_system.strategy_ops.compare_ops import normalized_data_for_compare
@dataclass
class SchemaDiffSample:
sample_id: str
local: Any
remote: Any
@dataclass
class SchemaFieldDiffStat:
field_name: str
compared_count: int = 0
mismatch_count: int = 0
schema_refs: set[str] = field(default_factory=set)
samples: list[SchemaDiffSample] = field(default_factory=list)
class SchemaDiffValidator:
def __init__(
self,
*,
schema_name: str,
ignore_fields: Iterable[str] | None = None,
ignore_list_item_fields: Mapping[str, list[str]] | None = None,
sample_limit: int = 3,
) -> None:
self.schema_name = schema_name
self.ignore_fields = {str(field) for field in (ignore_fields or []) if str(field)}
self.ignore_list_item_fields = dict(ignore_list_item_fields or {})
self.sample_limit = max(1, sample_limit)
self.reset()
def reset(self) -> None:
self.pairs_compared = 0
self.mismatched_pairs = 0
self._field_stats: dict[str, SchemaFieldDiffStat] = {}
def has_records(self) -> bool:
return self.pairs_compared > 0
def compare_payloads(
self,
local_payload: Mapping[str, Any] | None,
remote_payload: Mapping[str, Any] | None,
*,
sample_id: str | None = None,
schema_refs: Iterable[str] | None = None,
) -> dict[str, dict[str, Any]]:
local_normalized = normalized_data_for_compare(
dict(local_payload or {}),
ignore_fields=self.ignore_fields,
ignore_list_item_fields=self.ignore_list_item_fields,
)
remote_normalized = normalized_data_for_compare(
dict(remote_payload or {}),
ignore_fields=self.ignore_fields,
ignore_list_item_fields=self.ignore_list_item_fields,
)
field_names = sorted(set(local_normalized) | set(remote_normalized))
diff_map: dict[str, dict[str, Any]] = {}
refs = {self.schema_name, *(str(item) for item in (schema_refs or []) if str(item))}
self.pairs_compared += 1
current_sample_id = sample_id or f"pair-{self.pairs_compared}"
for field_name in field_names:
stat = self._field_stats.setdefault(field_name, SchemaFieldDiffStat(field_name=field_name))
stat.compared_count += 1
stat.schema_refs.update(refs)
local_value = local_normalized.get(field_name)
remote_value = remote_normalized.get(field_name)
if local_value == remote_value:
continue
stat.mismatch_count += 1
if len(stat.samples) < self.sample_limit:
stat.samples.append(
SchemaDiffSample(
sample_id=current_sample_id,
local=local_value,
remote=remote_value,
)
)
diff_map[field_name] = {"local": local_value, "remote": remote_value}
if diff_map:
self.mismatched_pairs += 1
return diff_map
def build_report(self) -> dict[str, Any]:
fields = []
for stat in sorted(
self._field_stats.values(),
key=lambda item: (-item.mismatch_count, item.field_name),
):
if stat.compared_count <= 0:
continue
fields.append(
{
"field_name": stat.field_name,
"mismatch_count": stat.mismatch_count,
"total_count": stat.compared_count,
"mismatch_rate": stat.mismatch_count / stat.compared_count,
"schema_refs": sorted(stat.schema_refs),
"samples": [
{
"sample_id": sample.sample_id,
"local": sample.local,
"remote": sample.remote,
}
for sample in stat.samples
],
}
)
return {
"schema_name": self.schema_name,
"pairs_compared": self.pairs_compared,
"mismatched_pairs": self.mismatched_pairs,
"ignore_fields": sorted(self.ignore_fields),
"fields": fields,
}
def format_summary_lines(self, *, max_fields: int = 20, max_samples: int = 1) -> list[str]:
report = self.build_report()
lines = [
(
f"schema={report['schema_name']} pairs={report['pairs_compared']} "
f"mismatched_pairs={report['mismatched_pairs']} ignored={report['ignore_fields']}"
)
]
for field_report in report["fields"][:max_fields]:
line = (
f"field={field_report['field_name']} mismatch={field_report['mismatch_count']}/"
f"{field_report['total_count']} refs={field_report['schema_refs']}"
)
samples = field_report["samples"][:max_samples]
if samples:
sample = samples[0]
line += f" sample[{sample['sample_id']}]={sample['local']!r}/{sample['remote']!r}"
lines.append(line)
return lines