from __future__ import annotations import argparse import json import random import sqlite3 import sys from collections import Counter from dataclasses import dataclass from pathlib import Path from typing import Any REASON_DESCRIPTIONS = { "hierarchy_shape": "层级结构不一致:一侧是顶层/短路径,另一侧保留了父级、完整 path 或 parent 信息。", "region_enrichment": "地区信息补全程度不一致:远端带国家、省市名称或完整 region_path,本地缺失或更短。", "audit_identity": "审计人字段不一致:远端保留了 created_by/updated_by,本地为空或使用不同来源。", "audit_time_format": "时间格式或更新时间不一致:本地常用 ISO 时间,远端常用空格分隔时间,且更新时间可能已被远端二次维护。", "company_nature_semantics": "company_nature 语义不一致:本地是较保守的单值/少值,远端是人工维护后的多选性质。", "sort_policy": "排序策略不一致:本地默认值或来源字段与远端现存排序值不同。", "null_empty_normalization": "空值归一化不一致:本地用 null,远端用空字符串或空数组。", } @dataclass class PersistedNode: collection_id: str node_id: str node_type: str data_id: str | None bind_data_id: str | None action: str | None status: str | None binding_status: str | None error: str | None sync_log: str data: Any origin_data: Any context: Any @dataclass class BoundPair: node_type: str local_id: str remote_id: str local_node: PersistedNode remote_node: PersistedNode diffs: list[tuple[str, Any, Any]] def _classify_reasons(pair: BoundPair) -> list[str]: diff_paths = {path for path, _, _ in pair.diffs} reasons: list[str] = [] if diff_paths & {"parent_id", "parent_name", "path", "path_name"}: reasons.append("hierarchy_shape") if diff_paths & {"country", "country_name", "province_name", "city_name", "region_path", "province", "city"}: reasons.append("region_enrichment") if diff_paths & {"created_by", "updated_by", "created_by_name", "updated_by_name"}: reasons.append("audit_identity") if diff_paths & {"created_at", "updated_at"}: reasons.append("audit_time_format") if "company_nature" in diff_paths: reasons.append("company_nature_semantics") if "sort" in diff_paths: reasons.append("sort_policy") if any( (local_value is None and remote_value in ("", [])) or (remote_value is None and local_value in ("", [])) for _, local_value, remote_value in pair.diffs ): reasons.append("null_empty_normalization") return reasons def _build_reason_summary(mismatches: list[BoundPair]) -> str: counter: Counter[str] = Counter() for pair in mismatches: counter.update(_classify_reasons(pair)) if not counter: return "No mismatch reasons inferred." lines = ["Likely mismatch reasons:"] for reason, count in counter.most_common(): description = REASON_DESCRIPTIONS.get(reason, reason) lines.append(f" - {reason}: {count}") lines.append(f" {description}") return "\n".join(lines) def _build_business_conclusion(node_type: str, mismatches: list[BoundPair]) -> str: if not mismatches: return "No mismatched bound pairs found." reason_counts: Counter[str] = Counter() for pair in mismatches: reason_counts.update(_classify_reasons(pair)) if node_type == "company": conclusions = [ "company 当前不一致主要不是绑定错,而是本地 company handler 输出更扁平。", ] if reason_counts["hierarchy_shape"]: conclusions.append("远端 company 已维护集团层级,本地只保留当前公司自身,导致 parent_id/path/path_name 不一致。") if reason_counts["region_enrichment"]: conclusions.append("远端 company 还维护了省市编码与名称,本地目前未补这些字段。") if reason_counts["sort_policy"]: conclusions.append("sort 来自两套不同维护规则,本地默认值与远端现存排序不一致。") return "\n".join(f"- {item}" for item in conclusions) if node_type == "supplier": conclusions = [ "supplier 当前大多数绑定能命中,但远端 supplier 是一套人工维护后更完整的数据快照。", ] if reason_counts["company_nature_semantics"]: conclusions.append("本地 company_nature 往往只有单值,远端 supplier 常是多选性质,因此语义层面天然不一致。") if reason_counts["region_enrichment"]: conclusions.append("远端 supplier 补了国家层、地区名称和完整 region_path,本地只有编码级最小信息。") if reason_counts["audit_identity"] or reason_counts["audit_time_format"]: conclusions.append("远端保存了创建/更新操作者和远端维护时间,本地导出使用的是 DEPM 审计字段,时间格式也不同。") if reason_counts["hierarchy_shape"] or reason_counts["null_empty_normalization"]: conclusions.append("parent_id/path 的空值和路径表达也不同,本地更偏同步输入形态,远端更偏展示/运营形态。") return "\n".join(f"- {item}" for item in conclusions) return "\n".join(f"- {REASON_DESCRIPTIONS.get(reason, reason)}" for reason, _ in reason_counts.most_common()) def _parse_json(value: str | None) -> Any: if not value: return None try: return json.loads(value) except json.JSONDecodeError: return value def _format_json(value: Any) -> str: return json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) def _short_repr(value: Any, *, max_len: int = 160) -> str: text = repr(value) if len(text) <= max_len: return text return text[: max_len - 3] + "..." def _diff_values(local_value: Any, remote_value: Any, *, path: str = "") -> list[tuple[str, Any, Any]]: diffs: list[tuple[str, Any, Any]] = [] if type(local_value) is not type(remote_value): diffs.append((path or "", local_value, remote_value)) return diffs if isinstance(local_value, dict): keys = sorted(set(local_value.keys()) | set(remote_value.keys())) for key in keys: child_path = f"{path}.{key}" if path else str(key) if key not in local_value: diffs.append((child_path, None, remote_value[key])) continue if key not in remote_value: diffs.append((child_path, local_value[key], None)) continue diffs.extend(_diff_values(local_value[key], remote_value[key], path=child_path)) return diffs if isinstance(local_value, list): if local_value != remote_value: diffs.append((path or "", local_value, remote_value)) return diffs if local_value != remote_value: diffs.append((path or "", local_value, remote_value)) return diffs def _load_nodes(conn: sqlite3.Connection, node_type: str) -> dict[tuple[str, str], PersistedNode]: cur = conn.cursor() cur.execute( """ SELECT collection_id, node_id, node_type, data_id, bind_data_id, action, status, binding_status, error, sync_log, data, origin_data, context FROM nodes WHERE node_type = ? """, (node_type,), ) rows: dict[tuple[str, str], PersistedNode] = {} for row in cur.fetchall(): node = PersistedNode( collection_id=row[0], node_id=row[1], node_type=row[2], data_id=row[3], bind_data_id=row[4], action=row[5], status=row[6], binding_status=row[7], error=row[8], sync_log=row[9] or "", data=_parse_json(row[10]), origin_data=_parse_json(row[11]), context=_parse_json(row[12]), ) rows[(node.collection_id, node.node_id)] = node return rows def _load_bound_pairs(conn: sqlite3.Connection, node_type: str) -> list[BoundPair]: nodes = _load_nodes(conn, node_type) cur = conn.cursor() cur.execute( "SELECT node_type, local_id, remote_id FROM bindings WHERE node_type = ? AND remote_id IS NOT NULL", (node_type,), ) pairs: list[BoundPair] = [] for bound_node_type, local_id, remote_id in cur.fetchall(): local_node = nodes.get(("local", local_id)) remote_node = nodes.get(("remote", remote_id)) if local_node is None or remote_node is None: continue local_data = local_node.data or {} remote_data = remote_node.data or {} diffs = _diff_values(local_data, remote_data) pairs.append( BoundPair( node_type=bound_node_type, local_id=local_id, remote_id=remote_id, local_node=local_node, remote_node=remote_node, diffs=diffs, ) ) return pairs def _build_summary(pairs: list[BoundPair]) -> str: total = len(pairs) mismatches = [pair for pair in pairs if pair.diffs] matched = total - len(mismatches) diff_counter: Counter[str] = Counter() for pair in mismatches: for path, _, _ in pair.diffs: diff_counter[path] += 1 lines = [ f"Bound pairs: {total}", f"Matched pairs: {matched}", f"Mismatched pairs: {len(mismatches)}", ] if diff_counter: lines.append("Top diff fields:") for field, count in diff_counter.most_common(20): lines.append(f" - {field}: {count}") lines.append(_build_reason_summary(mismatches)) return "\n".join(lines) def _format_pair(pair: BoundPair, *, max_diff_items: int) -> str: diff_lines = [] for path, local_value, remote_value in pair.diffs[:max_diff_items]: diff_lines.append( f" - {path}: local={_short_repr(local_value)} | remote={_short_repr(remote_value)}" ) if len(pair.diffs) > max_diff_items: diff_lines.append(f" - ... {len(pair.diffs) - max_diff_items} more diff fields") return "\n".join( [ f"node_type={pair.node_type}", f"local.node_id={pair.local_node.node_id}", f"local.data_id={pair.local_node.data_id}", f"local.bind_data_id={pair.local_node.bind_data_id}", f"local.action={pair.local_node.action} local.status={pair.local_node.status} local.binding_status={pair.local_node.binding_status}", f"local.error={pair.local_node.error}", f"remote.node_id={pair.remote_node.node_id}", f"remote.data_id={pair.remote_node.data_id}", f"remote.bind_data_id={pair.remote_node.bind_data_id}", f"remote.action={pair.remote_node.action} remote.status={pair.remote_node.status} remote.binding_status={pair.remote_node.binding_status}", f"remote.error={pair.remote_node.error}", "Reason tags:", *(f" - {reason}: {REASON_DESCRIPTIONS.get(reason, reason)}" for reason in _classify_reasons(pair)), "Diff fields:", *diff_lines, "", "Local sync_log:", pair.local_node.sync_log or "", "", "Remote sync_log:", pair.remote_node.sync_log or "", "", "Local context:", _format_json(pair.local_node.context), "", "Remote context:", _format_json(pair.remote_node.context), "", "Local data:", _format_json(pair.local_node.data), "", "Remote data:", _format_json(pair.remote_node.data), "", "Local origin_data:", _format_json(pair.local_node.origin_data), "", "Remote origin_data:", _format_json(pair.remote_node.origin_data), ] ) def build_report( *, db_path: Path, node_type: str, limit: int, seed: int, max_diff_items: int, ) -> str: conn = sqlite3.connect(db_path) try: pairs = _load_bound_pairs(conn, node_type) finally: conn.close() mismatches = [pair for pair in pairs if pair.diffs] rng = random.Random(seed) sampled = list(mismatches) rng.shuffle(sampled) sampled = sampled[:limit] sections = [ f"# Persist Mismatch Report\n", f"db_path: {db_path}", f"node_type: {node_type}", f"sample_limit: {limit}", f"seed: {seed}", "", "## Summary", _build_summary(pairs), "", "## Business Conclusion", _build_business_conclusion(node_type, mismatches), ] if not sampled: sections.extend(["", "## Sampled Mismatches", "No mismatched bound pairs found."]) return "\n".join(sections) sections.append("") sections.append("## Sampled Mismatches") for index, pair in enumerate(sampled, start=1): sections.extend( [ "", f"### Sample {index}", _format_pair(pair, max_diff_items=max_diff_items), ] ) return "\n".join(sections) def parse_args(argv: list[str]) -> argparse.Namespace: parser = argparse.ArgumentParser(description="Analyze mismatched bound pairs in a persisted sync SQLite DB") parser.add_argument("db_path", help="Path to persisted sqlite db, e.g. backend/runtime/ecm_sync/.../depm_to_ecm.db") parser.add_argument("--node-type", required=True, help="Node type to analyze, e.g. company or supplier") parser.add_argument("--limit", type=int, default=3, help="Number of mismatched bound pairs to sample") parser.add_argument("--seed", type=int, default=11, help="Random seed for sampling") parser.add_argument("--max-diff-items", type=int, default=20, help="Max diff fields to print per sampled pair") parser.add_argument("--output", help="Optional output file path. If omitted, prints to stdout.") return parser.parse_args(argv) def main(argv: list[str] | None = None) -> int: args = parse_args(argv or sys.argv[1:]) db_path = Path(args.db_path).resolve() if not db_path.exists(): print(f"DB file not found: {db_path}", file=sys.stderr) return 1 report = build_report( db_path=db_path, node_type=args.node_type, limit=max(0, args.limit), seed=args.seed, max_diff_items=max(1, args.max_diff_items), ) if args.output: output_path = Path(args.output).resolve() output_path.parent.mkdir(parents=True, exist_ok=True) output_path.write_text(report, encoding="utf-8") print(f"Report written to: {output_path}") return 0 print(report) return 0 if __name__ == "__main__": raise SystemExit(main())