strepsiades
414 lines
15 KiB
Python
414 lines
15 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import random
|
|
import sqlite3
|
|
import sys
|
|
from collections import Counter
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
REASON_DESCRIPTIONS = {
|
|
"hierarchy_shape": "层级结构不一致：一侧是顶层/短路径，另一侧保留了父级、完整 path 或 parent 信息。",
|
|
"region_enrichment": "地区信息补全程度不一致：远端带国家、省市名称或完整 region_path，本地缺失或更短。",
|
|
"audit_identity": "审计人字段不一致：远端保留了 created_by/updated_by，本地为空或使用不同来源。",
|
|
"audit_time_format": "时间格式或更新时间不一致：本地常用 ISO 时间，远端常用空格分隔时间，且更新时间可能已被远端二次维护。",
|
|
"company_nature_semantics": "company_nature 语义不一致：本地是较保守的单值/少值，远端是人工维护后的多选性质。",
|
|
"sort_policy": "排序策略不一致：本地默认值或来源字段与远端现存排序值不同。",
|
|
"null_empty_normalization": "空值归一化不一致：本地用 null，远端用空字符串或空数组。",
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class PersistedNode:
|
|
collection_id: str
|
|
node_id: str
|
|
node_type: str
|
|
data_id: str | None
|
|
bind_data_id: str | None
|
|
action: str | None
|
|
status: str | None
|
|
binding_status: str | None
|
|
error: str | None
|
|
sync_log: str
|
|
data: Any
|
|
origin_data: Any
|
|
context: Any
|
|
|
|
|
|
@dataclass
|
|
class BoundPair:
|
|
node_type: str
|
|
local_id: str
|
|
remote_id: str
|
|
local_node: PersistedNode
|
|
remote_node: PersistedNode
|
|
diffs: list[tuple[str, Any, Any]]
|
|
|
|
|
|
def _classify_reasons(pair: BoundPair) -> list[str]:
|
|
diff_paths = {path for path, _, _ in pair.diffs}
|
|
reasons: list[str] = []
|
|
|
|
if diff_paths & {"parent_id", "parent_name", "path", "path_name"}:
|
|
reasons.append("hierarchy_shape")
|
|
|
|
if diff_paths & {"country", "country_name", "province_name", "city_name", "region_path", "province", "city"}:
|
|
reasons.append("region_enrichment")
|
|
|
|
if diff_paths & {"created_by", "updated_by", "created_by_name", "updated_by_name"}:
|
|
reasons.append("audit_identity")
|
|
|
|
if diff_paths & {"created_at", "updated_at"}:
|
|
reasons.append("audit_time_format")
|
|
|
|
if "company_nature" in diff_paths:
|
|
reasons.append("company_nature_semantics")
|
|
|
|
if "sort" in diff_paths:
|
|
reasons.append("sort_policy")
|
|
|
|
if any(
|
|
(local_value is None and remote_value in ("", [])) or (remote_value is None and local_value in ("", []))
|
|
for _, local_value, remote_value in pair.diffs
|
|
):
|
|
reasons.append("null_empty_normalization")
|
|
|
|
return reasons
|
|
|
|
|
|
def _build_reason_summary(mismatches: list[BoundPair]) -> str:
|
|
counter: Counter[str] = Counter()
|
|
for pair in mismatches:
|
|
counter.update(_classify_reasons(pair))
|
|
|
|
if not counter:
|
|
return "No mismatch reasons inferred."
|
|
|
|
lines = ["Likely mismatch reasons:"]
|
|
for reason, count in counter.most_common():
|
|
description = REASON_DESCRIPTIONS.get(reason, reason)
|
|
lines.append(f" - {reason}: {count}")
|
|
lines.append(f" {description}")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def _build_business_conclusion(node_type: str, mismatches: list[BoundPair]) -> str:
|
|
if not mismatches:
|
|
return "No mismatched bound pairs found."
|
|
|
|
reason_counts: Counter[str] = Counter()
|
|
for pair in mismatches:
|
|
reason_counts.update(_classify_reasons(pair))
|
|
|
|
if node_type == "company":
|
|
conclusions = [
|
|
"company 当前不一致主要不是绑定错，而是本地 company handler 输出更扁平。",
|
|
]
|
|
if reason_counts["hierarchy_shape"]:
|
|
conclusions.append("远端 company 已维护集团层级，本地只保留当前公司自身，导致 parent_id/path/path_name 不一致。")
|
|
if reason_counts["region_enrichment"]:
|
|
conclusions.append("远端 company 还维护了省市编码与名称，本地目前未补这些字段。")
|
|
if reason_counts["sort_policy"]:
|
|
conclusions.append("sort 来自两套不同维护规则，本地默认值与远端现存排序不一致。")
|
|
return "\n".join(f"- {item}" for item in conclusions)
|
|
|
|
if node_type == "supplier":
|
|
conclusions = [
|
|
"supplier 当前大多数绑定能命中，但远端 supplier 是一套人工维护后更完整的数据快照。",
|
|
]
|
|
if reason_counts["company_nature_semantics"]:
|
|
conclusions.append("本地 company_nature 往往只有单值，远端 supplier 常是多选性质，因此语义层面天然不一致。")
|
|
if reason_counts["region_enrichment"]:
|
|
conclusions.append("远端 supplier 补了国家层、地区名称和完整 region_path，本地只有编码级最小信息。")
|
|
if reason_counts["audit_identity"] or reason_counts["audit_time_format"]:
|
|
conclusions.append("远端保存了创建/更新操作者和远端维护时间，本地导出使用的是 DEPM 审计字段，时间格式也不同。")
|
|
if reason_counts["hierarchy_shape"] or reason_counts["null_empty_normalization"]:
|
|
conclusions.append("parent_id/path 的空值和路径表达也不同，本地更偏同步输入形态，远端更偏展示/运营形态。")
|
|
return "\n".join(f"- {item}" for item in conclusions)
|
|
|
|
return "\n".join(f"- {REASON_DESCRIPTIONS.get(reason, reason)}" for reason, _ in reason_counts.most_common())
|
|
|
|
|
|
def _parse_json(value: str | None) -> Any:
|
|
if not value:
|
|
return None
|
|
try:
|
|
return json.loads(value)
|
|
except json.JSONDecodeError:
|
|
return value
|
|
|
|
|
|
def _format_json(value: Any) -> str:
|
|
return json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True)
|
|
|
|
|
|
def _short_repr(value: Any, *, max_len: int = 160) -> str:
|
|
text = repr(value)
|
|
if len(text) <= max_len:
|
|
return text
|
|
return text[: max_len - 3] + "..."
|
|
|
|
|
|
def _diff_values(local_value: Any, remote_value: Any, *, path: str = "") -> list[tuple[str, Any, Any]]:
|
|
diffs: list[tuple[str, Any, Any]] = []
|
|
|
|
if type(local_value) is not type(remote_value):
|
|
diffs.append((path or "<root>", local_value, remote_value))
|
|
return diffs
|
|
|
|
if isinstance(local_value, dict):
|
|
keys = sorted(set(local_value.keys()) | set(remote_value.keys()))
|
|
for key in keys:
|
|
child_path = f"{path}.{key}" if path else str(key)
|
|
if key not in local_value:
|
|
diffs.append((child_path, None, remote_value[key]))
|
|
continue
|
|
if key not in remote_value:
|
|
diffs.append((child_path, local_value[key], None))
|
|
continue
|
|
diffs.extend(_diff_values(local_value[key], remote_value[key], path=child_path))
|
|
return diffs
|
|
|
|
if isinstance(local_value, list):
|
|
if local_value != remote_value:
|
|
diffs.append((path or "<root>", local_value, remote_value))
|
|
return diffs
|
|
|
|
if local_value != remote_value:
|
|
diffs.append((path or "<root>", local_value, remote_value))
|
|
return diffs
|
|
|
|
|
|
def _load_nodes(conn: sqlite3.Connection, node_type: str) -> dict[tuple[str, str], PersistedNode]:
|
|
cur = conn.cursor()
|
|
cur.execute(
|
|
"""
|
|
SELECT collection_id, node_id, node_type, data_id, bind_data_id, action, status,
|
|
binding_status, error, sync_log, data, origin_data, context
|
|
FROM nodes
|
|
WHERE node_type = ?
|
|
""",
|
|
(node_type,),
|
|
)
|
|
|
|
rows: dict[tuple[str, str], PersistedNode] = {}
|
|
for row in cur.fetchall():
|
|
node = PersistedNode(
|
|
collection_id=row[0],
|
|
node_id=row[1],
|
|
node_type=row[2],
|
|
data_id=row[3],
|
|
bind_data_id=row[4],
|
|
action=row[5],
|
|
status=row[6],
|
|
binding_status=row[7],
|
|
error=row[8],
|
|
sync_log=row[9] or "",
|
|
data=_parse_json(row[10]),
|
|
origin_data=_parse_json(row[11]),
|
|
context=_parse_json(row[12]),
|
|
)
|
|
rows[(node.collection_id, node.node_id)] = node
|
|
return rows
|
|
|
|
|
|
def _load_bound_pairs(conn: sqlite3.Connection, node_type: str) -> list[BoundPair]:
|
|
nodes = _load_nodes(conn, node_type)
|
|
cur = conn.cursor()
|
|
cur.execute(
|
|
"SELECT node_type, local_id, remote_id FROM bindings WHERE node_type = ? AND remote_id IS NOT NULL",
|
|
(node_type,),
|
|
)
|
|
|
|
pairs: list[BoundPair] = []
|
|
for bound_node_type, local_id, remote_id in cur.fetchall():
|
|
local_node = nodes.get(("local", local_id))
|
|
remote_node = nodes.get(("remote", remote_id))
|
|
if local_node is None or remote_node is None:
|
|
continue
|
|
local_data = local_node.data or {}
|
|
remote_data = remote_node.data or {}
|
|
diffs = _diff_values(local_data, remote_data)
|
|
pairs.append(
|
|
BoundPair(
|
|
node_type=bound_node_type,
|
|
local_id=local_id,
|
|
remote_id=remote_id,
|
|
local_node=local_node,
|
|
remote_node=remote_node,
|
|
diffs=diffs,
|
|
)
|
|
)
|
|
return pairs
|
|
|
|
|
|
def _build_summary(pairs: list[BoundPair]) -> str:
|
|
total = len(pairs)
|
|
mismatches = [pair for pair in pairs if pair.diffs]
|
|
matched = total - len(mismatches)
|
|
diff_counter: Counter[str] = Counter()
|
|
for pair in mismatches:
|
|
for path, _, _ in pair.diffs:
|
|
diff_counter[path] += 1
|
|
|
|
lines = [
|
|
f"Bound pairs: {total}",
|
|
f"Matched pairs: {matched}",
|
|
f"Mismatched pairs: {len(mismatches)}",
|
|
]
|
|
if diff_counter:
|
|
lines.append("Top diff fields:")
|
|
for field, count in diff_counter.most_common(20):
|
|
lines.append(f" - {field}: {count}")
|
|
lines.append(_build_reason_summary(mismatches))
|
|
return "\n".join(lines)
|
|
|
|
|
|
def _format_pair(pair: BoundPair, *, max_diff_items: int) -> str:
|
|
diff_lines = []
|
|
for path, local_value, remote_value in pair.diffs[:max_diff_items]:
|
|
diff_lines.append(
|
|
f" - {path}: local={_short_repr(local_value)} | remote={_short_repr(remote_value)}"
|
|
)
|
|
if len(pair.diffs) > max_diff_items:
|
|
diff_lines.append(f" - ... {len(pair.diffs) - max_diff_items} more diff fields")
|
|
|
|
return "\n".join(
|
|
[
|
|
f"node_type={pair.node_type}",
|
|
f"local.node_id={pair.local_node.node_id}",
|
|
f"local.data_id={pair.local_node.data_id}",
|
|
f"local.bind_data_id={pair.local_node.bind_data_id}",
|
|
f"local.action={pair.local_node.action} local.status={pair.local_node.status} local.binding_status={pair.local_node.binding_status}",
|
|
f"local.error={pair.local_node.error}",
|
|
f"remote.node_id={pair.remote_node.node_id}",
|
|
f"remote.data_id={pair.remote_node.data_id}",
|
|
f"remote.bind_data_id={pair.remote_node.bind_data_id}",
|
|
f"remote.action={pair.remote_node.action} remote.status={pair.remote_node.status} remote.binding_status={pair.remote_node.binding_status}",
|
|
f"remote.error={pair.remote_node.error}",
|
|
"Reason tags:",
|
|
*(f" - {reason}: {REASON_DESCRIPTIONS.get(reason, reason)}" for reason in _classify_reasons(pair)),
|
|
"Diff fields:",
|
|
*diff_lines,
|
|
"",
|
|
"Local sync_log:",
|
|
pair.local_node.sync_log or "<empty>",
|
|
"",
|
|
"Remote sync_log:",
|
|
pair.remote_node.sync_log or "<empty>",
|
|
"",
|
|
"Local context:",
|
|
_format_json(pair.local_node.context),
|
|
"",
|
|
"Remote context:",
|
|
_format_json(pair.remote_node.context),
|
|
"",
|
|
"Local data:",
|
|
_format_json(pair.local_node.data),
|
|
"",
|
|
"Remote data:",
|
|
_format_json(pair.remote_node.data),
|
|
"",
|
|
"Local origin_data:",
|
|
_format_json(pair.local_node.origin_data),
|
|
"",
|
|
"Remote origin_data:",
|
|
_format_json(pair.remote_node.origin_data),
|
|
]
|
|
)
|
|
|
|
|
|
def build_report(
|
|
*,
|
|
db_path: Path,
|
|
node_type: str,
|
|
limit: int,
|
|
seed: int,
|
|
max_diff_items: int,
|
|
) -> str:
|
|
conn = sqlite3.connect(db_path)
|
|
try:
|
|
pairs = _load_bound_pairs(conn, node_type)
|
|
finally:
|
|
conn.close()
|
|
|
|
mismatches = [pair for pair in pairs if pair.diffs]
|
|
rng = random.Random(seed)
|
|
sampled = list(mismatches)
|
|
rng.shuffle(sampled)
|
|
sampled = sampled[:limit]
|
|
|
|
sections = [
|
|
f"# Persist Mismatch Report\n",
|
|
f"db_path: {db_path}",
|
|
f"node_type: {node_type}",
|
|
f"sample_limit: {limit}",
|
|
f"seed: {seed}",
|
|
"",
|
|
"## Summary",
|
|
_build_summary(pairs),
|
|
"",
|
|
"## Business Conclusion",
|
|
_build_business_conclusion(node_type, mismatches),
|
|
]
|
|
|
|
if not sampled:
|
|
sections.extend(["", "## Sampled Mismatches", "No mismatched bound pairs found."])
|
|
return "\n".join(sections)
|
|
|
|
sections.append("")
|
|
sections.append("## Sampled Mismatches")
|
|
for index, pair in enumerate(sampled, start=1):
|
|
sections.extend(
|
|
[
|
|
"",
|
|
f"### Sample {index}",
|
|
_format_pair(pair, max_diff_items=max_diff_items),
|
|
]
|
|
)
|
|
return "\n".join(sections)
|
|
|
|
|
|
def parse_args(argv: list[str]) -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description="Analyze mismatched bound pairs in a persisted sync SQLite DB")
|
|
parser.add_argument("db_path", help="Path to persisted sqlite db, e.g. backend/runtime/ecm_sync/.../depm_to_ecm.db")
|
|
parser.add_argument("--node-type", required=True, help="Node type to analyze, e.g. company or supplier")
|
|
parser.add_argument("--limit", type=int, default=3, help="Number of mismatched bound pairs to sample")
|
|
parser.add_argument("--seed", type=int, default=11, help="Random seed for sampling")
|
|
parser.add_argument("--max-diff-items", type=int, default=20, help="Max diff fields to print per sampled pair")
|
|
parser.add_argument("--output", help="Optional output file path. If omitted, prints to stdout.")
|
|
return parser.parse_args(argv)
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = parse_args(argv or sys.argv[1:])
|
|
db_path = Path(args.db_path).resolve()
|
|
if not db_path.exists():
|
|
print(f"DB file not found: {db_path}", file=sys.stderr)
|
|
return 1
|
|
|
|
report = build_report(
|
|
db_path=db_path,
|
|
node_type=args.node_type,
|
|
limit=max(0, args.limit),
|
|
seed=args.seed,
|
|
max_diff_items=max(1, args.max_diff_items),
|
|
)
|
|
|
|
if args.output:
|
|
output_path = Path(args.output).resolve()
|
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
output_path.write_text(report, encoding="utf-8")
|
|
print(f"Report written to: {output_path}")
|
|
return 0
|
|
|
|
print(report)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main()) |