strepsiades
183 lines
8.2 KiB
Python
183 lines
8.2 KiB
Python
from __future__ import annotations
|
|
|
|
import logging
|
|
|
|
|
|
async def print_pipeline_summary(*, logger, node_types, binding_manager, local_datasource, remote_datasource, local_collection, remote_collection, consistency_by_type, stats) -> None:
|
|
scoped_logger = logger
|
|
plain_logger = logger.logger if isinstance(logger, logging.LoggerAdapter) else logger
|
|
|
|
def _log_header(message: str) -> None:
|
|
scoped_logger.info(message)
|
|
|
|
def _log_body(message: str) -> None:
|
|
plain_logger.info(message)
|
|
|
|
binding_counts = {}
|
|
for node_type in node_types:
|
|
records = await binding_manager.get_all_records(node_type)
|
|
binding_counts[node_type] = len(records)
|
|
|
|
def format_action_sft(summary: dict, key: str) -> str:
|
|
action = summary.get(key, {})
|
|
success = int(action.get("success", 0))
|
|
failed = int(action.get("failed", 0)) + int(action.get("skipped", 0))
|
|
total = int(action.get("total", 0))
|
|
return f"{success}/{failed}/{total}"
|
|
|
|
def format_consistency_sft(node_type: str) -> str:
|
|
consistency = consistency_by_type.get(node_type, {})
|
|
checked = int(consistency.get("checked_pairs", 0))
|
|
mismatch = int(consistency.get("mismatch_count", 0))
|
|
matched = max(0, checked - mismatch)
|
|
return f"{matched}/{mismatch}/{checked}"
|
|
|
|
def _clip(text: str, width: int) -> str:
|
|
if len(text) <= width:
|
|
return text
|
|
if width <= 1:
|
|
return text[:width]
|
|
return text[: width - 1] + "…"
|
|
|
|
def print_consistency_summary() -> None:
|
|
line_width = 96
|
|
|
|
def _format_sample(sample: dict) -> str:
|
|
local_text = _clip(repr(sample.get("local")), 72)
|
|
remote_text = _clip(repr(sample.get("remote")), 72)
|
|
return f"{local_text}/{remote_text}"
|
|
|
|
_log_header(f"\n{'=' * line_width}")
|
|
_log_header("🔎 Consistency Summary")
|
|
_log_header(f"{'=' * line_width}")
|
|
|
|
for node_type in node_types:
|
|
item = consistency_by_type.get(node_type, {}) or {}
|
|
checked_pairs = int(item.get("checked_pairs", 0))
|
|
mismatch_count = int(item.get("mismatch_count", 0))
|
|
_log_body(f"{node_type} (checked_pairs={checked_pairs}, mismatches={mismatch_count})")
|
|
|
|
fields = item.get("fields", []) or []
|
|
differing_fields = [field for field in fields if int(field.get("mismatch_count", 0)) > 0]
|
|
if not differing_fields:
|
|
_log_body(" pass")
|
|
continue
|
|
|
|
_log_body(" field mismatch/total samples")
|
|
_log_body(" -----------------------------------------------------------")
|
|
for field_report in differing_fields:
|
|
field_name = str(field_report.get("field_name", ""))
|
|
field_mismatch = int(field_report.get("mismatch_count", 0))
|
|
field_total = int(field_report.get("total_count", 0))
|
|
samples = field_report.get("samples", []) or []
|
|
sample_text = _format_sample(samples[0]) if samples else "-"
|
|
_log_body(
|
|
f" {_clip(field_name, 30):<30} {field_mismatch:>4}/{field_total:<4} "
|
|
f"{sample_text}"
|
|
)
|
|
|
|
def print_collection_summary(title: str, datasource, collection) -> None:
|
|
action_summary = datasource.get_action_summary()
|
|
line_width = 133
|
|
|
|
node_type_w = 28
|
|
bound_w = 16
|
|
action_w = 16
|
|
consistency_w = 18
|
|
|
|
_log_header(f"\n{'='*line_width}")
|
|
_log_header(f"📊 {title} Summary")
|
|
_log_header(f"{'='*line_width}")
|
|
_log_body("说明: Create/Update/Delete = 成功/失败/总数 (失败=FAILED+SKIPPED)")
|
|
_log_body("说明: Consistent = 一致对数/不一致对数/已检查绑定对数")
|
|
_log_body(
|
|
f"{'Node Type':<{node_type_w}} {'Bound/Rec/Load':<{bound_w}} {'Create(S/F/T)':<{action_w}} {'Update(S/F/T)':<{action_w}} {'Delete(S/F/T)':<{action_w}} {'Consistent(S/F/T)':<{consistency_w}}"
|
|
)
|
|
_log_body(f"{'-'*line_width}")
|
|
|
|
total_bound_normal = 0
|
|
total_bindings = 0
|
|
total_loaded = 0
|
|
total_create = {"total": 0, "success": 0, "failed_effective": 0}
|
|
total_update = {"total": 0, "success": 0, "failed_effective": 0}
|
|
total_delete = {"total": 0, "success": 0, "failed_effective": 0}
|
|
total_consistency = {"checked": 0, "mismatch": 0}
|
|
|
|
for node_type in node_types:
|
|
summary = action_summary.get(node_type, {})
|
|
bindings = binding_counts.get(node_type, 0)
|
|
nodes = collection.filter(node_type=node_type)
|
|
loaded_count = len(nodes)
|
|
bound_normal_count = len([n for n in nodes if n.binding_status.value == "normal"])
|
|
|
|
create_str = format_action_sft(summary, "create")
|
|
update_str = format_action_sft(summary, "update")
|
|
delete_str = format_action_sft(summary, "delete")
|
|
consistency_str = format_consistency_sft(node_type)
|
|
|
|
bound_str = f"{bound_normal_count}/{bindings}/{loaded_count}"
|
|
_log_body(
|
|
f"{_clip(node_type, node_type_w):<{node_type_w}} "
|
|
f"{_clip(bound_str, bound_w):<{bound_w}} "
|
|
f"{_clip(create_str, action_w):<{action_w}} "
|
|
f"{_clip(update_str, action_w):<{action_w}} "
|
|
f"{_clip(delete_str, action_w):<{action_w}} "
|
|
f"{_clip(consistency_str, consistency_w):<{consistency_w}}"
|
|
)
|
|
|
|
total_bound_normal += bound_normal_count
|
|
total_bindings += bindings
|
|
total_loaded += loaded_count
|
|
for bucket, key in (
|
|
(total_create, "create"),
|
|
(total_update, "update"),
|
|
(total_delete, "delete"),
|
|
):
|
|
action = summary.get(key, {})
|
|
bucket["total"] += action.get("total", 0)
|
|
bucket["success"] += action.get("success", 0)
|
|
bucket["failed_effective"] += action.get("failed", 0) + action.get("skipped", 0)
|
|
|
|
consistency = consistency_by_type.get(node_type, {})
|
|
total_consistency["checked"] += int(consistency.get("checked_pairs", 0))
|
|
total_consistency["mismatch"] += int(consistency.get("mismatch_count", 0))
|
|
|
|
_log_body(f"{'-'*line_width}")
|
|
total_consistent = max(0, total_consistency["checked"] - total_consistency["mismatch"])
|
|
total_create_sft = f"{total_create['success']}/{total_create['failed_effective']}/{total_create['total']}"
|
|
total_update_sft = f"{total_update['success']}/{total_update['failed_effective']}/{total_update['total']}"
|
|
total_delete_sft = f"{total_delete['success']}/{total_delete['failed_effective']}/{total_delete['total']}"
|
|
total_consistency_sft = f"{total_consistent}/{total_consistency['mismatch']}/{total_consistency['checked']}"
|
|
total_bound_str = f"{total_bound_normal}/{total_bindings}/{total_loaded}"
|
|
_log_body(
|
|
f"{'TOTAL':<{node_type_w}} "
|
|
f"{_clip(total_bound_str, bound_w):<{bound_w}} "
|
|
f"{_clip(total_create_sft, action_w):<{action_w}} "
|
|
f"{_clip(total_update_sft, action_w):<{action_w}} "
|
|
f"{_clip(total_delete_sft, action_w):<{action_w}} "
|
|
f"{_clip(total_consistency_sft, consistency_w):<{consistency_w}}"
|
|
)
|
|
|
|
mismatch_node_types = [
|
|
node_type
|
|
for node_type, item in consistency_by_type.items()
|
|
if int(item.get("mismatch_count", 0)) > 0
|
|
]
|
|
|
|
loaded = int(stats.get("loaded", 0))
|
|
synced = int(stats.get("synced", 0))
|
|
failed = int(stats.get("failed", 0))
|
|
post_check_passed = bool(stats.get("post_check_passed", True))
|
|
|
|
_log_header("\n" + "=" * 96)
|
|
_log_header(
|
|
f"📈 Pipeline Stats: loaded={loaded}, synced={synced}, failed={failed}, post_check_passed={post_check_passed}, mismatch_types={len(mismatch_node_types)}"
|
|
)
|
|
if mismatch_node_types:
|
|
_log_body(f"🔎 Mismatch Node Types: {', '.join(mismatch_node_types)}")
|
|
_log_header("=" * 96)
|
|
|
|
print_consistency_summary()
|
|
print_collection_summary("Local", local_datasource, local_collection)
|
|
print_collection_summary("Remote", remote_datasource, remote_collection)
|