Files
ecm_sync_system/scripts/run_auto_test.py
T
strepsiades 4a9c444b10 first commit
2026-03-09 16:31:42 +08:00

554 lines
24 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
from __future__ import annotations
import argparse
import importlib.util
import json
import shutil
from pathlib import Path
from typing import Any
from auto_test_init import clear_environment, initialize_environment
from auto_test_lib import (
load_yaml_config,
now_slug,
recreate_dir,
tail_text_file,
)
def load_domain_module(config: dict[str, Any], domain_name: str):
script_dir = Path(config["domains"]["script_dir"])
script_path = script_dir / f"{domain_name}_case.py"
module_name = f"auto_test_domain_{domain_name}"
spec = importlib.util.spec_from_file_location(module_name, script_path)
if spec is None or spec.loader is None:
raise ImportError(f"无法加载 domain 脚本: {script_path}")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def run_domain(config: dict[str, Any], domain_name: str, report_root: Path) -> dict[str, Any]:
module = load_domain_module(config, domain_name)
if not hasattr(module, "run_case"):
raise AttributeError(f"domain 脚本缺少 run_case(): {domain_name}")
domain_report = module.run_case(config, report_root)
report_path = report_root / f"{domain_name}.json"
report_path.write_text(json.dumps(domain_report, ensure_ascii=False, indent=2), encoding="utf-8")
return domain_report
def archive_run_logs(report_root: Path, init_result: dict[str, Any], domain_reports: list[dict[str, Any]]) -> None:
logs_dir = report_root / "logs"
logs_dir.mkdir(parents=True, exist_ok=True)
manifest: dict[str, Any] = {
"backend_log": None,
"pipeline_logs": [],
}
backend_log_file = init_result.get("backend_log_file")
if backend_log_file:
src = Path(backend_log_file)
if src.exists():
target = logs_dir / "backend.log"
shutil.copy2(src, target)
init_result["archived_backend_log_file"] = str(target)
manifest["backend_log"] = str(target)
for domain in domain_reports:
for round_report in domain.get("rounds", []):
latest_log_file = round_report.get("latest_log_file")
if not latest_log_file:
continue
src = Path(latest_log_file)
if not src.exists():
continue
target = logs_dir / f"{domain['domain']}__{round_report['name']}.log"
shutil.copy2(src, target)
round_report["archived_log_file"] = str(target)
manifest["pipeline_logs"].append(
{
"domain": domain["domain"],
"round": round_report["name"],
"source": str(src),
"archived": str(target),
}
)
(logs_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
def build_test_summary(domain_reports: list[dict[str, Any]]) -> dict[str, Any]:
cases: list[dict[str, Any]] = []
passed_domains = 0
failed_domains = 0
for domain in domain_reports:
domain_passed = bool(domain.get("passed"))
if domain_passed:
passed_domains += 1
else:
failed_domains += 1
for round_report in domain.get("rounds", []):
cases.append(
{
"nodeid": f"{domain['domain']}::{round_report['name']}",
"domain": domain["domain"],
"case": round_report["name"],
"purpose": round_report.get("purpose", ""),
"passed": bool(round_report.get("passed")),
"result": "PASSED" if round_report.get("passed") else "FAILED",
"returncode": round_report.get("returncode"),
"issue_count": len(round_report.get("issues", [])),
"issues": round_report.get("issues", []),
}
)
passed_cases = sum(1 for item in cases if item["passed"])
failed_cases = len(cases) - passed_cases
return {
"total_domains": len(domain_reports),
"passed_domains": passed_domains,
"failed_domains": failed_domains,
"total_cases": len(cases),
"passed_cases": passed_cases,
"failed_cases": failed_cases,
"cases": cases,
}
def build_markdown_report(run_id: str, init_result: dict[str, Any], domain_reports: list[dict[str, Any]]) -> str:
summary = build_test_summary(domain_reports)
if init_result.get("skipped"):
init_lines = ["- 本次跳过初始化,沿用现有测试环境"]
else:
init_lines = [
f"- 后端进程 PID: {init_result.get('backend_pid')}",
f"- 后端日志: {init_result.get('backend_log_file')}",
f"- 状态库: {init_result.get('persistence_db')}",
f"- 临时数据目录: {init_result.get('temp_data_dir')}",
]
lines = [
f"# 自动测试报告 {run_id}",
"",
"## 初始化结果",
"",
*init_lines,
"",
]
if domain_reports:
lines.extend([
"## Pytest 风格汇总",
"",
f"collected {summary['total_cases']} cases across {summary['total_domains']} domains",
"",
])
for case in summary["cases"]:
lines.append(f"- {case['nodeid']} {case['result']}")
lines.extend([
"",
f"== {summary['passed_cases']} passed, {summary['failed_cases']} failed, {summary['passed_domains']} passed domains, {summary['failed_domains']} failed domains ==",
"",
])
lines.extend([
"## Domain 汇总",
"",
"| Domain | 说明 | 结果 | 轮次数 | 问题数 |",
"| --- | --- | --- | --- | --- |",
])
for domain in domain_reports:
lines.append(
f"| {domain['domain']} | {domain.get('description', '')} | {'通过' if domain.get('passed') else '失败'} | {len(domain.get('rounds', []))} | {len(domain.get('issues', []))} |"
)
lines.append("")
for domain in domain_reports:
lines.append(f"## Domain: {domain['domain']}")
if domain.get("description"):
lines.append("")
lines.append(domain["description"])
lines.append("")
lines.append(f"- 结果: {'通过' if domain.get('passed') else '失败'}")
if domain.get("issues"):
lines.append("- 问题:")
for issue in domain["issues"]:
lines.append(f" - {issue}")
lines.append("")
for round_report in domain.get("rounds", []):
lines.extend([
f"### {round_report['name']}",
f"- 目的: {round_report['purpose']}",
f"- 返回码: {round_report['returncode']}",
f"- 结果: {'通过' if round_report.get('passed') else '失败'}",
f"- 数据目录: {round_report['dataset_dir']}",
f"- 最新日志: {round_report['latest_log_file']}",
])
coverage = round_report.get("coverage") or {}
if coverage:
if coverage.get("keys"):
lines.append(f"- 覆盖 key: {', '.join(coverage['keys'])}")
if coverage.get("api"):
lines.append(f"- 覆盖接口: {', '.join(coverage['api'])}")
if coverage.get("notes"):
lines.append(f"- 覆盖说明: {coverage['notes']}")
if round_report.get("archived_log_file"):
lines.append(f"- 归档日志: {round_report['archived_log_file']}")
if round_report.get("issues"):
lines.append("- 本轮问题:")
for issue in round_report["issues"]:
lines.append(f" - {issue}")
dataset_diffs = round_report.get("dataset_diffs") or {}
if dataset_diffs:
lines.append("- 本轮数据变更:")
for node_type, record_map in dataset_diffs.items():
for record_id, fields in record_map.items():
lines.append(f" - {node_type}.{record_id}: {', '.join(fields)}")
bindings = round_report.get("required_bindings", {})
if bindings:
lines.append("- 绑定检查:")
for node_type, items in bindings.items():
for item in items:
lines.append(
f" - {node_type} {item['local_data_id']}: bound={item['bound']} local_node_id={item['local_node_id']} remote_data_id={item['remote_data_id']}"
)
verified_bound_fields = round_report.get("verified_bound_fields") or []
if verified_bound_fields:
lines.append("- 字段校验:")
for item in verified_bound_fields:
status = "matched" if item.get("matched") else "mismatched"
lines.append(
f" - {item['collection_id']}.{item['node_type']} local={item['local_data_id']} target={item['target_data_id']} status={status}"
)
for field_path, actual_value in (item.get("actual_fields") or {}).items():
expected_value = (item.get("expected_fields") or {}).get(field_path)
lines.append(
f" - {field_path}: actual={actual_value!r} expected={expected_value!r}"
)
problem_nodes = round_report.get("problem_nodes", [])
if problem_nodes:
lines.append("- 问题节点:")
for item in problem_nodes:
lines.append(
f" - {item['collection_id']}.{item['node_type']} data_id={item.get('data_id')} action={item.get('action')} status={item.get('status')} binding_status={item.get('binding_status')} error={item.get('error') or '<empty>'}"
)
lines.append("")
return "\n".join(lines)
def sanitize_config_for_record(config: dict[str, Any]) -> dict[str, Any]:
payload = json.loads(json.dumps(config, ensure_ascii=False))
remote = payload.get("remote_datasource", {})
if remote.get("api_secret"):
remote["api_secret"] = "***"
backend = payload.get("backend", {})
backend_env = backend.get("env", {})
for key in ["COUCHDB_PASSWORD"]:
if backend_env.get(key):
backend_env[key] = "***"
couchdb = payload.get("couchdb", {})
if couchdb.get("password"):
couchdb["password"] = "***"
return payload
def collect_failed_node_details(domain_reports: list[dict[str, Any]]) -> list[dict[str, Any]]:
failed_nodes: list[dict[str, Any]] = []
for domain in domain_reports:
for round_report in domain.get("rounds", []):
for item in round_report.get("problem_nodes", []):
failed_nodes.append(
{
"domain": domain["domain"],
"round": round_report["name"],
"node_type": item.get("node_type"),
"collection_id": item.get("collection_id"),
"local_data_id": item.get("data_id"),
"local_node_id": item.get("node_id"),
"remote_node_id": None,
"remote_data_id": item.get("bind_data_id"),
"issues": round_report.get("issues", []),
"node_status": round_report.get("persistence_summary", {}).get("node_status", []),
"status": item.get("status"),
"binding_status": item.get("binding_status"),
"error": item.get("error"),
"sync_log_tail": item.get("sync_log_tail", ""),
}
)
for node_type, items in round_report.get("required_bindings", {}).items():
for item in items:
if item.get("bound"):
continue
failed_nodes.append(
{
"domain": domain["domain"],
"round": round_report["name"],
"node_type": node_type,
"local_data_id": item.get("local_data_id"),
"local_node_id": item.get("local_node_id"),
"remote_node_id": item.get("remote_node_id"),
"remote_data_id": item.get("remote_data_id"),
"issues": round_report.get("issues", []),
"node_status": round_report.get("persistence_summary", {}).get("node_status", []),
"status": None,
"binding_status": None,
"error": None,
"sync_log_tail": "",
}
)
return failed_nodes
def build_detailed_record(
run_id: str,
config: dict[str, Any],
init_result: dict[str, Any],
domain_reports: list[dict[str, Any]],
) -> dict[str, Any]:
backend_log_tail = ""
if init_result.get("backend_log_file"):
backend_log_tail = tail_text_file(init_result["backend_log_file"], 8000)
rounds: list[dict[str, Any]] = []
for domain in domain_reports:
for round_report in domain.get("rounds", []):
rounds.append(
{
"domain": domain["domain"],
"round": round_report["name"],
"purpose": round_report.get("purpose"),
"passed": round_report.get("passed"),
"returncode": round_report.get("returncode"),
"dataset_dir": round_report.get("dataset_dir"),
"latest_log_file": round_report.get("latest_log_file"),
"archived_log_file": round_report.get("archived_log_file"),
"issues": round_report.get("issues", []),
"dataset_diff": round_report.get("dataset_diff", {}),
"dataset_diffs": round_report.get("dataset_diffs", {}),
"required_bindings": round_report.get("required_bindings", {}),
"verified_bound_fields": round_report.get("verified_bound_fields", []),
"problem_nodes": round_report.get("problem_nodes", []),
"persistence_summary": round_report.get("persistence_summary", {}),
"stdout_tail": round_report.get("stdout_tail", ""),
"stderr_tail": round_report.get("stderr_tail", ""),
"pipeline_log_tail": tail_text_file(round_report.get("latest_log_file") or "", 8000),
"coverage": round_report.get("coverage", {}),
}
)
return {
"run_id": run_id,
"config_snapshot": sanitize_config_for_record(config),
"init": init_result,
"backend_log_tail": backend_log_tail,
"failed_nodes": collect_failed_node_details(domain_reports),
"domains": domain_reports,
"round_details": rounds,
}
def build_detailed_markdown(record: dict[str, Any]) -> str:
init_result = record.get("init", {})
lines = [
f"# 自动测试详细记录 {record['run_id']}",
"",
"## 环境初始化",
"",
]
for key in [
"backend_root_dir",
"backend_ready_url",
"backend_log_file",
"archived_backend_log_file",
"persistence_db",
"temp_data_dir",
"pipeline_log_dir",
"backend_pid",
]:
if init_result.get(key) is not None:
lines.append(f"- {key}: {init_result.get(key)}")
if init_result.get("backend_start_command"):
lines.append(f"- backend_start_command: {' '.join(init_result['backend_start_command'])}")
if init_result.get("backend_restore_command"):
lines.append(f"- backend_restore_command: {' '.join(init_result['backend_restore_command'])}")
if init_result.get("backend_env"):
lines.append("- backend_env:")
for key, value in init_result["backend_env"].items():
lines.append(f" - {key}: {value}")
if init_result.get("restore_stdout_tail"):
lines.extend(["", "### restore stdout tail", "", "```text", init_result["restore_stdout_tail"], "```"])
if init_result.get("restore_stderr_tail"):
lines.extend(["", "### restore stderr tail", "", "```text", init_result["restore_stderr_tail"], "```"])
failed_nodes = record.get("failed_nodes", [])
lines.extend(["", "## 失败节点", ""])
if failed_nodes:
lines.extend([
"| Domain | Round | Node Type | local_data_id | local_node_id | 问题 |",
"| --- | --- | --- | --- | --- | --- |",
])
for item in failed_nodes:
issue_text = "".join(item.get("issues", []))
lines.append(
f"| {item['domain']} | {item['round']} | {item.get('collection_id', '')}.{item['node_type']} | {item['local_data_id']} | {item['local_node_id']} | {issue_text or item.get('error', '')} |"
)
lines.append("")
else:
lines.append("- 无失败节点")
lines.extend(["", "## Backend 日志尾部", "", "```text", record.get("backend_log_tail", ""), "```", ""])
for round_detail in record.get("round_details", []):
lines.extend([
f"## {round_detail['domain']} / {round_detail['round']}",
"",
f"- purpose: {round_detail.get('purpose')}",
f"- passed: {round_detail.get('passed')}",
f"- returncode: {round_detail.get('returncode')}",
f"- dataset_dir: {round_detail.get('dataset_dir')}",
f"- latest_log_file: {round_detail.get('latest_log_file')}",
])
coverage = round_detail.get("coverage") or {}
if coverage.get("keys"):
lines.append(f"- coverage.keys: {', '.join(coverage['keys'])}")
if coverage.get("api"):
lines.append(f"- coverage.api: {', '.join(coverage['api'])}")
if coverage.get("notes"):
lines.append(f"- coverage.notes: {coverage['notes']}")
if round_detail.get("archived_log_file"):
lines.append(f"- archived_log_file: {round_detail.get('archived_log_file')}")
if round_detail.get("issues"):
lines.append("- issues:")
for issue in round_detail["issues"]:
lines.append(f" - {issue}")
if round_detail.get("dataset_diffs"):
lines.append("- dataset_diffs:")
for node_type, record_map in round_detail["dataset_diffs"].items():
for record_id, fields in record_map.items():
lines.append(f" - {node_type}.{record_id}: {', '.join(fields)}")
if round_detail.get("verified_bound_fields"):
lines.append("- verified_bound_fields:")
for item in round_detail["verified_bound_fields"]:
lines.append(
f" - {item.get('collection_id')}.{item.get('node_type')} local={item.get('local_data_id')} target={item.get('target_data_id')} matched={item.get('matched')}"
)
for field_path, actual_value in (item.get("actual_fields") or {}).items():
expected_value = (item.get("expected_fields") or {}).get(field_path)
lines.append(f" - {field_path}: actual={actual_value!r} expected={expected_value!r}")
if round_detail.get("problem_nodes"):
lines.append("- problem_nodes:")
for item in round_detail["problem_nodes"]:
lines.append(
f" - {item.get('collection_id')}.{item.get('node_type')} data_id={item.get('data_id')} action={item.get('action')} status={item.get('status')} binding_status={item.get('binding_status')} error={item.get('error') or '<empty>'}"
)
if item.get("sync_log_tail"):
lines.append(" sync_log_tail:")
for log_line in str(item["sync_log_tail"]).splitlines():
lines.append(f" {log_line}")
if round_detail.get("persistence_summary", {}).get("node_status"):
lines.append("- node_status:")
for item in round_detail["persistence_summary"]["node_status"]:
lines.append(
f" - collection={item.get('collection_id')} action={item.get('action')} status={item.get('status')} binding_status={item.get('binding_status')} count={item.get('count')}"
)
lines.extend([
"",
"### pipeline log tail",
"",
"```text",
round_detail.get("pipeline_log_tail", ""),
"```",
"",
"### stdout tail",
"",
"```text",
round_detail.get("stdout_tail", ""),
"```",
"",
"### stderr tail",
"",
"```text",
round_detail.get("stderr_tail", ""),
"```",
"",
])
return "\n".join(lines)
def main() -> int:
parser = argparse.ArgumentParser(description="运行脚本式自动测试")
parser.add_argument(
"--config",
default="tests/fixtures/auto_sync/auto_test_config.yaml",
help="自动测试配置文件路径",
)
parser.add_argument(
"--domain",
action="append",
default=None,
help="只跑指定 domain,可重复传入",
)
parser.add_argument(
"--skip-init",
action="store_true",
help="跳过初始化阶段",
)
parser.add_argument(
"--clear-only",
action="store_true",
help="只清空环境,不执行 domain 用例",
)
args = parser.parse_args()
config = load_yaml_config(args.config)
report_root = Path(config["paths"]["report_dir"]) / now_slug()
recreate_dir(report_root)
init_result = {"skipped": True}
if args.clear_only:
init_result = clear_environment(args.config)
archive_run_logs(report_root, init_result, [])
(report_root / "report.md").write_text(build_markdown_report(report_root.name, init_result, []), encoding="utf-8")
(report_root / "summary.json").write_text(
json.dumps({"init": init_result, "summary": build_test_summary([]), "domains": []}, ensure_ascii=False, indent=2),
encoding="utf-8",
)
detail_record = build_detailed_record(report_root.name, config, init_result, [])
(report_root / "detail.json").write_text(json.dumps(detail_record, ensure_ascii=False, indent=2), encoding="utf-8")
(report_root / "detail.md").write_text(build_detailed_markdown(detail_record), encoding="utf-8")
print(f"报告已生成: {report_root / 'report.md'}")
return 0
if not args.skip_init:
init_result = initialize_environment(args.config)
selected_domains = args.domain or config["domains"].get("enabled", [])
domain_reports = []
for domain_name in selected_domains:
domain_reports.append(run_domain(config, domain_name, report_root))
overall_passed = all(item.get("passed", False) for item in domain_reports)
archive_run_logs(report_root, init_result, domain_reports)
markdown = build_markdown_report(report_root.name, init_result, domain_reports)
(report_root / "report.md").write_text(markdown, encoding="utf-8")
summary = build_test_summary(domain_reports)
(report_root / "summary.json").write_text(
json.dumps(
{"init": init_result, "summary": summary, "domains": domain_reports, "passed": overall_passed},
ensure_ascii=False,
indent=2,
),
encoding="utf-8",
)
detail_record = build_detailed_record(report_root.name, config, init_result, domain_reports)
(report_root / "detail.json").write_text(json.dumps(detail_record, ensure_ascii=False, indent=2), encoding="utf-8")
(report_root / "detail.md").write_text(build_detailed_markdown(detail_record), encoding="utf-8")
print(f"报告已生成: {report_root / 'report.md'}")
return 0
if __name__ == "__main__":
raise SystemExit(main())