from __future__ import annotations import argparse import importlib.util import json import shutil from pathlib import Path from typing import Any from auto_test_init import clear_environment, initialize_environment from auto_test_lib import ( load_yaml_config, now_slug, recreate_dir, tail_text_file, ) def load_domain_module(config: dict[str, Any], domain_name: str): script_dir = Path(config["domains"]["script_dir"]) script_path = script_dir / f"{domain_name}_case.py" module_name = f"auto_test_domain_{domain_name}" spec = importlib.util.spec_from_file_location(module_name, script_path) if spec is None or spec.loader is None: raise ImportError(f"无法加载 domain 脚本: {script_path}") module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module def run_domain(config: dict[str, Any], domain_name: str, report_root: Path) -> dict[str, Any]: module = load_domain_module(config, domain_name) if not hasattr(module, "run_case"): raise AttributeError(f"domain 脚本缺少 run_case(): {domain_name}") domain_report = module.run_case(config, report_root) report_path = report_root / f"{domain_name}.json" report_path.write_text(json.dumps(domain_report, ensure_ascii=False, indent=2), encoding="utf-8") return domain_report def archive_run_logs(report_root: Path, init_result: dict[str, Any], domain_reports: list[dict[str, Any]]) -> None: logs_dir = report_root / "logs" logs_dir.mkdir(parents=True, exist_ok=True) manifest: dict[str, Any] = { "backend_log": None, "pipeline_logs": [], } backend_log_file = init_result.get("backend_log_file") if backend_log_file: src = Path(backend_log_file) if src.exists(): target = logs_dir / "backend.log" shutil.copy2(src, target) init_result["archived_backend_log_file"] = str(target) manifest["backend_log"] = str(target) for domain in domain_reports: for round_report in domain.get("rounds", []): latest_log_file = round_report.get("latest_log_file") if not latest_log_file: continue src = Path(latest_log_file) if not src.exists(): continue target = logs_dir / f"{domain['domain']}__{round_report['name']}.log" shutil.copy2(src, target) round_report["archived_log_file"] = str(target) manifest["pipeline_logs"].append( { "domain": domain["domain"], "round": round_report["name"], "source": str(src), "archived": str(target), } ) (logs_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8") def build_test_summary(domain_reports: list[dict[str, Any]]) -> dict[str, Any]: cases: list[dict[str, Any]] = [] passed_domains = 0 failed_domains = 0 for domain in domain_reports: domain_passed = bool(domain.get("passed")) if domain_passed: passed_domains += 1 else: failed_domains += 1 for round_report in domain.get("rounds", []): cases.append( { "nodeid": f"{domain['domain']}::{round_report['name']}", "domain": domain["domain"], "case": round_report["name"], "purpose": round_report.get("purpose", ""), "passed": bool(round_report.get("passed")), "result": "PASSED" if round_report.get("passed") else "FAILED", "returncode": round_report.get("returncode"), "issue_count": len(round_report.get("issues", [])), "issues": round_report.get("issues", []), } ) passed_cases = sum(1 for item in cases if item["passed"]) failed_cases = len(cases) - passed_cases return { "total_domains": len(domain_reports), "passed_domains": passed_domains, "failed_domains": failed_domains, "total_cases": len(cases), "passed_cases": passed_cases, "failed_cases": failed_cases, "cases": cases, } def build_markdown_report(run_id: str, init_result: dict[str, Any], domain_reports: list[dict[str, Any]]) -> str: summary = build_test_summary(domain_reports) if init_result.get("skipped"): init_lines = ["- 本次跳过初始化，沿用现有测试环境"] else: init_lines = [ f"- 后端进程 PID: {init_result.get('backend_pid')}", f"- 后端日志: {init_result.get('backend_log_file')}", f"- 状态库: {init_result.get('persistence_db')}", f"- 临时数据目录: {init_result.get('temp_data_dir')}", ] lines = [ f"# 自动测试报告 {run_id}", "", "## 初始化结果", "", *init_lines, "", ] if domain_reports: lines.extend([ "## Pytest 风格汇总", "", f"collected {summary['total_cases']} cases across {summary['total_domains']} domains", "", ]) for case in summary["cases"]: lines.append(f"- {case['nodeid']} {case['result']}") lines.extend([ "", f"== {summary['passed_cases']} passed, {summary['failed_cases']} failed, {summary['passed_domains']} passed domains, {summary['failed_domains']} failed domains ==", "", ]) lines.extend([ "## Domain 汇总", "", "| Domain | 说明 | 结果 | 轮次数 | 问题数 |", "| --- | --- | --- | --- | --- |", ]) for domain in domain_reports: lines.append( f"| {domain['domain']} | {domain.get('description', '')} | {'通过' if domain.get('passed') else '失败'} | {len(domain.get('rounds', []))} | {len(domain.get('issues', []))} |" ) lines.append("") for domain in domain_reports: lines.append(f"## Domain: {domain['domain']}") if domain.get("description"): lines.append("") lines.append(domain["description"]) lines.append("") lines.append(f"- 结果: {'通过' if domain.get('passed') else '失败'}") if domain.get("issues"): lines.append("- 问题:") for issue in domain["issues"]: lines.append(f" - {issue}") lines.append("") for round_report in domain.get("rounds", []): lines.extend([ f"### {round_report['name']}", f"- 目的: {round_report['purpose']}", f"- 返回码: {round_report['returncode']}", f"- 结果: {'通过' if round_report.get('passed') else '失败'}", f"- 数据目录: {round_report['dataset_dir']}", f"- 最新日志: {round_report['latest_log_file']}", ]) coverage = round_report.get("coverage") or {} if coverage: if coverage.get("keys"): lines.append(f"- 覆盖 key: {', '.join(coverage['keys'])}") if coverage.get("api"): lines.append(f"- 覆盖接口: {', '.join(coverage['api'])}") if coverage.get("notes"): lines.append(f"- 覆盖说明: {coverage['notes']}") if round_report.get("archived_log_file"): lines.append(f"- 归档日志: {round_report['archived_log_file']}") if round_report.get("issues"): lines.append("- 本轮问题:") for issue in round_report["issues"]: lines.append(f" - {issue}") dataset_diffs = round_report.get("dataset_diffs") or {} if dataset_diffs: lines.append("- 本轮数据变更:") for node_type, record_map in dataset_diffs.items(): for record_id, fields in record_map.items(): lines.append(f" - {node_type}.{record_id}: {', '.join(fields)}") bindings = round_report.get("required_bindings", {}) if bindings: lines.append("- 绑定检查:") for node_type, items in bindings.items(): for item in items: lines.append( f" - {node_type} {item['local_data_id']}: bound={item['bound']} local_node_id={item['local_node_id']} remote_data_id={item['remote_data_id']}" ) verified_bound_fields = round_report.get("verified_bound_fields") or [] if verified_bound_fields: lines.append("- 字段校验:") for item in verified_bound_fields: status = "matched" if item.get("matched") else "mismatched" lines.append( f" - {item['collection_id']}.{item['node_type']} local={item['local_data_id']} target={item['target_data_id']} status={status}" ) for field_path, actual_value in (item.get("actual_fields") or {}).items(): expected_value = (item.get("expected_fields") or {}).get(field_path) lines.append( f" - {field_path}: actual={actual_value!r} expected={expected_value!r}" ) problem_nodes = round_report.get("problem_nodes", []) if problem_nodes: lines.append("- 问题节点:") for item in problem_nodes: lines.append( f" - {item['collection_id']}.{item['node_type']} data_id={item.get('data_id')} action={item.get('action')} status={item.get('status')} binding_status={item.get('binding_status')} error={item.get('error') or ''}" ) lines.append("") return "\n".join(lines) def sanitize_config_for_record(config: dict[str, Any]) -> dict[str, Any]: payload = json.loads(json.dumps(config, ensure_ascii=False)) remote = payload.get("remote_datasource", {}) if remote.get("api_secret"): remote["api_secret"] = "***" backend = payload.get("backend", {}) backend_env = backend.get("env", {}) for key in ["COUCHDB_PASSWORD"]: if backend_env.get(key): backend_env[key] = "***" couchdb = payload.get("couchdb", {}) if couchdb.get("password"): couchdb["password"] = "***" return payload def collect_failed_node_details(domain_reports: list[dict[str, Any]]) -> list[dict[str, Any]]: failed_nodes: list[dict[str, Any]] = [] for domain in domain_reports: for round_report in domain.get("rounds", []): for item in round_report.get("problem_nodes", []): failed_nodes.append( { "domain": domain["domain"], "round": round_report["name"], "node_type": item.get("node_type"), "collection_id": item.get("collection_id"), "local_data_id": item.get("data_id"), "local_node_id": item.get("node_id"), "remote_node_id": None, "remote_data_id": item.get("bind_data_id"), "issues": round_report.get("issues", []), "node_status": round_report.get("persistence_summary", {}).get("node_status", []), "status": item.get("status"), "binding_status": item.get("binding_status"), "error": item.get("error"), "sync_log_tail": item.get("sync_log_tail", ""), } ) for node_type, items in round_report.get("required_bindings", {}).items(): for item in items: if item.get("bound"): continue failed_nodes.append( { "domain": domain["domain"], "round": round_report["name"], "node_type": node_type, "local_data_id": item.get("local_data_id"), "local_node_id": item.get("local_node_id"), "remote_node_id": item.get("remote_node_id"), "remote_data_id": item.get("remote_data_id"), "issues": round_report.get("issues", []), "node_status": round_report.get("persistence_summary", {}).get("node_status", []), "status": None, "binding_status": None, "error": None, "sync_log_tail": "", } ) return failed_nodes def build_detailed_record( run_id: str, config: dict[str, Any], init_result: dict[str, Any], domain_reports: list[dict[str, Any]], ) -> dict[str, Any]: backend_log_tail = "" if init_result.get("backend_log_file"): backend_log_tail = tail_text_file(init_result["backend_log_file"], 8000) rounds: list[dict[str, Any]] = [] for domain in domain_reports: for round_report in domain.get("rounds", []): rounds.append( { "domain": domain["domain"], "round": round_report["name"], "purpose": round_report.get("purpose"), "passed": round_report.get("passed"), "returncode": round_report.get("returncode"), "dataset_dir": round_report.get("dataset_dir"), "latest_log_file": round_report.get("latest_log_file"), "archived_log_file": round_report.get("archived_log_file"), "issues": round_report.get("issues", []), "dataset_diff": round_report.get("dataset_diff", {}), "dataset_diffs": round_report.get("dataset_diffs", {}), "required_bindings": round_report.get("required_bindings", {}), "verified_bound_fields": round_report.get("verified_bound_fields", []), "problem_nodes": round_report.get("problem_nodes", []), "persistence_summary": round_report.get("persistence_summary", {}), "stdout_tail": round_report.get("stdout_tail", ""), "stderr_tail": round_report.get("stderr_tail", ""), "pipeline_log_tail": tail_text_file(round_report.get("latest_log_file") or "", 8000), "coverage": round_report.get("coverage", {}), } ) return { "run_id": run_id, "config_snapshot": sanitize_config_for_record(config), "init": init_result, "backend_log_tail": backend_log_tail, "failed_nodes": collect_failed_node_details(domain_reports), "domains": domain_reports, "round_details": rounds, } def build_detailed_markdown(record: dict[str, Any]) -> str: init_result = record.get("init", {}) lines = [ f"# 自动测试详细记录 {record['run_id']}", "", "## 环境初始化", "", ] for key in [ "backend_root_dir", "backend_ready_url", "backend_log_file", "archived_backend_log_file", "persistence_db", "temp_data_dir", "pipeline_log_dir", "backend_pid", ]: if init_result.get(key) is not None: lines.append(f"- {key}: {init_result.get(key)}") if init_result.get("backend_start_command"): lines.append(f"- backend_start_command: {' '.join(init_result['backend_start_command'])}") if init_result.get("backend_restore_command"): lines.append(f"- backend_restore_command: {' '.join(init_result['backend_restore_command'])}") if init_result.get("backend_env"): lines.append("- backend_env:") for key, value in init_result["backend_env"].items(): lines.append(f" - {key}: {value}") if init_result.get("restore_stdout_tail"): lines.extend(["", "### restore stdout tail", "", "```text", init_result["restore_stdout_tail"], "```"]) if init_result.get("restore_stderr_tail"): lines.extend(["", "### restore stderr tail", "", "```text", init_result["restore_stderr_tail"], "```"]) failed_nodes = record.get("failed_nodes", []) lines.extend(["", "## 失败节点", ""]) if failed_nodes: lines.extend([ "| Domain | Round | Node Type | local_data_id | local_node_id | 问题 |", "| --- | --- | --- | --- | --- | --- |", ]) for item in failed_nodes: issue_text = "；".join(item.get("issues", [])) lines.append( f"| {item['domain']} | {item['round']} | {item.get('collection_id', '')}.{item['node_type']} | {item['local_data_id']} | {item['local_node_id']} | {issue_text or item.get('error', '')} |" ) lines.append("") else: lines.append("- 无失败节点") lines.extend(["", "## Backend 日志尾部", "", "```text", record.get("backend_log_tail", ""), "```", ""]) for round_detail in record.get("round_details", []): lines.extend([ f"## {round_detail['domain']} / {round_detail['round']}", "", f"- purpose: {round_detail.get('purpose')}", f"- passed: {round_detail.get('passed')}", f"- returncode: {round_detail.get('returncode')}", f"- dataset_dir: {round_detail.get('dataset_dir')}", f"- latest_log_file: {round_detail.get('latest_log_file')}", ]) coverage = round_detail.get("coverage") or {} if coverage.get("keys"): lines.append(f"- coverage.keys: {', '.join(coverage['keys'])}") if coverage.get("api"): lines.append(f"- coverage.api: {', '.join(coverage['api'])}") if coverage.get("notes"): lines.append(f"- coverage.notes: {coverage['notes']}") if round_detail.get("archived_log_file"): lines.append(f"- archived_log_file: {round_detail.get('archived_log_file')}") if round_detail.get("issues"): lines.append("- issues:") for issue in round_detail["issues"]: lines.append(f" - {issue}") if round_detail.get("dataset_diffs"): lines.append("- dataset_diffs:") for node_type, record_map in round_detail["dataset_diffs"].items(): for record_id, fields in record_map.items(): lines.append(f" - {node_type}.{record_id}: {', '.join(fields)}") if round_detail.get("verified_bound_fields"): lines.append("- verified_bound_fields:") for item in round_detail["verified_bound_fields"]: lines.append( f" - {item.get('collection_id')}.{item.get('node_type')} local={item.get('local_data_id')} target={item.get('target_data_id')} matched={item.get('matched')}" ) for field_path, actual_value in (item.get("actual_fields") or {}).items(): expected_value = (item.get("expected_fields") or {}).get(field_path) lines.append(f" - {field_path}: actual={actual_value!r} expected={expected_value!r}") if round_detail.get("problem_nodes"): lines.append("- problem_nodes:") for item in round_detail["problem_nodes"]: lines.append( f" - {item.get('collection_id')}.{item.get('node_type')} data_id={item.get('data_id')} action={item.get('action')} status={item.get('status')} binding_status={item.get('binding_status')} error={item.get('error') or ''}" ) if item.get("sync_log_tail"): lines.append(" sync_log_tail:") for log_line in str(item["sync_log_tail"]).splitlines(): lines.append(f" {log_line}") if round_detail.get("persistence_summary", {}).get("node_status"): lines.append("- node_status:") for item in round_detail["persistence_summary"]["node_status"]: lines.append( f" - collection={item.get('collection_id')} action={item.get('action')} status={item.get('status')} binding_status={item.get('binding_status')} count={item.get('count')}" ) lines.extend([ "", "### pipeline log tail", "", "```text", round_detail.get("pipeline_log_tail", ""), "```", "", "### stdout tail", "", "```text", round_detail.get("stdout_tail", ""), "```", "", "### stderr tail", "", "```text", round_detail.get("stderr_tail", ""), "```", "", ]) return "\n".join(lines) def main() -> int: parser = argparse.ArgumentParser(description="运行脚本式自动测试") parser.add_argument( "--config", default="tests/fixtures/auto_sync/auto_test_config.yaml", help="自动测试配置文件路径", ) parser.add_argument( "--domain", action="append", default=None, help="只跑指定 domain，可重复传入", ) parser.add_argument( "--skip-init", action="store_true", help="跳过初始化阶段", ) parser.add_argument( "--clear-only", action="store_true", help="只清空环境，不执行 domain 用例", ) args = parser.parse_args() config = load_yaml_config(args.config) report_root = Path(config["paths"]["report_dir"]) / now_slug() recreate_dir(report_root) init_result = {"skipped": True} if args.clear_only: init_result = clear_environment(args.config) archive_run_logs(report_root, init_result, []) (report_root / "report.md").write_text(build_markdown_report(report_root.name, init_result, []), encoding="utf-8") (report_root / "summary.json").write_text( json.dumps({"init": init_result, "summary": build_test_summary([]), "domains": []}, ensure_ascii=False, indent=2), encoding="utf-8", ) detail_record = build_detailed_record(report_root.name, config, init_result, []) (report_root / "detail.json").write_text(json.dumps(detail_record, ensure_ascii=False, indent=2), encoding="utf-8") (report_root / "detail.md").write_text(build_detailed_markdown(detail_record), encoding="utf-8") print(f"报告已生成: {report_root / 'report.md'}") return 0 if not args.skip_init: init_result = initialize_environment(args.config) selected_domains = args.domain or config["domains"].get("enabled", []) domain_reports = [] for domain_name in selected_domains: domain_reports.append(run_domain(config, domain_name, report_root)) overall_passed = all(item.get("passed", False) for item in domain_reports) archive_run_logs(report_root, init_result, domain_reports) markdown = build_markdown_report(report_root.name, init_result, domain_reports) (report_root / "report.md").write_text(markdown, encoding="utf-8") summary = build_test_summary(domain_reports) (report_root / "summary.json").write_text( json.dumps( {"init": init_result, "summary": summary, "domains": domain_reports, "passed": overall_passed}, ensure_ascii=False, indent=2, ), encoding="utf-8", ) detail_record = build_detailed_record(report_root.name, config, init_result, domain_reports) (report_root / "detail.json").write_text(json.dumps(detail_record, ensure_ascii=False, indent=2), encoding="utf-8") (report_root / "detail.md").write_text(build_detailed_markdown(detail_record), encoding="utf-8") print(f"报告已生成: {report_root / 'report.md'}") return 0 if __name__ == "__main__": raise SystemExit(main())