from __future__ import annotations import argparse import fnmatch import json import re import subprocess import sys from dataclasses import dataclass from pathlib import Path from typing import Any import yaml from auto_test_lib import PROJECT_ROOT, load_yaml_config, now_slug DEFAULT_DOC_PATH = PROJECT_ROOT / "docs/auto_test_spec/large_scale_test_control.md" RESULT_START_MARKER = "" RESULT_END_MARKER = "" @dataclass class ControlDocument: path: Path frontmatter: dict[str, Any] body: str def read_control_document(doc_path: str | Path) -> ControlDocument: path = Path(doc_path) if not path.is_absolute(): path = PROJECT_ROOT / path text = path.read_text(encoding="utf-8") match = re.match(r"^---\n(.*?)\n---\n?", text, re.DOTALL) if not match: raise ValueError(f"控制文档缺少 YAML frontmatter: {path}") payload = yaml.safe_load(match.group(1)) or {} body = text[match.end() :] return ControlDocument(path=path, frontmatter=payload, body=body) def write_control_document(doc: ControlDocument, generated_section: str) -> None: body = doc.body if RESULT_START_MARKER in body and RESULT_END_MARKER in body: body = re.sub( rf"{re.escape(RESULT_START_MARKER)}.*?{re.escape(RESULT_END_MARKER)}", f"{RESULT_START_MARKER}\n\n{generated_section}\n\n{RESULT_END_MARKER}", body, flags=re.DOTALL, ) else: body = ( body.rstrip() + "\n\n" + RESULT_START_MARKER + "\n\n" + generated_section + "\n\n" + RESULT_END_MARKER + "\n" ) frontmatter_text = yaml.safe_dump(doc.frontmatter, allow_unicode=True, sort_keys=False).strip() final_text = f"---\n{frontmatter_text}\n---\n{body.lstrip()}" doc.path.write_text(final_text, encoding="utf-8") def repo_relative(path: str | Path | None) -> str: if not path: return "" resolved = Path(path) if not resolved.is_absolute(): return str(resolved) try: return str(resolved.relative_to(PROJECT_ROOT)) except ValueError: return str(resolved) def list_report_dirs(report_base: Path) -> set[Path]: if not report_base.exists(): return set() return {item for item in report_base.iterdir() if item.is_dir()} def extract_report_dir(stdout: str, stderr: str) -> Path | None: combined = "\n".join(part for part in [stdout, stderr] if part) match = re.search(r"(?:聚合)?报告已生成:\s*(.+?/report\.md)", combined) if not match: return None return Path(match.group(1).strip()).parent def build_command(script_dir: Path, config_path: str, round_config: dict[str, Any], control: dict[str, Any]) -> list[str]: runner = str(round_config.get("runner") or control.get("default_runner") or "aggregate") init_mode = str(round_config.get("init_mode") or control.get("default_init_mode") or "fresh") if runner not in {"aggregate", "domains"}: raise ValueError(f"不支持的 runner: {runner}") if init_mode not in {"fresh", "reuse", "clear-only"}: raise ValueError(f"不支持的 init_mode: {init_mode}") if runner == "aggregate": command = [sys.executable, str(script_dir / "run_auto_test_all_domains.py"), "--config", config_path] else: domains = round_config.get("domains") or [] if not domains: raise ValueError(f"domains runner 必须显式提供 domains: {round_config.get('id')}") command = [sys.executable, str(script_dir / "run_auto_test.py"), "--config", config_path] for domain in domains: command.extend(["--domain", str(domain)]) if init_mode == "reuse": command.append("--skip-init") elif init_mode == "clear-only": command.append("--clear-only") return command def case_map(summary_payload: dict[str, Any]) -> dict[str, dict[str, Any]]: summary = summary_payload.get("summary", {}) return { case["nodeid"]: case for case in summary.get("cases", []) if isinstance(case, dict) and case.get("nodeid") } def match_classification(case_id: str, round_config: dict[str, Any], control: dict[str, Any]) -> str: merged: dict[str, str] = {} global_rules = control.get("classification", {}) or {} round_rules = round_config.get("classification", {}) or {} merged.update({str(key): str(value) for key, value in global_rules.items()}) merged.update({str(key): str(value) for key, value in round_rules.items()}) for pattern, category in merged.items(): if fnmatch.fnmatch(case_id, pattern): return category return "unclassified" def find_compare_target(previous_rounds: list[dict[str, Any]], current_cases: dict[str, dict[str, Any]]) -> tuple[dict[str, Any] | None, set[str]]: current_ids = set(current_cases) for item in reversed(previous_rounds): previous_cases = item.get("case_results", {}) overlap = current_ids & set(previous_cases) if overlap: return item, overlap return None, set() def compare_rounds(previous_round: dict[str, Any] | None, overlap: set[str], current_cases: dict[str, dict[str, Any]]) -> dict[str, Any]: if previous_round is None or not overlap: return { "compare_round_id": None, "overlap_count": 0, "fixed": [], "regressed": [], "persistent_failed": [], "stable_passed": [], } previous_cases = previous_round["case_results"] fixed: list[str] = [] regressed: list[str] = [] persistent_failed: list[str] = [] stable_passed: list[str] = [] for case_id in sorted(overlap): old_passed = bool(previous_cases[case_id].get("passed")) new_passed = bool(current_cases[case_id].get("passed")) if not old_passed and new_passed: fixed.append(case_id) elif old_passed and not new_passed: regressed.append(case_id) elif not old_passed and not new_passed: persistent_failed.append(case_id) else: stable_passed.append(case_id) return { "compare_round_id": previous_round.get("id"), "overlap_count": len(overlap), "fixed": fixed, "regressed": regressed, "persistent_failed": persistent_failed, "stable_passed": stable_passed, } def execute_round( *, script_dir: Path, config_path: str, report_base: Path, round_config: dict[str, Any], control: dict[str, Any], previous_rounds: list[dict[str, Any]], ) -> dict[str, Any]: command = build_command(script_dir, config_path, round_config, control) before_dirs = list_report_dirs(report_base) result = subprocess.run(command, cwd=str(PROJECT_ROOT), text=True, capture_output=True, check=False) after_dirs = list_report_dirs(report_base) report_dir = extract_report_dir(result.stdout or "", result.stderr or "") if report_dir is None: new_dirs = sorted(after_dirs - before_dirs) if new_dirs: report_dir = new_dirs[-1] summary_payload: dict[str, Any] = {} if report_dir is not None: summary_path = report_dir / "summary.json" if summary_path.exists(): summary_payload = json.loads(summary_path.read_text(encoding="utf-8")) current_cases = case_map(summary_payload) compare_round, overlap = find_compare_target(previous_rounds, current_cases) comparison = compare_rounds(compare_round, overlap, current_cases) failed_case_details = [] for case_id, item in sorted(current_cases.items()): if item.get("passed"): continue failed_case_details.append( { "nodeid": case_id, "classification": match_classification(case_id, round_config, control), "issues": item.get("issues", []), "issue_count": item.get("issue_count", len(item.get("issues", []))), } ) summary = summary_payload.get("summary", {}) return { "id": str(round_config.get("id") or ""), "title": str(round_config.get("title") or ""), "objective": str(round_config.get("objective") or ""), "focus": [str(item) for item in (round_config.get("focus") or [])], "runner": str(round_config.get("runner") or control.get("default_runner") or "aggregate"), "init_mode": str(round_config.get("init_mode") or control.get("default_init_mode") or "fresh"), "domains": [str(item) for item in (round_config.get("domains") or [])], "next_goal": str(round_config.get("next_goal") or ""), "returncode": result.returncode, "command": command, "report_dir": str(report_dir) if report_dir else None, "report_md": str(report_dir / "report.md") if report_dir else None, "summary_json": str(report_dir / "summary.json") if report_dir else None, "summary": { "total_domains": int(summary.get("total_domains", 0)), "passed_domains": int(summary.get("passed_domains", 0)), "failed_domains": int(summary.get("failed_domains", 0)), "total_cases": int(summary.get("total_cases", 0)), "passed_cases": int(summary.get("passed_cases", 0)), "failed_cases": int(summary.get("failed_cases", 0)), }, "has_summary": bool(summary_payload), "case_results": current_cases, "failed_cases": failed_case_details, "comparison": comparison, "stdout_tail": (result.stdout or "")[-4000:], "stderr_tail": (result.stderr or "")[-4000:], } def build_results_markdown(run_id: str, doc_path: Path, results: list[dict[str, Any]]) -> str: lines = [ f"## 最近一次执行 {run_id}", "", f"- 控制文档: {repo_relative(doc_path)}", f"- 执行轮次数: {len(results)}", "- 说明: 默认优先使用 fresh 初始化;若使用 aggregate runner,则每个 domain 仍会独立初始化。", "", "### 轮次总览", "", "| Round | 标题 | Runner | Init | Cases | Failed | 对比轮次 | 报告 |", "| --- | --- | --- | --- | --- | --- | --- | --- |", ] for item in results: summary = item["summary"] compare_round_id = item["comparison"].get("compare_round_id") or "-" report_link = repo_relative(item.get("report_md")) or "-" lines.append( f"| {item['id']} | {item['title']} | {item['runner']} | {item['init_mode']} | {summary['total_cases']} | {summary['failed_cases']} | {compare_round_id} | {report_link} |" ) for item in results: summary = item["summary"] comparison = item["comparison"] lines.extend( [ "", f"### {item['id']} - {item['title']}", "", f"- 目标: {item['objective']}", f"- 运行方式: {item['runner']}", f"- 初始化策略: {item['init_mode']}", f"- 执行命令: {' '.join(item['command'])}", f"- 单轮结果: {summary['passed_cases']} passed / {summary['failed_cases']} failed / {summary['total_cases']} total cases", f"- Domain 结果: {summary['passed_domains']} passed / {summary['failed_domains']} failed / {summary['total_domains']} total domains", f"- 原始报告: {repo_relative(item.get('report_md')) or '-'}", f"- 原始汇总: {repo_relative(item.get('summary_json')) or '-'}", ] ) if item["focus"]: lines.append("- 本轮关注点:") for focus in item["focus"]: lines.append(f" - {focus}") lines.extend( [ f"- 对比轮次: {comparison.get('compare_round_id') or '无可比较轮次'}", f"- 可比较 case 数: {comparison.get('overlap_count', 0)}", ] ) if comparison.get("fixed"): lines.append("- 本轮改进:") for case_id in comparison["fixed"]: lines.append(f" - fixed: {case_id}") if comparison.get("regressed"): lines.append("- 本轮回退:") for case_id in comparison["regressed"]: lines.append(f" - regressed: {case_id}") if comparison.get("persistent_failed"): lines.append("- 本轮遗留:") for case_id in comparison["persistent_failed"]: lines.append(f" - persistent_failed: {case_id}") if not comparison.get("fixed") and not comparison.get("regressed") and not comparison.get("persistent_failed"): lines.append("- 对比结果: 本轮没有可比改进项,或与此前轮次无重叠 case。") if item["failed_cases"]: lines.append("- 失败归因:") for failed in item["failed_cases"]: issue_text = ";".join(str(issue) for issue in failed.get("issues", [])) or "" lines.append( f" - {failed['nodeid']} | {failed['classification']} | issues={failed['issue_count']} | {issue_text}" ) else: lines.append("- 失败归因: 本轮无失败 case。") if item.get("next_goal"): lines.append(f"- 下一轮目标: {item['next_goal']}") if item.get("returncode") not in (0, None): lines.append(f"- 脚本返回码: {item['returncode']}") return "\n".join(lines) def save_archive(output_dir: Path, run_id: str, doc_path: Path, results: list[dict[str, Any]], markdown: str) -> None: run_dir = output_dir / run_id run_dir.mkdir(parents=True, exist_ok=True) payload = { "run_id": run_id, "control_doc": repo_relative(doc_path), "results": results, } (run_dir / "summary.json").write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") (run_dir / "report.md").write_text(markdown, encoding="utf-8") def main() -> int: parser = argparse.ArgumentParser(description="按控制文档执行大规模自动测试") parser.add_argument( "--doc", default=str(DEFAULT_DOC_PATH), help="大规模测试控制文档路径", ) parser.add_argument( "--round", action="append", default=None, help="只执行指定 round id,可重复传入", ) parser.add_argument( "--dry-run", action="store_true", help="只解析文档和打印计划,不实际执行", ) args = parser.parse_args() control_doc = read_control_document(args.doc) control = control_doc.frontmatter.get("large_scale_test", {}) or {} if not control: raise ValueError("控制文档 frontmatter 缺少 large_scale_test 配置") config_path = str(control.get("config_path") or "tests/fixtures/auto_sync/auto_test_config.yaml") auto_test_config = load_yaml_config(config_path) report_base = Path(auto_test_config["paths"]["report_dir"]) output_dir = Path(control.get("output_dir") or "test_results/auto_test/large_scale") if not output_dir.is_absolute(): output_dir = PROJECT_ROOT / output_dir output_dir.mkdir(parents=True, exist_ok=True) configured_rounds = control.get("rounds") or [] selected_round_ids = set(args.round or []) rounds: list[dict[str, Any]] = [] for item in configured_rounds: if not isinstance(item, dict): continue if not item.get("enabled", True): continue if selected_round_ids and str(item.get("id")) not in selected_round_ids: continue rounds.append(item) if not rounds: raise ValueError("没有可执行的 round;请检查 enabled 或 --round 参数") if args.dry_run: for item in rounds: print(f"{item.get('id')}: {item.get('title')}") return 0 run_id = now_slug() script_dir = PROJECT_ROOT / "scripts" results: list[dict[str, Any]] = [] for round_config in rounds: results.append( execute_round( script_dir=script_dir, config_path=config_path, report_base=report_base, round_config=round_config, control=control, previous_rounds=results, ) ) markdown = build_results_markdown(run_id, control_doc.path, results) save_archive(output_dir, run_id, control_doc.path, results, markdown) write_control_document(control_doc, markdown) fail_on_failed_cases = bool(control.get("fail_on_failed_cases", False)) if fail_on_failed_cases and any(item["summary"].get("failed_cases", 0) > 0 for item in results): return 1 if any(item.get("returncode") not in (0, None) and not item.get("has_summary") for item in results): return 1 return 0 if __name__ == "__main__": raise SystemExit(main())