strepsiades
444 lines
17 KiB
Python
444 lines
17 KiB
Python
from __future__ import annotations
|
||
|
||
import argparse
|
||
import fnmatch
|
||
import json
|
||
import re
|
||
import subprocess
|
||
import sys
|
||
from dataclasses import dataclass
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import yaml
|
||
|
||
from auto_test_lib import PROJECT_ROOT, load_yaml_config, now_slug
|
||
|
||
|
||
DEFAULT_DOC_PATH = PROJECT_ROOT / "docs/auto_test_spec/large_scale_test_control.md"
|
||
RESULT_START_MARKER = "<!-- LARGE_SCALE_RESULTS:START -->"
|
||
RESULT_END_MARKER = "<!-- LARGE_SCALE_RESULTS:END -->"
|
||
|
||
|
||
@dataclass
|
||
class ControlDocument:
|
||
path: Path
|
||
frontmatter: dict[str, Any]
|
||
body: str
|
||
|
||
|
||
def read_control_document(doc_path: str | Path) -> ControlDocument:
|
||
path = Path(doc_path)
|
||
if not path.is_absolute():
|
||
path = PROJECT_ROOT / path
|
||
text = path.read_text(encoding="utf-8")
|
||
match = re.match(r"^---\n(.*?)\n---\n?", text, re.DOTALL)
|
||
if not match:
|
||
raise ValueError(f"控制文档缺少 YAML frontmatter: {path}")
|
||
payload = yaml.safe_load(match.group(1)) or {}
|
||
body = text[match.end() :]
|
||
return ControlDocument(path=path, frontmatter=payload, body=body)
|
||
|
||
|
||
def write_control_document(doc: ControlDocument, generated_section: str) -> None:
|
||
body = doc.body
|
||
if RESULT_START_MARKER in body and RESULT_END_MARKER in body:
|
||
body = re.sub(
|
||
rf"{re.escape(RESULT_START_MARKER)}.*?{re.escape(RESULT_END_MARKER)}",
|
||
f"{RESULT_START_MARKER}\n\n{generated_section}\n\n{RESULT_END_MARKER}",
|
||
body,
|
||
flags=re.DOTALL,
|
||
)
|
||
else:
|
||
body = (
|
||
body.rstrip()
|
||
+ "\n\n"
|
||
+ RESULT_START_MARKER
|
||
+ "\n\n"
|
||
+ generated_section
|
||
+ "\n\n"
|
||
+ RESULT_END_MARKER
|
||
+ "\n"
|
||
)
|
||
|
||
frontmatter_text = yaml.safe_dump(doc.frontmatter, allow_unicode=True, sort_keys=False).strip()
|
||
final_text = f"---\n{frontmatter_text}\n---\n{body.lstrip()}"
|
||
doc.path.write_text(final_text, encoding="utf-8")
|
||
|
||
|
||
def repo_relative(path: str | Path | None) -> str:
|
||
if not path:
|
||
return ""
|
||
resolved = Path(path)
|
||
if not resolved.is_absolute():
|
||
return str(resolved)
|
||
try:
|
||
return str(resolved.relative_to(PROJECT_ROOT))
|
||
except ValueError:
|
||
return str(resolved)
|
||
|
||
|
||
def list_report_dirs(report_base: Path) -> set[Path]:
|
||
if not report_base.exists():
|
||
return set()
|
||
return {item for item in report_base.iterdir() if item.is_dir()}
|
||
|
||
|
||
def extract_report_dir(stdout: str, stderr: str) -> Path | None:
|
||
combined = "\n".join(part for part in [stdout, stderr] if part)
|
||
match = re.search(r"(?:聚合)?报告已生成:\s*(.+?/report\.md)", combined)
|
||
if not match:
|
||
return None
|
||
return Path(match.group(1).strip()).parent
|
||
|
||
|
||
def build_command(script_dir: Path, config_path: str, round_config: dict[str, Any], control: dict[str, Any]) -> list[str]:
|
||
runner = str(round_config.get("runner") or control.get("default_runner") or "aggregate")
|
||
init_mode = str(round_config.get("init_mode") or control.get("default_init_mode") or "fresh")
|
||
|
||
if runner not in {"aggregate", "domains"}:
|
||
raise ValueError(f"不支持的 runner: {runner}")
|
||
if init_mode not in {"fresh", "reuse", "clear-only"}:
|
||
raise ValueError(f"不支持的 init_mode: {init_mode}")
|
||
|
||
if runner == "aggregate":
|
||
command = [sys.executable, str(script_dir / "run_auto_test_all_domains.py"), "--config", config_path]
|
||
else:
|
||
domains = round_config.get("domains") or []
|
||
if not domains:
|
||
raise ValueError(f"domains runner 必须显式提供 domains: {round_config.get('id')}")
|
||
command = [sys.executable, str(script_dir / "run_auto_test.py"), "--config", config_path]
|
||
for domain in domains:
|
||
command.extend(["--domain", str(domain)])
|
||
|
||
if init_mode == "reuse":
|
||
command.append("--skip-init")
|
||
elif init_mode == "clear-only":
|
||
command.append("--clear-only")
|
||
|
||
return command
|
||
|
||
|
||
def case_map(summary_payload: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
||
summary = summary_payload.get("summary", {})
|
||
return {
|
||
case["nodeid"]: case
|
||
for case in summary.get("cases", [])
|
||
if isinstance(case, dict) and case.get("nodeid")
|
||
}
|
||
|
||
|
||
def match_classification(case_id: str, round_config: dict[str, Any], control: dict[str, Any]) -> str:
|
||
merged: dict[str, str] = {}
|
||
global_rules = control.get("classification", {}) or {}
|
||
round_rules = round_config.get("classification", {}) or {}
|
||
merged.update({str(key): str(value) for key, value in global_rules.items()})
|
||
merged.update({str(key): str(value) for key, value in round_rules.items()})
|
||
for pattern, category in merged.items():
|
||
if fnmatch.fnmatch(case_id, pattern):
|
||
return category
|
||
return "unclassified"
|
||
|
||
|
||
def find_compare_target(previous_rounds: list[dict[str, Any]], current_cases: dict[str, dict[str, Any]]) -> tuple[dict[str, Any] | None, set[str]]:
|
||
current_ids = set(current_cases)
|
||
for item in reversed(previous_rounds):
|
||
previous_cases = item.get("case_results", {})
|
||
overlap = current_ids & set(previous_cases)
|
||
if overlap:
|
||
return item, overlap
|
||
return None, set()
|
||
|
||
|
||
def compare_rounds(previous_round: dict[str, Any] | None, overlap: set[str], current_cases: dict[str, dict[str, Any]]) -> dict[str, Any]:
|
||
if previous_round is None or not overlap:
|
||
return {
|
||
"compare_round_id": None,
|
||
"overlap_count": 0,
|
||
"fixed": [],
|
||
"regressed": [],
|
||
"persistent_failed": [],
|
||
"stable_passed": [],
|
||
}
|
||
|
||
previous_cases = previous_round["case_results"]
|
||
fixed: list[str] = []
|
||
regressed: list[str] = []
|
||
persistent_failed: list[str] = []
|
||
stable_passed: list[str] = []
|
||
|
||
for case_id in sorted(overlap):
|
||
old_passed = bool(previous_cases[case_id].get("passed"))
|
||
new_passed = bool(current_cases[case_id].get("passed"))
|
||
if not old_passed and new_passed:
|
||
fixed.append(case_id)
|
||
elif old_passed and not new_passed:
|
||
regressed.append(case_id)
|
||
elif not old_passed and not new_passed:
|
||
persistent_failed.append(case_id)
|
||
else:
|
||
stable_passed.append(case_id)
|
||
|
||
return {
|
||
"compare_round_id": previous_round.get("id"),
|
||
"overlap_count": len(overlap),
|
||
"fixed": fixed,
|
||
"regressed": regressed,
|
||
"persistent_failed": persistent_failed,
|
||
"stable_passed": stable_passed,
|
||
}
|
||
|
||
|
||
def execute_round(
|
||
*,
|
||
script_dir: Path,
|
||
config_path: str,
|
||
report_base: Path,
|
||
round_config: dict[str, Any],
|
||
control: dict[str, Any],
|
||
previous_rounds: list[dict[str, Any]],
|
||
) -> dict[str, Any]:
|
||
command = build_command(script_dir, config_path, round_config, control)
|
||
before_dirs = list_report_dirs(report_base)
|
||
result = subprocess.run(command, cwd=str(PROJECT_ROOT), text=True, capture_output=True, check=False)
|
||
after_dirs = list_report_dirs(report_base)
|
||
report_dir = extract_report_dir(result.stdout or "", result.stderr or "")
|
||
if report_dir is None:
|
||
new_dirs = sorted(after_dirs - before_dirs)
|
||
if new_dirs:
|
||
report_dir = new_dirs[-1]
|
||
|
||
summary_payload: dict[str, Any] = {}
|
||
if report_dir is not None:
|
||
summary_path = report_dir / "summary.json"
|
||
if summary_path.exists():
|
||
summary_payload = json.loads(summary_path.read_text(encoding="utf-8"))
|
||
|
||
current_cases = case_map(summary_payload)
|
||
compare_round, overlap = find_compare_target(previous_rounds, current_cases)
|
||
comparison = compare_rounds(compare_round, overlap, current_cases)
|
||
|
||
failed_case_details = []
|
||
for case_id, item in sorted(current_cases.items()):
|
||
if item.get("passed"):
|
||
continue
|
||
failed_case_details.append(
|
||
{
|
||
"nodeid": case_id,
|
||
"classification": match_classification(case_id, round_config, control),
|
||
"issues": item.get("issues", []),
|
||
"issue_count": item.get("issue_count", len(item.get("issues", []))),
|
||
}
|
||
)
|
||
|
||
summary = summary_payload.get("summary", {})
|
||
return {
|
||
"id": str(round_config.get("id") or ""),
|
||
"title": str(round_config.get("title") or ""),
|
||
"objective": str(round_config.get("objective") or ""),
|
||
"focus": [str(item) for item in (round_config.get("focus") or [])],
|
||
"runner": str(round_config.get("runner") or control.get("default_runner") or "aggregate"),
|
||
"init_mode": str(round_config.get("init_mode") or control.get("default_init_mode") or "fresh"),
|
||
"domains": [str(item) for item in (round_config.get("domains") or [])],
|
||
"next_goal": str(round_config.get("next_goal") or ""),
|
||
"returncode": result.returncode,
|
||
"command": command,
|
||
"report_dir": str(report_dir) if report_dir else None,
|
||
"report_md": str(report_dir / "report.md") if report_dir else None,
|
||
"summary_json": str(report_dir / "summary.json") if report_dir else None,
|
||
"summary": {
|
||
"total_domains": int(summary.get("total_domains", 0)),
|
||
"passed_domains": int(summary.get("passed_domains", 0)),
|
||
"failed_domains": int(summary.get("failed_domains", 0)),
|
||
"total_cases": int(summary.get("total_cases", 0)),
|
||
"passed_cases": int(summary.get("passed_cases", 0)),
|
||
"failed_cases": int(summary.get("failed_cases", 0)),
|
||
},
|
||
"has_summary": bool(summary_payload),
|
||
"case_results": current_cases,
|
||
"failed_cases": failed_case_details,
|
||
"comparison": comparison,
|
||
"stdout_tail": (result.stdout or "")[-4000:],
|
||
"stderr_tail": (result.stderr or "")[-4000:],
|
||
}
|
||
|
||
|
||
def build_results_markdown(run_id: str, doc_path: Path, results: list[dict[str, Any]]) -> str:
|
||
lines = [
|
||
f"## 最近一次执行 {run_id}",
|
||
"",
|
||
f"- 控制文档: {repo_relative(doc_path)}",
|
||
f"- 执行轮次数: {len(results)}",
|
||
"- 说明: 默认优先使用 fresh 初始化；若使用 aggregate runner，则每个 domain 仍会独立初始化。",
|
||
"",
|
||
"### 轮次总览",
|
||
"",
|
||
"| Round | 标题 | Runner | Init | Cases | Failed | 对比轮次 | 报告 |",
|
||
"| --- | --- | --- | --- | --- | --- | --- | --- |",
|
||
]
|
||
|
||
for item in results:
|
||
summary = item["summary"]
|
||
compare_round_id = item["comparison"].get("compare_round_id") or "-"
|
||
report_link = repo_relative(item.get("report_md")) or "-"
|
||
lines.append(
|
||
f"| {item['id']} | {item['title']} | {item['runner']} | {item['init_mode']} | {summary['total_cases']} | {summary['failed_cases']} | {compare_round_id} | {report_link} |"
|
||
)
|
||
|
||
for item in results:
|
||
summary = item["summary"]
|
||
comparison = item["comparison"]
|
||
lines.extend(
|
||
[
|
||
"",
|
||
f"### {item['id']} - {item['title']}",
|
||
"",
|
||
f"- 目标: {item['objective']}",
|
||
f"- 运行方式: {item['runner']}",
|
||
f"- 初始化策略: {item['init_mode']}",
|
||
f"- 执行命令: {' '.join(item['command'])}",
|
||
f"- 单轮结果: {summary['passed_cases']} passed / {summary['failed_cases']} failed / {summary['total_cases']} total cases",
|
||
f"- Domain 结果: {summary['passed_domains']} passed / {summary['failed_domains']} failed / {summary['total_domains']} total domains",
|
||
f"- 原始报告: {repo_relative(item.get('report_md')) or '-'}",
|
||
f"- 原始汇总: {repo_relative(item.get('summary_json')) or '-'}",
|
||
]
|
||
)
|
||
if item["focus"]:
|
||
lines.append("- 本轮关注点:")
|
||
for focus in item["focus"]:
|
||
lines.append(f" - {focus}")
|
||
|
||
lines.extend(
|
||
[
|
||
f"- 对比轮次: {comparison.get('compare_round_id') or '无可比较轮次'}",
|
||
f"- 可比较 case 数: {comparison.get('overlap_count', 0)}",
|
||
]
|
||
)
|
||
if comparison.get("fixed"):
|
||
lines.append("- 本轮改进:")
|
||
for case_id in comparison["fixed"]:
|
||
lines.append(f" - fixed: {case_id}")
|
||
if comparison.get("regressed"):
|
||
lines.append("- 本轮回退:")
|
||
for case_id in comparison["regressed"]:
|
||
lines.append(f" - regressed: {case_id}")
|
||
if comparison.get("persistent_failed"):
|
||
lines.append("- 本轮遗留:")
|
||
for case_id in comparison["persistent_failed"]:
|
||
lines.append(f" - persistent_failed: {case_id}")
|
||
if not comparison.get("fixed") and not comparison.get("regressed") and not comparison.get("persistent_failed"):
|
||
lines.append("- 对比结果: 本轮没有可比改进项，或与此前轮次无重叠 case。")
|
||
|
||
if item["failed_cases"]:
|
||
lines.append("- 失败归因:")
|
||
for failed in item["failed_cases"]:
|
||
issue_text = "；".join(str(issue) for issue in failed.get("issues", [])) or "<empty>"
|
||
lines.append(
|
||
f" - {failed['nodeid']} | {failed['classification']} | issues={failed['issue_count']} | {issue_text}"
|
||
)
|
||
else:
|
||
lines.append("- 失败归因: 本轮无失败 case。")
|
||
|
||
if item.get("next_goal"):
|
||
lines.append(f"- 下一轮目标: {item['next_goal']}")
|
||
if item.get("returncode") not in (0, None):
|
||
lines.append(f"- 脚本返回码: {item['returncode']}")
|
||
|
||
return "\n".join(lines)
|
||
|
||
|
||
def save_archive(output_dir: Path, run_id: str, doc_path: Path, results: list[dict[str, Any]], markdown: str) -> None:
|
||
run_dir = output_dir / run_id
|
||
run_dir.mkdir(parents=True, exist_ok=True)
|
||
payload = {
|
||
"run_id": run_id,
|
||
"control_doc": repo_relative(doc_path),
|
||
"results": results,
|
||
}
|
||
(run_dir / "summary.json").write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
(run_dir / "report.md").write_text(markdown, encoding="utf-8")
|
||
|
||
|
||
def main() -> int:
|
||
parser = argparse.ArgumentParser(description="按控制文档执行大规模自动测试")
|
||
parser.add_argument(
|
||
"--doc",
|
||
default=str(DEFAULT_DOC_PATH),
|
||
help="大规模测试控制文档路径",
|
||
)
|
||
parser.add_argument(
|
||
"--round",
|
||
action="append",
|
||
default=None,
|
||
help="只执行指定 round id，可重复传入",
|
||
)
|
||
parser.add_argument(
|
||
"--dry-run",
|
||
action="store_true",
|
||
help="只解析文档和打印计划，不实际执行",
|
||
)
|
||
args = parser.parse_args()
|
||
|
||
control_doc = read_control_document(args.doc)
|
||
control = control_doc.frontmatter.get("large_scale_test", {}) or {}
|
||
if not control:
|
||
raise ValueError("控制文档 frontmatter 缺少 large_scale_test 配置")
|
||
|
||
config_path = str(control.get("config_path") or "tests/fixtures/auto_sync/auto_test_config.yaml")
|
||
auto_test_config = load_yaml_config(config_path)
|
||
report_base = Path(auto_test_config["paths"]["report_dir"])
|
||
output_dir = Path(control.get("output_dir") or "test_results/auto_test/large_scale")
|
||
if not output_dir.is_absolute():
|
||
output_dir = PROJECT_ROOT / output_dir
|
||
output_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
configured_rounds = control.get("rounds") or []
|
||
selected_round_ids = set(args.round or [])
|
||
rounds: list[dict[str, Any]] = []
|
||
for item in configured_rounds:
|
||
if not isinstance(item, dict):
|
||
continue
|
||
if not item.get("enabled", True):
|
||
continue
|
||
if selected_round_ids and str(item.get("id")) not in selected_round_ids:
|
||
continue
|
||
rounds.append(item)
|
||
|
||
if not rounds:
|
||
raise ValueError("没有可执行的 round；请检查 enabled 或 --round 参数")
|
||
|
||
if args.dry_run:
|
||
for item in rounds:
|
||
print(f"{item.get('id')}: {item.get('title')}")
|
||
return 0
|
||
|
||
run_id = now_slug()
|
||
script_dir = PROJECT_ROOT / "scripts"
|
||
results: list[dict[str, Any]] = []
|
||
|
||
for round_config in rounds:
|
||
results.append(
|
||
execute_round(
|
||
script_dir=script_dir,
|
||
config_path=config_path,
|
||
report_base=report_base,
|
||
round_config=round_config,
|
||
control=control,
|
||
previous_rounds=results,
|
||
)
|
||
)
|
||
|
||
markdown = build_results_markdown(run_id, control_doc.path, results)
|
||
save_archive(output_dir, run_id, control_doc.path, results, markdown)
|
||
write_control_document(control_doc, markdown)
|
||
|
||
fail_on_failed_cases = bool(control.get("fail_on_failed_cases", False))
|
||
if fail_on_failed_cases and any(item["summary"].get("failed_cases", 0) > 0 for item in results):
|
||
return 1
|
||
if any(item.get("returncode") not in (0, None) and not item.get("has_summary") for item in results):
|
||
return 1
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main()) |