from __future__ import annotations import json import uuid from pathlib import Path from typing import Any import yaml from auto_test_lib import ( copy_selected_files, copy_tree_contents, diff_record_fields, enrich_supplier_display_fields, latest_log_file, merge_tree_contents, normalize_contract_settlement_records, normalize_construction_task_plan_nodes, persistence_summary, read_jsonl_records, load_yaml_config, run_command, sqlite_rows, ) def _tail_text(value: str | None, max_chars: int = 1000) -> str: if not value: return "" if len(value) <= max_chars: return value return value[-max_chars:] def collect_required_bindings(db_path: str, required_bindings: dict[str, list[str]]) -> dict[str, Any]: result: dict[str, Any] = {} for node_type, local_ids in required_bindings.items(): result[node_type] = [] local_nodes = sqlite_rows( db_path, "SELECT node_id, data_id FROM nodes WHERE collection_id = 'local' AND node_type = ?", (node_type,), ) remote_nodes = sqlite_rows( db_path, "SELECT node_id, data_id FROM nodes WHERE collection_id = 'remote' AND node_type = ?", (node_type,), ) bindings = sqlite_rows( db_path, "SELECT local_id, remote_id FROM bindings WHERE node_type = ?", (node_type,), ) local_data_to_node = {item["data_id"]: item["node_id"] for item in local_nodes} remote_node_to_data = {item["node_id"]: item["data_id"] for item in remote_nodes} bound_map = {item["local_id"]: item.get("remote_id") for item in bindings} for local_id in local_ids: local_node_id = local_data_to_node.get(local_id) remote_node_id = bound_map.get(local_node_id) if local_node_id else None result[node_type].append( { "local_data_id": local_id, "local_node_id": local_node_id, "remote_node_id": remote_node_id, "remote_data_id": remote_node_to_data.get(remote_node_id) if remote_node_id else None, "bound": remote_node_id is not None, } ) return result def collect_problem_nodes(db_path: str, node_types: list[str] | None = None) -> list[dict[str, Any]]: problems: list[dict[str, Any]] = [] seen: set[tuple[str | None, str | None]] = set() sql = """ SELECT collection_id, node_type, node_id, data_id, bind_data_id, action, status, binding_status, error, sync_log, context FROM nodes """ params: tuple[Any, ...] = () if node_types: placeholders = ", ".join("?" for _ in node_types) sql += f" WHERE node_type IN ({placeholders})" params = tuple(node_types) sql += " ORDER BY node_type, collection_id, node_id" rows = sqlite_rows(db_path, sql, params) for row in rows: status = row.get("status") binding_status = row.get("binding_status") error = row.get("error") has_problem = bool(error) or status == "FAILED" or (binding_status not in (None, "", "normal")) if not has_problem: continue key = (row.get("collection_id"), row.get("node_id")) if key in seen: continue seen.add(key) problems.append( { "collection_id": row.get("collection_id"), "node_type": row.get("node_type"), "node_id": row.get("node_id"), "data_id": row.get("data_id"), "bind_data_id": row.get("bind_data_id"), "action": row.get("action"), "status": status, "binding_status": binding_status, "error": error, "sync_log_tail": _tail_text(row.get("sync_log"), 1600), "context": row.get("context"), } ) return problems def collect_round_diffs(previous_dataset_dir: str | None, current_dataset_dir: str) -> dict[str, dict[str, list[str]]]: if not previous_dataset_dir: return {} previous_root = Path(previous_dataset_dir) current_root = Path(current_dataset_dir) diffs: dict[str, dict[str, list[str]]] = {} for after_file in sorted(current_root.glob("*.jsonl")): before_file = previous_root / after_file.name if not before_file.exists(): continue node_type = _infer_node_type_from_jsonl(after_file) before_records = read_jsonl_records(before_file) after_records = read_jsonl_records(after_file) file_diff: dict[str, list[str]] = {} for record_id, after in after_records.items(): before = before_records.get(record_id) if not before: continue changed = diff_record_fields(before, after) if changed: file_diff[record_id] = changed if file_diff: diffs[node_type] = file_diff return diffs def collect_round_diff(previous_dataset_dir: str | None, current_dataset_dir: str, domain_name: str) -> dict[str, list[str]]: return collect_round_diffs(previous_dataset_dir, current_dataset_dir).get(domain_name, {}) def _resolve_field_path(data: Any, field_path: str) -> Any: current = data for part in field_path.split("."): if isinstance(current, dict): current = current.get(part) continue if isinstance(current, list) and part.isdigit(): index = int(part) if 0 <= index < len(current): current = current[index] continue return None return current def collect_node_data_by_data_ids( db_path: str, *, collection_id: str, node_type: str, data_ids: list[str], ) -> dict[str, Any]: if not data_ids: return {} placeholders = ", ".join("?" for _ in data_ids) rows = sqlite_rows( db_path, ( "SELECT data_id, data FROM nodes " "WHERE collection_id = ? AND node_type = ? AND data_id IN (" f"{placeholders})" ), (collection_id, node_type, *data_ids), ) result: dict[str, Any] = {} for row in rows: raw = row.get("data") data_id = row.get("data_id") if data_id is None or not raw: continue try: result[str(data_id)] = json.loads(raw) except json.JSONDecodeError: continue return result def make_local_data_id(node_type: str, remote_data_id: str) -> str: return str(uuid.uuid5(uuid.NAMESPACE_URL, f"auto-test-local:{node_type}:{remote_data_id}")) def _infer_node_type_from_jsonl(file_path: Path) -> str: name = file_path.name if name.endswith("_1.jsonl"): return name[:-8] if name.endswith(".jsonl"): return name[:-6] return file_path.stem def remap_project_scope_local_ids( dataset_root: str | Path, *, remote_project_id: str, local_project_id: str, remap_node_types: set[str], ) -> None: root = Path(dataset_root) for file_path in sorted(root.glob("*.jsonl")): node_type = _infer_node_type_from_jsonl(file_path) raw_lines = file_path.read_text(encoding="utf-8").splitlines() rewritten: list[str] = [] changed = False for line in raw_lines: raw = line.strip() if not raw: continue item = json.loads(raw) original = json.dumps(item, ensure_ascii=False, sort_keys=True) if item.get("project_id") == remote_project_id: item["project_id"] = local_project_id if node_type == "project" and item.get("_id") == remote_project_id: item["_id"] = local_project_id elif node_type in remap_node_types and item.get("project_id") == local_project_id and item.get("_id"): item["_id"] = make_local_data_id(node_type, str(item["_id"])) changed = changed or (json.dumps(item, ensure_ascii=False, sort_keys=True) != original) rewritten.append(json.dumps(item, ensure_ascii=False)) if changed: file_path.write_text("\n".join(rewritten) + "\n", encoding="utf-8") def remap_project_scope_dataset_diffs( dataset_diffs: dict[str, dict[str, list[str]]], *, remote_project_id: str, local_project_id: str, remap_node_types: set[str], ) -> dict[str, dict[str, list[str]]]: remapped: dict[str, dict[str, list[str]]] = {} for node_type, record_changes in dataset_diffs.items(): remapped[node_type] = {} for record_id, changed_fields in record_changes.items(): new_record_id = record_id if node_type == "project" and record_id == remote_project_id: new_record_id = local_project_id elif node_type in remap_node_types: new_record_id = make_local_data_id(node_type, record_id) remapped[node_type][new_record_id] = changed_fields return remapped def build_temp_run_profile_for_project_auto_bind( *, config: dict[str, Any], report_root: str | Path, profile_name: str, local_project_id: str, remote_project_id: str, project_update_direction: str | None = None, project_local_orphan_action: str | None = None, project_remote_orphan_action: str | None = None, project_detail_push_keys: set[str] | None = None, ) -> tuple[str, list[str]]: profile = load_yaml_config(config["pipeline"]["run_profile"]) profile.setdefault("local_datasource", {})["target_project_ids"] = [local_project_id] profile.setdefault("remote_datasource", {})["target_project_ids"] = [remote_project_id] project_strategy = profile.setdefault("strategies", {}).setdefault("project", {}).setdefault("config", {}) project_strategy["auto_bind"] = False project_strategy["auto_bind_fields"] = [] project_strategy["pre_bind_data_id"] = [ { "local_data_id": local_project_id, "remote_data_id": remote_project_id, } ] project_strategy["local_orphan_action"] = project_local_orphan_action or "none" project_strategy["remote_orphan_action"] = project_remote_orphan_action or "none" project_strategy["update_direction"] = project_update_direction or "none" if project_detail_push_keys: project_detail_config = profile.setdefault("strategies", {}).setdefault("project_detail_data", {}).setdefault("config", {}) overrides = dict(project_detail_config.get("field_direction_overrides", {})) for key in project_detail_push_keys: overrides.pop(key, None) project_detail_config["field_direction_overrides"] = overrides runtime_dir = Path(report_root) / "runtime_configs" runtime_dir.mkdir(parents=True, exist_ok=True) profile_path = runtime_dir / f"{profile_name}.yaml" profile_path.write_text(yaml.safe_dump(profile, allow_unicode=True, sort_keys=False), encoding="utf-8") command = list(config["pipeline"]["command"]) if "--config_path" in command: index = command.index("--config_path") if index + 1 < len(command): command[index + 1] = str(profile_path) else: command.append(str(profile_path)) else: command.extend(["--config_path", str(profile_path)]) return str(profile_path), command def run_pipeline_round( *, config: dict[str, Any], dataset_dir: str, previous_dataset_dir: str | None, base_dataset_dir: str | None = None, base_dataset_files: list[str] | None = None, domain_name: str, round_name: str, purpose: str, expected_changes: dict[str, Any] | None = None, required_bindings: dict[str, list[str]] | None = None, expected_binding_pairs: dict[str, dict[str, str]] | None = None, require_distinct_binding_ids: dict[str, list[str]] | None = None, expected_bound_fields: dict[str, dict[str, dict[str, dict[str, Any]]]] | None = None, project_id_remap: dict[str, Any] | None = None, project_detail_push_keys: set[str] | None = None, report_root: str | Path | None = None, coverage: dict[str, Any] | None = None, ) -> dict[str, Any]: pipeline = config["pipeline"] paths = config["paths"] if base_dataset_dir and base_dataset_files: copy_selected_files(base_dataset_dir, paths["temp_data_dir"], base_dataset_files) merge_tree_contents(dataset_dir, paths["temp_data_dir"]) else: copy_tree_contents(dataset_dir, paths["temp_data_dir"]) common_dataset_dir = paths.get("common_dataset_dir") if common_dataset_dir and Path(common_dataset_dir).exists(): merge_tree_contents(common_dataset_dir, paths["temp_data_dir"]) enrich_supplier_display_fields( paths["temp_data_dir"], f"{config['project_root']}/filtered_datasource/datasource/filtered/region_1.jsonl", ) normalize_contract_settlement_records(paths["temp_data_dir"]) normalize_construction_task_plan_nodes(paths["temp_data_dir"]) command = pipeline["command"] temp_run_profile_path = None if project_id_remap: local_project_id = str(project_id_remap["local_project_id"]) remote_project_id = str(project_id_remap["remote_project_id"]) remap_project_scope_local_ids( paths["temp_data_dir"], remote_project_id=remote_project_id, local_project_id=local_project_id, remap_node_types=set(project_id_remap.get("remap_node_types", set())), ) if report_root is None: raise ValueError("project_id_remap requires report_root") temp_run_profile_path, command = build_temp_run_profile_for_project_auto_bind( config=config, report_root=report_root, profile_name=f"{domain_name}_{round_name}", local_project_id=local_project_id, remote_project_id=remote_project_id, project_update_direction=project_id_remap.get("project_update_direction"), project_local_orphan_action=project_id_remap.get("project_local_orphan_action"), project_remote_orphan_action=project_id_remap.get("project_remote_orphan_action"), project_detail_push_keys=project_detail_push_keys, ) result = run_command(command, pipeline["project_root"]) latest_log = latest_log_file(paths["pipeline_log_dir"]) dataset_diff = collect_round_diff(previous_dataset_dir, dataset_dir, domain_name) dataset_diffs = collect_round_diffs(previous_dataset_dir, dataset_dir) if project_id_remap: dataset_diffs = remap_project_scope_dataset_diffs( dataset_diffs, remote_project_id=str(project_id_remap["remote_project_id"]), local_project_id=str(project_id_remap["local_project_id"]), remap_node_types=set(project_id_remap.get("remap_node_types", set())), ) dataset_diff = dataset_diffs.get(domain_name, {}) binding_report = collect_required_bindings(paths["persistence_db"], required_bindings or {}) persist_report = persistence_summary(paths["persistence_db"], domain_name) watched_problem_types = set(required_bindings or {}) watched_problem_types.update(expected_changes or {}) for collection_node_types in (expected_bound_fields or {}).values(): watched_problem_types.update(collection_node_types) watched_problem_types.add(domain_name) problem_nodes = collect_problem_nodes( paths["persistence_db"], sorted(watched_problem_types) if watched_problem_types else None, ) issues: list[str] = [] if result.returncode != 0: issues.append(f"pipeline 返回码不是 0: {result.returncode}") for node_type, items in binding_report.items(): for item in items: if not item["bound"]: issues.append(f"{node_type} 未建立绑定: {item['local_data_id']}") expected_binding_pairs = expected_binding_pairs or {} for node_type, local_to_remote in expected_binding_pairs.items(): actual_items = {item["local_data_id"]: item for item in binding_report.get(node_type, [])} for local_id, expected_remote_id in local_to_remote.items(): actual = actual_items.get(local_id) if not actual: issues.append(f"{node_type} 缺少绑定检查记录: {local_id}") continue if actual.get("remote_data_id") != expected_remote_id: issues.append( f"{node_type} 绑定目标不符合预期: local={local_id} remote={actual.get('remote_data_id')} expected={expected_remote_id}" ) require_distinct_binding_ids = require_distinct_binding_ids or {} for node_type, local_ids in require_distinct_binding_ids.items(): actual_items = {item["local_data_id"]: item for item in binding_report.get(node_type, [])} for local_id in local_ids: actual = actual_items.get(local_id) if not actual or not actual.get("bound"): continue if actual.get("remote_data_id") == local_id: issues.append(f"{node_type} 绑定后 local/remote data_id 不应相同: {local_id}") expected_bound_fields = expected_bound_fields or {} verified_bound_fields: list[dict[str, Any]] = [] for collection_id, node_types in expected_bound_fields.items(): for node_type, local_expectations in node_types.items(): binding_items = {item["local_data_id"]: item for item in binding_report.get(node_type, [])} target_id_by_local: dict[str, str] = {} for local_data_id in local_expectations: if collection_id == "local": target_id_by_local[local_data_id] = local_data_id continue remote_data_id = (binding_items.get(local_data_id) or {}).get("remote_data_id") if remote_data_id: target_id_by_local[local_data_id] = remote_data_id records = collect_node_data_by_data_ids( paths["persistence_db"], collection_id=collection_id, node_type=node_type, data_ids=list(target_id_by_local.values()), ) for local_data_id, expected_fields in local_expectations.items(): target_data_id = target_id_by_local.get(local_data_id) record = records.get(target_data_id) if target_data_id else None if not target_data_id or record is None: issues.append( f"{collection_id}.{node_type} 字段校验缺少目标记录: local={local_data_id} resolved={target_data_id}" ) continue actual_fields: dict[str, Any] = {} mismatches: list[str] = [] for field_path, expected_value in expected_fields.items(): actual_value = _resolve_field_path(record, field_path) actual_fields[field_path] = actual_value if actual_value != expected_value: mismatches.append( f"{field_path}: actual={actual_value!r} expected={expected_value!r}" ) verified_bound_fields.append( { "collection_id": collection_id, "node_type": node_type, "local_data_id": local_data_id, "target_data_id": target_data_id, "matched": not mismatches, "expected_fields": expected_fields, "actual_fields": actual_fields, } ) if mismatches: issues.append( f"{collection_id}.{node_type} 字段校验失败: local={local_data_id} target={target_data_id}; " + "; ".join(mismatches) ) relevant_local_ids: set[str] = set() for local_ids in (required_bindings or {}).values(): relevant_local_ids.update(local_ids) relevant_remote_ids: set[str] = set() for items in binding_report.values(): for item in items: remote_data_id = item.get("remote_data_id") if remote_data_id: relevant_remote_ids.add(str(remote_data_id)) for remote_map in (expected_binding_pairs or {}).values(): relevant_remote_ids.update(str(remote_id) for remote_id in remote_map.values() if remote_id) matched_verified_local_keys = { (entry["node_type"], entry["local_data_id"]) for entry in verified_bound_fields if entry.get("matched") } for item in problem_nodes: item_data_id = item.get("data_id") if item.get("collection_id") == "local": if item_data_id not in relevant_local_ids: continue elif item.get("collection_id") == "remote": if not item_data_id or item_data_id not in relevant_remote_ids: continue error_text = item.get("error") or "" if item.get("status") == "FAILED": issues.append( f"{item['collection_id']}.{item['node_type']} 节点失败: data_id={item.get('data_id')} action={item.get('action')} error={error_text or ''}" ) elif item.get("binding_status") not in (None, "", "normal"): if ( item.get("collection_id") == "local" and (item.get("node_type"), item.get("data_id")) in matched_verified_local_keys ): continue issues.append( f"{item['collection_id']}.{item['node_type']} 绑定异常: data_id={item.get('data_id')} binding_status={item.get('binding_status')}" ) elif error_text: continue expected_changes = expected_changes or {} for node_type, record_changes in expected_changes.items(): node_diff = dataset_diffs.get(node_type, {}) for record_id, expected_fields in record_changes.items(): actual_changed_fields: list[str] = node_diff.get(record_id, []) missing = [field for field in expected_fields if field not in actual_changed_fields] if missing: issues.append(f"数据变更字段不完整: {record_id} 缺少 {', '.join(missing)}") return { "name": round_name, "purpose": purpose, "dataset_dir": dataset_dir, "returncode": result.returncode, "passed": not issues, "issues": issues, "stdout_tail": result.stdout[-2000:], "stderr_tail": result.stderr[-2000:], "latest_log_file": latest_log, "expected_changes": expected_changes, "dataset_diff": dataset_diff, "dataset_diffs": dataset_diffs, "required_bindings": binding_report, "expected_binding_pairs": expected_binding_pairs, "require_distinct_binding_ids": require_distinct_binding_ids, "verified_bound_fields": verified_bound_fields, "problem_nodes": problem_nodes, "persistence_summary": persist_report, "temp_run_profile_path": temp_run_profile_path, "coverage": coverage or {}, } def build_domain_report(domain: str, description: str, rounds: list[dict[str, Any]]) -> dict[str, Any]: issues: list[str] = [] for round_report in rounds: for issue in round_report.get("issues", []): issues.append(f"{round_report['name']}: {issue}") return { "domain": domain, "description": description, "passed": len(issues) == 0, "issues": issues, "rounds": rounds, }