from __future__ import annotations import argparse import base64 import json import os import shutil import signal import subprocess import sys import time from copy import deepcopy from pathlib import Path from typing import Any from urllib.parse import urlparse from urllib import error, request import yaml PROJECT_ROOT = Path(__file__).resolve().parents[1] STATE_DIR = PROJECT_ROOT / "runtime" / "remote_env" STATE_FILE = STATE_DIR / "state.json" DEFAULT_CONFIG: dict[str, Any] = { "couchdb": { "enabled": True, "base_url": "http://localhost:5984", "url": "http://127.0.0.1", "port": "5984", "database": "test_push_auto", "username": "admin", "password": "password123", "recreate_after_delete": False, }, "data": { "data_dir": "${PROJECT_ROOT}/working_local_datasource/filtered", }, "service": { "ecm_root": "/Users/zyl/my_projects/GroupSideProjectManagementSystem", "python_path": "/Users/zyl/my_projects/GroupSideProjectManagementSystem/.venv/bin/python", "cwd": "${ECM_ROOT}", "start_command": ["${PYTHON}", "run.py"], "restore_command": [ "${PYTHON}", "run.py", "restore-database", "-d", "${COUCHDB_DATABASE}", "-s", "${DATA_DIR}", ], "env": {}, "ready_url": "http://localhost:8000/docs", "ready_timeout_seconds": 120, "ready_interval_seconds": 2, "log_file": "${PROJECT_ROOT}/runtime/remote_env/ecm_remote.log", }, "env_file": { "enabled": True, "path": "${ECM_ROOT}/.env", "restore_on_stop": True, "overrides": { "COUCHDB_DBNAME": "${COUCHDB_DATABASE}", "COUCHDB_USER": "${COUCHDB_USERNAME}", "COUCHDB_PASSWORD": "${COUCHDB_PASSWORD}", "COUCHDB_URL": "${COUCHDB_URL}", "COUCHDB_PORT": "${COUCHDB_PORT}", }, }, } def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]: merged = deepcopy(base) for key, value in override.items(): if isinstance(value, dict) and isinstance(merged.get(key), dict): merged[key] = _deep_merge(merged[key], value) else: merged[key] = value return merged def _replace_placeholders(value: Any, placeholders: dict[str, str]) -> Any: if isinstance(value, str): result = value for _ in range(5): previous = result for key, replacement in placeholders.items(): result = result.replace(f"${{{key}}}", replacement) if result == previous: break return result if isinstance(value, list): return [_replace_placeholders(item, placeholders) for item in value] if isinstance(value, dict): return {key: _replace_placeholders(item, placeholders) for key, item in value.items()} return value def _ensure_parent(path: str | Path) -> Path: resolved = Path(path) resolved.parent.mkdir(parents=True, exist_ok=True) return resolved def _load_raw_config(config_path: str | Path) -> tuple[dict[str, Any], Path]: path = Path(config_path) if not path.is_absolute(): path = PROJECT_ROOT / path payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {} if not isinstance(payload, dict): raise ValueError(f"Config root must be a mapping: {path}") return payload, path def _apply_cli_overrides(config: dict[str, Any], args: argparse.Namespace) -> dict[str, Any]: result = deepcopy(config) couchdb = result.setdefault("couchdb", {}) service = result.setdefault("service", {}) data = result.setdefault("data", {}) if args.db_name is not None: couchdb["database"] = args.db_name if args.db_user is not None: couchdb["username"] = args.db_user if args.db_password is not None: couchdb["password"] = args.db_password if args.couchdb_base_url is not None: couchdb["base_url"] = args.couchdb_base_url if args.couchdb_url is not None: couchdb["url"] = args.couchdb_url if args.couchdb_port is not None: couchdb["port"] = str(args.couchdb_port) if args.ecm_root is not None: service["ecm_root"] = args.ecm_root if args.python_path is not None: service["python_path"] = args.python_path if args.data_dir is not None: data["data_dir"] = args.data_dir if args.ready_url is not None: service["ready_url"] = args.ready_url return result def load_config(config_path: str | Path, args: argparse.Namespace) -> tuple[dict[str, Any], Path]: raw_config, resolved_path = _load_raw_config(config_path) merged = _deep_merge(DEFAULT_CONFIG, raw_config) merged = _apply_cli_overrides(merged, args) config_dir = resolved_path.parent.resolve() service = merged.setdefault("service", {}) couchdb = merged.setdefault("couchdb", {}) data = merged.setdefault("data", {}) placeholders = { "PROJECT_ROOT": str(PROJECT_ROOT), "CONFIG_DIR": str(config_dir), "ECM_ROOT": str(service.get("ecm_root", "")), "PYTHON": str(service.get("python_path", "")), "DATA_DIR": str(data.get("data_dir", "")), "COUCHDB_DATABASE": str(couchdb.get("database", "")), "COUCHDB_USERNAME": str(couchdb.get("username", "")), "COUCHDB_PASSWORD": str(couchdb.get("password", "")), "COUCHDB_URL": str(couchdb.get("url", "")), "COUCHDB_PORT": str(couchdb.get("port", "")), } resolved = _replace_placeholders(merged, placeholders) service = resolved.setdefault("service", {}) couchdb = resolved.setdefault("couchdb", {}) data = resolved.setdefault("data", {}) env_file = resolved.setdefault("env_file", {}) for key in ("ecm_root", "python_path", "cwd", "log_file"): if service.get(key): service[key] = str(Path(service[key]).expanduser().resolve()) if data.get("data_dir"): data["data_dir"] = str(Path(data["data_dir"]).expanduser().resolve()) if env_file.get("path"): env_file["path"] = str(Path(env_file["path"]).expanduser().resolve()) service["ready_timeout_seconds"] = int(service.get("ready_timeout_seconds", 120)) service["ready_interval_seconds"] = int(service.get("ready_interval_seconds", 2)) return resolved, resolved_path def validate_config(config: dict[str, Any], command: str) -> None: couchdb = config.get("couchdb", {}) data = config.get("data", {}) service = config.get("service", {}) env_file = config.get("env_file", {}) required_service = [ (service.get("ecm_root"), "service.ecm_root"), (service.get("python_path"), "service.python_path"), ] missing_service = [name for value, name in required_service if value in (None, "")] if missing_service: raise ValueError("Missing required config fields: " + ", ".join(missing_service)) if not Path(service["ecm_root"]).exists(): raise FileNotFoundError(f"ECM root does not exist: {service['ecm_root']}") if not Path(service["python_path"]).exists(): raise FileNotFoundError(f"Python path does not exist: {service['python_path']}") if not isinstance(service.get("start_command"), list) or not service.get("start_command"): raise ValueError("service.start_command must be a non-empty list") if not isinstance(service.get("restore_command"), list) or not service.get("restore_command"): raise ValueError("service.restore_command must be a non-empty list") if env_file.get("enabled", True): env_path = Path(env_file.get("path", "")) if not env_path.exists(): raise FileNotFoundError(f"env_file.path does not exist: {env_path}") if command in {"clear", "init"} and couchdb.get("enabled", True): required_couchdb = [ (couchdb.get("base_url"), "couchdb.base_url"), (couchdb.get("database"), "couchdb.database"), ] missing_couchdb = [name for value, name in required_couchdb if value in (None, "")] if missing_couchdb: raise ValueError("Missing required config fields: " + ", ".join(missing_couchdb)) if command == "init": if data.get("data_dir") in (None, ""): raise ValueError("Missing required config fields: data.data_dir") if not Path(data["data_dir"]).exists(): raise FileNotFoundError(f"Data directory does not exist: {data['data_dir']}") def _auth_header(username: str, password: str) -> dict[str, str]: if not username: return {} token = base64.b64encode(f"{username}:{password}".encode("utf-8")).decode("ascii") return {"Authorization": f"Basic {token}"} def _delete_couchdb_database(base_url: str, database: str, username: str = "", password: str = "") -> int: url = f"{base_url.rstrip('/')}/{database}" req = request.Request(url, method="DELETE", headers=_auth_header(username, password)) try: with request.urlopen(req, timeout=10) as response: return response.status except error.HTTPError as exc: if exc.code == 404: return 404 raise def _create_couchdb_database(base_url: str, database: str, username: str = "", password: str = "") -> int: url = f"{base_url.rstrip('/')}/{database}" req = request.Request(url, method="PUT", headers=_auth_header(username, password)) with request.urlopen(req, timeout=10) as response: return response.status def _run_command(command: list[str], *, cwd: str | Path, env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]: return subprocess.run( command, cwd=str(cwd), text=True, capture_output=True, check=False, env=env, ) def _run_or_raise(command: list[str], *, cwd: str | Path, env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]: result = _run_command(command, cwd=cwd, env=env) if result.returncode != 0: raise RuntimeError( "Command failed\n" f"command={command}\n" f"stdout:\n{result.stdout}\n\n" f"stderr:\n{result.stderr}" ) return result def _find_backend_pgids(backend_root_dir: str | Path) -> list[int]: backend_root = str(Path(backend_root_dir).resolve()) result = _run_command(["ps", "-ax", "-o", "pid=,pgid=,command="], cwd=PROJECT_ROOT) if result.returncode != 0: return [] pgids: list[int] = [] for line in result.stdout.splitlines(): raw = line.strip() if not raw or backend_root not in raw or " run.py" not in raw: continue parts = raw.split(None, 2) if len(parts) < 3: continue try: pgid = int(parts[1]) except ValueError: continue if pgid not in pgids: pgids.append(pgid) return pgids def _port_from_ready_url(url: str | None) -> int | None: if not url: return None parsed = urlparse(url) if parsed.port is not None: return int(parsed.port) if parsed.scheme == "https": return 443 if parsed.scheme == "http": return 80 return None def _find_port_listener_pgids(port: int | None) -> list[int]: if port is None: return [] result = _run_command(["lsof", "-nP", f"-iTCP:{port}", "-sTCP:LISTEN", "-t"], cwd=PROJECT_ROOT) if result.returncode not in (0, 1): return [] pgids: list[int] = [] for line in result.stdout.splitlines(): raw = line.strip() if not raw: continue try: pid = int(raw) pgid = os.getpgid(pid) except (ValueError, ProcessLookupError, PermissionError): continue if pgid not in pgids: pgids.append(pgid) return pgids def _find_managed_pgids(service: dict[str, Any]) -> list[int]: pgids: list[int] = [] for pgid in _find_backend_pgids(service["ecm_root"]): if pgid not in pgids: pgids.append(pgid) for pgid in _find_port_listener_pgids(_port_from_ready_url(service.get("ready_url"))): if pgid not in pgids: pgids.append(pgid) return pgids def _kill_process_group(pid: int) -> bool: try: os.killpg(pid, signal.SIGTERM) time.sleep(1) return True except PermissionError: try: os.kill(pid, signal.SIGTERM) time.sleep(1) return True except (PermissionError, ProcessLookupError): return False except ProcessLookupError: return False def _wait_http_ready(url: str, timeout_seconds: int, interval_seconds: int) -> None: deadline = time.time() + timeout_seconds last_error = "" while time.time() < deadline: try: with request.urlopen(url, timeout=5) as response: if 200 <= response.status < 500: return except Exception as exc: # noqa: BLE001 last_error = str(exc) time.sleep(interval_seconds) raise TimeoutError(f"Service not ready: {url}; last_error={last_error}") def _load_state() -> dict[str, Any]: if not STATE_FILE.exists(): return {} try: return json.loads(STATE_FILE.read_text(encoding="utf-8")) except json.JSONDecodeError: return {} def _save_state(state: dict[str, Any]) -> None: STATE_DIR.mkdir(parents=True, exist_ok=True) STATE_FILE.write_text(json.dumps(state, ensure_ascii=False, indent=2), encoding="utf-8") def _clear_state() -> None: STATE_FILE.unlink(missing_ok=True) def _parse_env_line(line: str) -> tuple[str, str] | None: stripped = line.strip() if not stripped or stripped.startswith("#") or "=" not in stripped: return None key, value = stripped.split("=", 1) return key.strip(), value def _read_env_file(path: Path) -> list[str]: if not path.exists(): return [] return path.read_text(encoding="utf-8").splitlines() def _write_env_file(path: Path, lines: list[str]) -> None: _ensure_parent(path) path.write_text("\n".join(lines) + "\n", encoding="utf-8") def _apply_env_overlay(config: dict[str, Any]) -> dict[str, Any]: env_config = config.get("env_file", {}) if not env_config.get("enabled", True): return {"applied": False, "reason": "disabled"} env_path = Path(env_config["path"]) overrides = {str(key): str(value) for key, value in (env_config.get("overrides") or {}).items()} state = _load_state() env_state = state.setdefault("env_file", {}) if not env_state.get("backup_path"): backup_dir = STATE_DIR / "env_backup" backup_dir.mkdir(parents=True, exist_ok=True) backup_path = backup_dir / env_path.name shutil.copy2(env_path, backup_path) env_state["backup_path"] = str(backup_path) env_state["path"] = str(env_path) lines = _read_env_file(env_path) key_to_index: dict[str, int] = {} for index, line in enumerate(lines): parsed = _parse_env_line(line) if parsed is None: continue key_to_index[parsed[0]] = index for key, value in overrides.items(): new_line = f"{key}={value}" if key in key_to_index: lines[key_to_index[key]] = new_line else: lines.append(new_line) _write_env_file(env_path, lines) _save_state(state) return { "applied": True, "path": str(env_path), "overrides": sorted(overrides.keys()), "backup_path": env_state["backup_path"], } def _restore_env_overlay(config: dict[str, Any]) -> dict[str, Any]: env_config = config.get("env_file", {}) if not env_config.get("enabled", True) or not env_config.get("restore_on_stop", True): return {"restored": False, "reason": "disabled"} state = _load_state() env_state = state.get("env_file") or {} backup_path = env_state.get("backup_path") env_path = env_state.get("path") or env_config.get("path") if not backup_path or not env_path: return {"restored": False, "reason": "no_backup"} backup = Path(backup_path) target = Path(env_path) if not backup.exists(): state.pop("env_file", None) _save_state(state) return {"restored": False, "reason": "backup_missing"} shutil.copy2(backup, target) backup.unlink(missing_ok=True) state.pop("env_file", None) if state: _save_state(state) else: _clear_state() return {"restored": True, "path": str(target)} def _build_process_env(config: dict[str, Any]) -> dict[str, str]: env = os.environ.copy() env.update({key: str(value) for key, value in (config.get("service", {}).get("env") or {}).items()}) return env def _state_pid() -> int | None: state = _load_state() pid = state.get("pid") return int(pid) if isinstance(pid, int) else None def _set_state_pid(pid: int | None) -> None: state = _load_state() if pid is None: state.pop("pid", None) else: state["pid"] = pid if state: _save_state(state) else: _clear_state() def clear_database(config: dict[str, Any]) -> dict[str, Any]: couchdb = config["couchdb"] if not couchdb.get("enabled", True): return {"enabled": False} delete_status = _delete_couchdb_database( couchdb["base_url"], couchdb["database"], couchdb.get("username", ""), couchdb.get("password", ""), ) create_status = None if couchdb.get("recreate_after_delete", False): create_status = _create_couchdb_database( couchdb["base_url"], couchdb["database"], couchdb.get("username", ""), couchdb.get("password", ""), ) return { "database": couchdb["database"], "delete_status": delete_status, "create_status": create_status, } def import_init_data(config: dict[str, Any]) -> dict[str, Any]: service = config["service"] result = _run_or_raise(service["restore_command"], cwd=service["cwd"], env=_build_process_env(config)) return { "data_dir": config["data"]["data_dir"], "restore_command": service["restore_command"], "stdout_tail": result.stdout[-2000:], "stderr_tail": result.stderr[-2000:], } def stop_service(config: dict[str, Any], *, restore_env: bool = True) -> dict[str, Any]: service = config["service"] killed: list[int] = [] pid = _state_pid() if pid is not None and _kill_process_group(pid): killed.append(pid) for pgid in _find_managed_pgids(service): if pgid in killed: continue if _kill_process_group(pgid): killed.append(pgid) _set_state_pid(None) env_restore = _restore_env_overlay(config) if restore_env else {"restored": False, "reason": "skipped"} return { "stopped": bool(killed), "killed_pgids": killed, "env_file": env_restore, } def run_service(config: dict[str, Any]) -> dict[str, Any]: service = config["service"] existing_pgids = _find_managed_pgids(service) if existing_pgids: return { "started": False, "reason": "already_running", "pgids": existing_pgids, "log_file": service["log_file"], } env_result = _apply_env_overlay(config) log_path = _ensure_parent(service["log_file"]) handle = open(log_path, "a", encoding="utf-8") process = subprocess.Popen( service["start_command"], cwd=service["cwd"], stdout=handle, stderr=subprocess.STDOUT, text=True, start_new_session=True, env=_build_process_env(config), ) _set_state_pid(process.pid) _wait_http_ready( str(service["ready_url"]), int(service.get("ready_timeout_seconds", 120)), int(service.get("ready_interval_seconds", 2)), ) return { "started": True, "pid": process.pid, "log_file": str(log_path), "cwd": service["cwd"], "command": service["start_command"], "ready_url": service["ready_url"], "env_file": env_result, } def do_clear(config: dict[str, Any]) -> dict[str, Any]: stop_result = stop_service(config, restore_env=True) clear_result = clear_database(config) return { "command": "clear", "stop": stop_result, "database": clear_result, } def do_init(config: dict[str, Any]) -> dict[str, Any]: stop_result = stop_service(config, restore_env=False) env_result = _apply_env_overlay(config) clear_result = clear_database(config) init_result = import_init_data(config) return { "command": "init", "stop": stop_result, "env_file": env_result, "database": clear_result, "init": init_result, } def do_run(config: dict[str, Any]) -> dict[str, Any]: return { "command": "run", "service": run_service(config), } def do_stop(config: dict[str, Any]) -> dict[str, Any]: return { "command": "stop", "service": stop_service(config, restore_env=True), } def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="远程 ECM 测试环境管理工具") parser.add_argument( "command", choices=["clear", "init", "run", "stop"], help="clear: 清空 CouchDB 测试库;init: 通过 run.py restore-database 恢复数据;run: 后台启动远程服务;stop: 停止远程服务并恢复 .env", ) parser.add_argument("--config", default="tools/remote_env.example.yaml", help="YAML 配置文件路径") parser.add_argument("--db-name", help="覆盖 couchdb.database") parser.add_argument("--db-user", help="覆盖 couchdb.username") parser.add_argument("--db-password", help="覆盖 couchdb.password") parser.add_argument("--couchdb-base-url", help="覆盖 couchdb.base_url") parser.add_argument("--couchdb-url", help="覆盖 couchdb.url") parser.add_argument("--couchdb-port", help="覆盖 couchdb.port") parser.add_argument("--ecm-root", help="覆盖 service.ecm_root") parser.add_argument("--python-path", help="覆盖 service.python_path") parser.add_argument("--data-dir", help="覆盖 data.data_dir") parser.add_argument("--ready-url", help="覆盖 service.ready_url") return parser def main(argv: list[str] | None = None) -> int: parser = build_parser() args = parser.parse_args(argv) config, config_path = load_config(args.config, args) validate_config(config, args.command) if args.command == "clear": result = do_clear(config) elif args.command == "init": result = do_init(config) elif args.command == "run": result = do_run(config) elif args.command == "stop": result = do_stop(config) else: raise AssertionError(f"Unhandled command: {args.command}") result["config_path"] = str(config_path) result["state_file"] = str(STATE_FILE) print(json.dumps(result, ensure_ascii=False, indent=2)) return 0 if __name__ == "__main__": try: raise SystemExit(main()) except Exception as exc: # noqa: BLE001 print(json.dumps({"error": str(exc)}, ensure_ascii=False, indent=2), file=sys.stderr) raise SystemExit(1)