strepsiades
545 lines
17 KiB
Python
545 lines
17 KiB
Python
from __future__ import annotations
|
|
|
|
import base64
|
|
import json
|
|
import os
|
|
import shutil
|
|
import signal
|
|
import sqlite3
|
|
import subprocess
|
|
import time
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from urllib import error, request
|
|
|
|
import yaml
|
|
|
|
from sync_state_machine.common.persistence import PersistenceBackend
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def replace_placeholders(value: Any) -> Any:
|
|
if isinstance(value, str):
|
|
return value.replace("${PROJECT_ROOT}", str(PROJECT_ROOT))
|
|
if isinstance(value, list):
|
|
return [replace_placeholders(item) for item in value]
|
|
if isinstance(value, dict):
|
|
return {key: replace_placeholders(item) for key, item in value.items()}
|
|
return value
|
|
|
|
|
|
def load_yaml_config(path: str | Path) -> dict[str, Any]:
|
|
raw_path = Path(path)
|
|
if not raw_path.is_absolute():
|
|
raw_path = PROJECT_ROOT / raw_path
|
|
payload = yaml.safe_load(raw_path.read_text(encoding="utf-8")) or {}
|
|
return replace_placeholders(payload)
|
|
|
|
|
|
def ensure_parent(path: str | Path) -> Path:
|
|
resolved = Path(path)
|
|
resolved.parent.mkdir(parents=True, exist_ok=True)
|
|
return resolved
|
|
|
|
|
|
def remove_path(path: str | Path) -> None:
|
|
target = Path(path)
|
|
if not target.exists():
|
|
return
|
|
if target.is_dir():
|
|
shutil.rmtree(target, ignore_errors=True)
|
|
else:
|
|
try:
|
|
target.unlink()
|
|
except FileNotFoundError:
|
|
pass
|
|
|
|
|
|
def recreate_dir(path: str | Path) -> Path:
|
|
target = Path(path)
|
|
remove_path(target)
|
|
target.mkdir(parents=True, exist_ok=True)
|
|
return target
|
|
|
|
|
|
def copy_tree_contents(src: str | Path, dst: str | Path) -> None:
|
|
src_path = Path(src)
|
|
dst_path = recreate_dir(dst)
|
|
for child in src_path.iterdir():
|
|
target = dst_path / child.name
|
|
if child.is_dir():
|
|
shutil.copytree(child, target)
|
|
else:
|
|
shutil.copy2(child, target)
|
|
|
|
|
|
def copy_selected_files(src: str | Path, dst: str | Path, file_names: list[str]) -> None:
|
|
src_path = Path(src)
|
|
dst_path = recreate_dir(dst)
|
|
for file_name in file_names:
|
|
source = src_path / file_name
|
|
if not source.exists():
|
|
raise FileNotFoundError(f"基础数据文件不存在: {source}")
|
|
shutil.copy2(source, dst_path / file_name)
|
|
|
|
|
|
def _jsonl_record_id(item: dict[str, Any]) -> str:
|
|
record_id = item.get("_id") or item.get("id")
|
|
if not record_id:
|
|
raise ValueError(f"JSONL 记录缺少 _id/id: {item}")
|
|
return str(record_id)
|
|
|
|
|
|
def merge_jsonl_file(src_file: str | Path, dst_file: str | Path) -> None:
|
|
src_path = Path(src_file)
|
|
dst_path = Path(dst_file)
|
|
|
|
merged: dict[str, dict[str, Any]] = {}
|
|
ordered_ids: list[str] = []
|
|
|
|
if dst_path.exists():
|
|
for line in dst_path.read_text(encoding="utf-8").splitlines():
|
|
raw = line.strip()
|
|
if not raw:
|
|
continue
|
|
item = json.loads(raw)
|
|
record_id = _jsonl_record_id(item)
|
|
merged[record_id] = item
|
|
ordered_ids.append(record_id)
|
|
|
|
for line in src_path.read_text(encoding="utf-8").splitlines():
|
|
raw = line.strip()
|
|
if not raw:
|
|
continue
|
|
item = json.loads(raw)
|
|
record_id = _jsonl_record_id(item)
|
|
if record_id in merged:
|
|
merged[record_id] = {**merged[record_id], **item}
|
|
else:
|
|
merged[record_id] = item
|
|
ordered_ids.append(record_id)
|
|
|
|
lines = [json.dumps(merged[record_id], ensure_ascii=False) for record_id in ordered_ids]
|
|
dst_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
|
|
|
|
def merge_tree_contents(src: str | Path, dst: str | Path) -> None:
|
|
src_path = Path(src)
|
|
dst_path = Path(dst)
|
|
dst_path.mkdir(parents=True, exist_ok=True)
|
|
for child in src_path.iterdir():
|
|
target = dst_path / child.name
|
|
if child.is_dir():
|
|
if target.exists():
|
|
shutil.rmtree(target)
|
|
shutil.copytree(child, target)
|
|
else:
|
|
if child.suffix == ".jsonl" and target.exists():
|
|
merge_jsonl_file(child, target)
|
|
else:
|
|
shutil.copy2(child, target)
|
|
|
|
|
|
def run_command(
|
|
command: list[str],
|
|
cwd: str | Path,
|
|
*,
|
|
capture_output: bool = True,
|
|
extra_env: dict[str, str] | None = None,
|
|
) -> subprocess.CompletedProcess[str]:
|
|
env = os.environ.copy()
|
|
if extra_env:
|
|
env.update({key: str(value) for key, value in extra_env.items()})
|
|
return subprocess.run(
|
|
command,
|
|
cwd=str(cwd),
|
|
text=True,
|
|
capture_output=capture_output,
|
|
check=False,
|
|
env=env,
|
|
)
|
|
|
|
|
|
def start_background_process(
|
|
command: list[str],
|
|
cwd: str | Path,
|
|
log_file: str | Path,
|
|
*,
|
|
extra_env: dict[str, str] | None = None,
|
|
) -> subprocess.Popen[str]:
|
|
log_path = ensure_parent(log_file)
|
|
handle = open(log_path, "a", encoding="utf-8")
|
|
env = os.environ.copy()
|
|
if extra_env:
|
|
env.update({key: str(value) for key, value in extra_env.items()})
|
|
return subprocess.Popen(
|
|
command,
|
|
cwd=str(cwd),
|
|
stdout=handle,
|
|
stderr=subprocess.STDOUT,
|
|
text=True,
|
|
start_new_session=True,
|
|
env=env,
|
|
)
|
|
|
|
|
|
def stop_process_from_pid_file(pid_file: str | Path) -> bool:
|
|
path = Path(pid_file)
|
|
if not path.exists():
|
|
return False
|
|
try:
|
|
pid = int(path.read_text(encoding="utf-8").strip())
|
|
except ValueError:
|
|
path.unlink(missing_ok=True)
|
|
return False
|
|
|
|
try:
|
|
os.killpg(pid, signal.SIGTERM)
|
|
time.sleep(1)
|
|
except PermissionError:
|
|
try:
|
|
os.kill(pid, signal.SIGTERM)
|
|
time.sleep(1)
|
|
except (PermissionError, ProcessLookupError):
|
|
pass
|
|
except ProcessLookupError:
|
|
pass
|
|
finally:
|
|
path.unlink(missing_ok=True)
|
|
return True
|
|
|
|
|
|
def stop_backend_processes(backend_root_dir: str | Path) -> list[int]:
|
|
backend_root = str(Path(backend_root_dir).resolve())
|
|
result = run_command(
|
|
["ps", "-ax", "-o", "pid=,pgid=,command="],
|
|
cwd=PROJECT_ROOT,
|
|
)
|
|
if result.returncode != 0:
|
|
return []
|
|
|
|
killed_pgids: list[int] = []
|
|
for line in result.stdout.splitlines():
|
|
raw = line.strip()
|
|
if not raw or backend_root not in raw or " run.py" not in raw:
|
|
continue
|
|
parts = raw.split(None, 2)
|
|
if len(parts) < 3:
|
|
continue
|
|
try:
|
|
pgid = int(parts[1])
|
|
except ValueError:
|
|
continue
|
|
if pgid in killed_pgids:
|
|
continue
|
|
try:
|
|
os.killpg(pgid, signal.SIGTERM)
|
|
time.sleep(1)
|
|
except ProcessLookupError:
|
|
pass
|
|
killed_pgids.append(pgid)
|
|
return killed_pgids
|
|
|
|
|
|
def wait_http_ready(url: str, timeout_seconds: int, interval_seconds: int) -> None:
|
|
deadline = time.time() + timeout_seconds
|
|
last_error = ""
|
|
while time.time() < deadline:
|
|
try:
|
|
with request.urlopen(url, timeout=5) as response:
|
|
if 200 <= response.status < 500:
|
|
return
|
|
except Exception as exc: # noqa: BLE001
|
|
last_error = str(exc)
|
|
time.sleep(interval_seconds)
|
|
raise TimeoutError(f"Backend not ready: {url}; last_error={last_error}")
|
|
|
|
|
|
def _auth_header(username: str, password: str) -> dict[str, str]:
|
|
if not username:
|
|
return {}
|
|
token = base64.b64encode(f"{username}:{password}".encode("utf-8")).decode("ascii")
|
|
return {"Authorization": f"Basic {token}"}
|
|
|
|
|
|
def delete_couchdb_database(base_url: str, database: str, username: str = "", password: str = "") -> int:
|
|
url = f"{base_url.rstrip('/')}/{database}"
|
|
req = request.Request(url, method="DELETE", headers=_auth_header(username, password))
|
|
try:
|
|
with request.urlopen(req, timeout=10) as response:
|
|
return response.status
|
|
except error.HTTPError as exc:
|
|
if exc.code == 404:
|
|
return 404
|
|
raise
|
|
|
|
|
|
def create_couchdb_database(base_url: str, database: str, username: str = "", password: str = "") -> int:
|
|
url = f"{base_url.rstrip('/')}/{database}"
|
|
req = request.Request(url, method="PUT", headers=_auth_header(username, password))
|
|
with request.urlopen(req, timeout=10) as response:
|
|
return response.status
|
|
|
|
|
|
def latest_log_file(log_dir: str | Path) -> str | None:
|
|
path = Path(log_dir)
|
|
if not path.exists():
|
|
return None
|
|
files = [item for item in path.iterdir() if item.is_file() and item.suffix == ".log"]
|
|
if not files:
|
|
return None
|
|
files.sort(key=lambda item: item.stat().st_mtime, reverse=True)
|
|
return str(files[0])
|
|
|
|
|
|
def tail_text_file(file_path: str | Path, max_chars: int = 4000) -> str:
|
|
path = Path(file_path)
|
|
if not path.exists() or not path.is_file():
|
|
return ""
|
|
content = path.read_text(encoding="utf-8", errors="replace")
|
|
if len(content) <= max_chars:
|
|
return content
|
|
return content[-max_chars:]
|
|
|
|
|
|
def sqlite_rows(db_path: str | Path, sql: str, params: tuple[Any, ...] = ()) -> list[dict[str, Any]]:
|
|
path = Path(db_path)
|
|
if not path.exists():
|
|
return []
|
|
with sqlite3.connect(path) as conn:
|
|
conn.row_factory = sqlite3.Row
|
|
rows = conn.execute(sql, params).fetchall()
|
|
return [{key: row[key] for key in row.keys()} for row in rows]
|
|
|
|
|
|
def persistence_rows(
|
|
*,
|
|
backend: str,
|
|
sql: str,
|
|
db_path: str | Path | None = None,
|
|
url: str | None = None,
|
|
limit: int = 200,
|
|
) -> list[dict[str, Any]]:
|
|
_, rows = PersistenceBackend.query_records(
|
|
backend=backend,
|
|
db_path=str(db_path) if db_path is not None else None,
|
|
url=url,
|
|
sql=sql,
|
|
limit=limit,
|
|
)
|
|
return rows
|
|
|
|
|
|
def persistence_summary(
|
|
db_path: str | Path | None,
|
|
node_type: str | None = None,
|
|
*,
|
|
backend: str = "sqlite",
|
|
url: str | None = None,
|
|
) -> dict[str, Any]:
|
|
records = PersistenceBackend.records_summary(
|
|
backend=backend,
|
|
db_path=str(db_path) if db_path is not None else None,
|
|
url=url,
|
|
)
|
|
summary: dict[str, Any] = {
|
|
"backend": backend,
|
|
"db_path": str(db_path),
|
|
"url": url,
|
|
"db_exists": records.get("db_exists", False),
|
|
"bindings": [],
|
|
"node_status": [],
|
|
}
|
|
if not summary["db_exists"]:
|
|
return summary
|
|
|
|
target_db_path = str(db_path) if db_path is not None else None
|
|
if node_type:
|
|
escaped_node_type = node_type.replace("'", "''")
|
|
summary["bindings"] = persistence_rows(
|
|
backend=backend,
|
|
db_path=target_db_path,
|
|
url=url,
|
|
sql=(
|
|
"SELECT node_type, local_id, remote_id, last_updated "
|
|
f"FROM bindings WHERE node_type = '{escaped_node_type}' ORDER BY last_updated DESC LIMIT 20"
|
|
),
|
|
)
|
|
summary["node_status"] = persistence_rows(
|
|
backend=backend,
|
|
db_path=target_db_path,
|
|
url=url,
|
|
sql=(
|
|
"SELECT collection_id, node_type, action, status, binding_status, COUNT(1) AS count "
|
|
f"FROM nodes WHERE node_type = '{escaped_node_type}' "
|
|
"GROUP BY collection_id, node_type, action, status, binding_status "
|
|
"ORDER BY collection_id, action, status"
|
|
),
|
|
)
|
|
return summary
|
|
|
|
|
|
def read_jsonl_records(file_path: str | Path) -> dict[str, dict[str, Any]]:
|
|
records: dict[str, dict[str, Any]] = {}
|
|
path = Path(file_path)
|
|
if not path.exists():
|
|
return records
|
|
for line in path.read_text(encoding="utf-8").splitlines():
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
item = json.loads(line)
|
|
records[str(item.get("_id") or item.get("id"))] = item
|
|
return records
|
|
|
|
|
|
def diff_record_fields(before: dict[str, Any], after: dict[str, Any]) -> list[str]:
|
|
keys = set(before) | set(after)
|
|
ignored = {"_rev", "updated_at", "updated_by", "updated_by_name"}
|
|
changed = [key for key in sorted(keys) if key not in ignored and before.get(key) != after.get(key)]
|
|
return changed
|
|
|
|
|
|
def now_slug() -> str:
|
|
return datetime.now().strftime("%Y%m%d-%H%M%S")
|
|
|
|
|
|
def enrich_supplier_display_fields(data_dir: str | Path, region_file: str | Path) -> None:
|
|
data_dir_path = Path(data_dir)
|
|
supplier_file = data_dir_path / "supplier_1.jsonl"
|
|
region_path = Path(region_file)
|
|
if not supplier_file.exists() or not region_path.exists():
|
|
return
|
|
|
|
country_name_map: dict[str, str] = {}
|
|
province_name_map: dict[str, str] = {}
|
|
city_name_map: dict[str, str] = {}
|
|
for line in region_path.read_text(encoding="utf-8").splitlines():
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
item = json.loads(line)
|
|
region_name = str(item.get("name") or "")
|
|
region_code = str(item.get("code") or "")
|
|
region_type = str(item.get("region_type") or "")
|
|
if not region_code:
|
|
continue
|
|
if region_type == "country":
|
|
country_name_map[region_code] = region_name
|
|
elif region_type == "province":
|
|
province_name_map[region_code] = region_name
|
|
elif region_type == "city":
|
|
city_name_map[region_code] = region_name
|
|
|
|
normalized_lines: list[str] = []
|
|
changed = False
|
|
for line in supplier_file.read_text(encoding="utf-8").splitlines():
|
|
raw = line.strip()
|
|
if not raw:
|
|
continue
|
|
item = json.loads(raw)
|
|
country = item.get("country")
|
|
province = item.get("province")
|
|
city = item.get("city")
|
|
expected_country_name = country_name_map.get(str(country), "") if country else None
|
|
expected_province_name = province_name_map.get(str(province), "") if province else None
|
|
expected_city_name = city_name_map.get(str(city), "") if city else None
|
|
|
|
if item.get("country_name") != expected_country_name:
|
|
item["country_name"] = expected_country_name
|
|
changed = True
|
|
if item.get("province_name") != expected_province_name:
|
|
item["province_name"] = expected_province_name
|
|
changed = True
|
|
if item.get("city_name") != expected_city_name:
|
|
item["city_name"] = expected_city_name
|
|
changed = True
|
|
|
|
normalized_lines.append(json.dumps(item, ensure_ascii=False))
|
|
|
|
if changed:
|
|
supplier_file.write_text("\n".join(normalized_lines) + "\n", encoding="utf-8")
|
|
|
|
|
|
def normalize_construction_task_plan_nodes(data_dir: str | Path) -> None:
|
|
task_file = Path(data_dir) / "construction_task_1.jsonl"
|
|
if not task_file.exists():
|
|
return
|
|
|
|
normalized_lines: list[str] = []
|
|
changed = False
|
|
for line in task_file.read_text(encoding="utf-8").splitlines():
|
|
raw = line.strip()
|
|
if not raw:
|
|
continue
|
|
item = json.loads(raw)
|
|
plan_node = item.get("plan_node")
|
|
if isinstance(plan_node, list) and len(plan_node) >= 2:
|
|
trimmed = list(plan_node)
|
|
if trimmed and float(trimmed[0].get("quantity") or 0) == 0:
|
|
trimmed = trimmed[1:]
|
|
if trimmed and item.get("contract_quantity") is not None:
|
|
end_quantity = float(trimmed[-1].get("quantity") or 0)
|
|
contract_quantity = float(item.get("contract_quantity") or 0)
|
|
if end_quantity == contract_quantity:
|
|
trimmed = trimmed[:-1]
|
|
if trimmed != plan_node:
|
|
item["plan_node"] = trimmed
|
|
changed = True
|
|
if not trimmed:
|
|
for field_name in ["plan_start_date", "plan_complete_date", "contract_quantity"]:
|
|
if item.get(field_name) is not None:
|
|
item[field_name] = None
|
|
changed = True
|
|
normalized_lines.append(json.dumps(item, ensure_ascii=False))
|
|
|
|
if changed:
|
|
task_file.write_text("\n".join(normalized_lines) + "\n", encoding="utf-8")
|
|
|
|
|
|
def normalize_contract_settlement_records(data_dir: str | Path) -> None:
|
|
settlement_file = Path(data_dir) / "contract_settlement_1.jsonl"
|
|
if not settlement_file.exists():
|
|
return
|
|
|
|
excluded_ids = {
|
|
"544430cd-1301-43e0-8f7e-a471d9438983",
|
|
}
|
|
|
|
normalized_lines: list[str] = []
|
|
changed = False
|
|
for line in settlement_file.read_text(encoding="utf-8").splitlines():
|
|
raw = line.strip()
|
|
if not raw:
|
|
continue
|
|
item = json.loads(raw)
|
|
|
|
if str(item.get("_id") or item.get("id")) in excluded_ids:
|
|
changed = True
|
|
continue
|
|
|
|
if item.get("contract_id") == "1351be9d-3ca3-48f1-be29-fb39b2c3ff15" and item.get("show_name") == "施工总承包合同":
|
|
item["show_name"] = "施工总承包合同(不含设计、采购)"
|
|
changed = True
|
|
|
|
plan_node = item.get("plan_node")
|
|
if isinstance(plan_node, list) and plan_node:
|
|
normalized_plan_node: list[dict[str, Any]] = []
|
|
plan_changed = False
|
|
for node in plan_node:
|
|
if isinstance(node, dict) and node.get("is_audit") is False:
|
|
normalized_plan_node.append({**node, "is_audit": True})
|
|
plan_changed = True
|
|
else:
|
|
normalized_plan_node.append(node)
|
|
if plan_changed:
|
|
item["plan_node"] = normalized_plan_node
|
|
changed = True
|
|
|
|
normalized_lines.append(json.dumps(item, ensure_ascii=False))
|
|
|
|
if changed:
|
|
settlement_file.write_text("\n".join(normalized_lines) + "\n", encoding="utf-8")
|