strepsiades
232 lines
8.3 KiB
Python
232 lines
8.3 KiB
Python
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
from typing import Any, Dict, Mapping, Optional, Union
|
|
|
|
from pydantic import BaseModel
|
|
|
|
from ..common.registry import DomainRegistry
|
|
from .datasource_config import DataSourceConfig
|
|
from .pipeline_config import PipelineRunConfig
|
|
from .run_presets import load_overrides_from_file
|
|
from .strategy_config import ConfigPresets, StrategyRuntimeConfig, resolve_domain_option_config
|
|
|
|
|
|
def _deep_merge(base: Mapping[str, Any], override: Mapping[str, Any]) -> Dict[str, Any]:
|
|
merged: Dict[str, Any] = dict(base)
|
|
for key, value in override.items():
|
|
if isinstance(value, dict) and isinstance(merged.get(key), dict):
|
|
if key in {"local_datasource", "remote_datasource"}:
|
|
current_type = merged[key].get("type")
|
|
next_type = value.get("type", current_type)
|
|
if current_type != next_type:
|
|
merged[key] = dict(value)
|
|
continue
|
|
merged[key] = _deep_merge(merged[key], value)
|
|
else:
|
|
merged[key] = value
|
|
return merged
|
|
|
|
|
|
def _build_default_strategy_runtime_config(node_type: str) -> StrategyRuntimeConfig | None:
|
|
strategy_class = DomainRegistry.get_strategy_class(node_type)
|
|
if strategy_class is None:
|
|
return None
|
|
|
|
config = strategy_class.default_config.model_copy(deep=True)
|
|
config.domain_option = resolve_domain_option_config(
|
|
getattr(strategy_class, "domain_option_model", None),
|
|
config.domain_option,
|
|
)
|
|
|
|
return StrategyRuntimeConfig(
|
|
node_type=node_type,
|
|
config=config,
|
|
)
|
|
|
|
|
|
def _default_strategy_map(node_types: list[str]) -> Dict[str, StrategyRuntimeConfig]:
|
|
strategy_map: Dict[str, StrategyRuntimeConfig] = {}
|
|
for node_type in node_types:
|
|
runtime_cfg = _build_default_strategy_runtime_config(node_type)
|
|
if runtime_cfg is None:
|
|
continue
|
|
strategy_map[node_type] = runtime_cfg
|
|
return strategy_map
|
|
|
|
|
|
def _merge_strategy_runtime_config(
|
|
runtime_cfg: StrategyRuntimeConfig,
|
|
raw_override: Any,
|
|
) -> StrategyRuntimeConfig:
|
|
override = dict(raw_override or {})
|
|
override.pop("node_type", None)
|
|
|
|
deprecated_keys = [key for key in ("force_sync", "load_mode") if key in override]
|
|
if deprecated_keys:
|
|
joined = ", ".join(deprecated_keys)
|
|
raise ValueError(
|
|
f"Strategy override for {runtime_cfg.node_type!r} contains unsupported legacy keys: {joined}. "
|
|
"Use skip_sync under strategies.<node_type> or datasource.handler_configs instead."
|
|
)
|
|
|
|
preset = override.pop("preset", None)
|
|
if preset:
|
|
runtime_cfg.config = runtime_cfg.config.model_validate(
|
|
_deep_merge(
|
|
runtime_cfg.config.model_dump(),
|
|
ConfigPresets.get_preset(str(preset)).model_dump(exclude_defaults=True),
|
|
)
|
|
)
|
|
|
|
if "config" in override and isinstance(override["config"], dict):
|
|
runtime_cfg.config = runtime_cfg.config.model_validate(
|
|
_deep_merge(runtime_cfg.config.model_dump(), override.pop("config"))
|
|
)
|
|
|
|
if override:
|
|
runtime_cfg.config = runtime_cfg.config.model_validate(
|
|
_deep_merge(runtime_cfg.config.model_dump(), override)
|
|
)
|
|
|
|
strategy_class = DomainRegistry.get_strategy_class(runtime_cfg.node_type)
|
|
runtime_cfg.config.domain_option = resolve_domain_option_config(
|
|
getattr(strategy_class, "domain_option_model", None) if strategy_class is not None else None,
|
|
runtime_cfg.config.domain_option,
|
|
)
|
|
|
|
return runtime_cfg
|
|
|
|
|
|
def _apply_strategy_overrides(
|
|
node_types: list[str],
|
|
strategy_overrides: Any,
|
|
) -> list[StrategyRuntimeConfig]:
|
|
strategy_map = _default_strategy_map(node_types)
|
|
|
|
if isinstance(strategy_overrides, list):
|
|
for item in strategy_overrides:
|
|
raw_item = dict(item or {})
|
|
node_type = str(raw_item.get("node_type") or "")
|
|
if not node_type:
|
|
raise ValueError("Strategy override list item missing required node_type")
|
|
|
|
runtime_cfg = strategy_map.get(node_type)
|
|
if runtime_cfg is None:
|
|
runtime_cfg = _build_default_strategy_runtime_config(node_type) or StrategyRuntimeConfig(node_type=node_type)
|
|
|
|
strategy_map[node_type] = _merge_strategy_runtime_config(runtime_cfg, raw_item)
|
|
return [strategy_map[n] for n in node_types if n in strategy_map]
|
|
|
|
if not isinstance(strategy_overrides, dict):
|
|
return [strategy_map[n] for n in node_types if n in strategy_map]
|
|
|
|
for node_type, raw_override in strategy_overrides.items():
|
|
if node_type not in strategy_map:
|
|
strategy_map[node_type] = _build_default_strategy_runtime_config(node_type) or StrategyRuntimeConfig(node_type=node_type)
|
|
|
|
runtime_cfg = strategy_map[node_type]
|
|
strategy_map[node_type] = _merge_strategy_runtime_config(runtime_cfg, raw_override)
|
|
|
|
return [strategy_map[n] for n in node_types if n in strategy_map]
|
|
|
|
|
|
def config_snapshot(config: PipelineRunConfig) -> Dict[str, Any]:
|
|
snap = config.model_dump(mode="json")
|
|
for key in ("local_datasource", "remote_datasource"):
|
|
ds = snap.get(key) or {}
|
|
if ds.get("api_secret"):
|
|
ds["api_secret"] = "***"
|
|
return snap
|
|
|
|
|
|
def build_default_config(project_root: Path) -> PipelineRunConfig:
|
|
defaults = {
|
|
"local_datasource": {
|
|
"type": "jsonl",
|
|
"jsonl_dir": str(project_root / "filtered_datasource" / "datasource" / "ecm-2025-12-02"),
|
|
"read_only": False,
|
|
"handler_configs": {},
|
|
},
|
|
"remote_datasource": {
|
|
"type": "jsonl",
|
|
"jsonl_dir": str(project_root / "remote_datasource" / "datasource"),
|
|
"read_only": False,
|
|
"handler_configs": {},
|
|
},
|
|
"persist": {
|
|
"db_path": str(project_root / "test_results" / "sync_state_machine" / "pipeline_default.db"),
|
|
"enable": True,
|
|
"wipe_on_start": False,
|
|
},
|
|
"logging": {
|
|
"initialize": True,
|
|
"file_path": None,
|
|
"level": 20,
|
|
"suppress_node_logs": True,
|
|
},
|
|
}
|
|
|
|
config = PipelineRunConfig.model_validate(defaults)
|
|
config.strategies = _apply_strategy_overrides(config.node_types, {})
|
|
return config
|
|
|
|
|
|
def apply_config_overrides(config: PipelineRunConfig, overrides: Dict[str, Any]) -> PipelineRunConfig:
|
|
merged = _deep_merge(config.model_dump(), overrides)
|
|
raw_strategies = merged.pop("strategies", None)
|
|
|
|
resolved = PipelineRunConfig.model_validate(merged)
|
|
|
|
if raw_strategies is not None:
|
|
resolved.strategies = _apply_strategy_overrides(resolved.node_types, raw_strategies)
|
|
else:
|
|
resolved.strategies = _apply_strategy_overrides(resolved.node_types, {})
|
|
|
|
return resolved
|
|
|
|
|
|
def build_config(
|
|
*,
|
|
project_root: Path,
|
|
local_datasource: Optional[Union[DataSourceConfig, Dict[str, Any]]] = None,
|
|
remote_datasource: Optional[Union[DataSourceConfig, Dict[str, Any]]] = None,
|
|
overrides: Optional[Dict[str, Any]] = None,
|
|
overrides_file: Optional[Union[str, Path]] = None,
|
|
dotenv_file: Optional[Union[str, Path]] = None,
|
|
) -> PipelineRunConfig:
|
|
if dotenv_file is not None:
|
|
raise ValueError("dotenv_file is no longer supported; use overrides_file instead")
|
|
|
|
config = build_default_config(project_root)
|
|
|
|
if local_datasource is not None:
|
|
local_payload = (
|
|
local_datasource.model_dump() if isinstance(local_datasource, BaseModel) else dict(local_datasource)
|
|
)
|
|
config = apply_config_overrides(config, {"local_datasource": local_payload})
|
|
|
|
if remote_datasource is not None:
|
|
remote_payload = (
|
|
remote_datasource.model_dump() if isinstance(remote_datasource, BaseModel) else dict(remote_datasource)
|
|
)
|
|
config = apply_config_overrides(config, {"remote_datasource": remote_payload})
|
|
|
|
if overrides_file:
|
|
override_path = Path(overrides_file)
|
|
file_data = load_overrides_from_file(override_path, project_root)
|
|
config = apply_config_overrides(config, file_data)
|
|
|
|
if overrides:
|
|
config = apply_config_overrides(config, overrides)
|
|
|
|
return config
|
|
|
|
|
|
def build_config_from_file(
|
|
*,
|
|
project_root: Path,
|
|
file_path: Union[str, Path],
|
|
) -> PipelineRunConfig:
|
|
return build_config(project_root=project_root, overrides_file=Path(file_path))
|