from __future__ import annotations from pathlib import Path from typing import Any, Dict, Mapping, Optional, Union from pydantic import BaseModel from ..common.registry import DomainRegistry from .datasource_config import DataSourceConfig from .pipeline_config import PipelineRunConfig from .run_presets import load_overrides_from_file from .strategy_config import ConfigPresets, StrategyRuntimeConfig def _deep_merge(base: Mapping[str, Any], override: Mapping[str, Any]) -> Dict[str, Any]: merged: Dict[str, Any] = dict(base) for key, value in override.items(): if isinstance(value, dict) and isinstance(merged.get(key), dict): if key in {"local_datasource", "remote_datasource"}: current_type = merged[key].get("type") next_type = value.get("type", current_type) if current_type != next_type: merged[key] = dict(value) continue merged[key] = _deep_merge(merged[key], value) else: merged[key] = value return merged def _default_strategy_map(node_types: list[str]) -> Dict[str, StrategyRuntimeConfig]: strategy_map: Dict[str, StrategyRuntimeConfig] = {} for node_type in node_types: strategy_class = DomainRegistry.get_strategy_class(node_type) if strategy_class is None: continue strategy_map[node_type] = StrategyRuntimeConfig( node_type=node_type, config=strategy_class.default_config.model_copy(deep=True), ) return strategy_map def _apply_strategy_overrides( node_types: list[str], strategy_overrides: Any, ) -> list[StrategyRuntimeConfig]: strategy_map = _default_strategy_map(node_types) if isinstance(strategy_overrides, list): for item in strategy_overrides: runtime_cfg = StrategyRuntimeConfig.model_validate(item) strategy_map[runtime_cfg.node_type] = runtime_cfg return [strategy_map[n] for n in node_types if n in strategy_map] if not isinstance(strategy_overrides, dict): return [strategy_map[n] for n in node_types if n in strategy_map] for node_type, raw_override in strategy_overrides.items(): if node_type not in strategy_map: strategy_map[node_type] = StrategyRuntimeConfig(node_type=node_type) runtime_cfg = strategy_map[node_type] override = dict(raw_override or {}) deprecated_keys = [key for key in ("force_sync", "load_mode") if key in override] if deprecated_keys: joined = ", ".join(deprecated_keys) raise ValueError( f"Strategy override for {node_type!r} contains unsupported legacy keys: {joined}. " "Use skip_sync under strategies. or datasource.handler_configs instead." ) preset = override.pop("preset", None) if preset: runtime_cfg.config = ConfigPresets.get_preset(str(preset)) skip_sync = override.pop("skip_sync", None) skip_post_check = override.pop("skip_post_check", None) if skip_sync is not None: runtime_cfg.skip_sync = bool(skip_sync) if skip_post_check is not None: runtime_cfg.skip_post_check = bool(skip_post_check) if "config" in override and isinstance(override["config"], dict): runtime_cfg.config = runtime_cfg.config.model_validate( _deep_merge(runtime_cfg.config.model_dump(), override.pop("config")) ) if override: runtime_cfg.config = runtime_cfg.config.model_validate( _deep_merge(runtime_cfg.config.model_dump(), override) ) return [strategy_map[n] for n in node_types if n in strategy_map] def config_snapshot(config: PipelineRunConfig) -> Dict[str, Any]: snap = config.model_dump(mode="json") for key in ("local_datasource", "remote_datasource"): ds = snap.get(key) or {} if ds.get("api_secret"): ds["api_secret"] = "***" return snap def build_default_config(project_root: Path) -> PipelineRunConfig: defaults = { "local_datasource": { "type": "jsonl", "jsonl_dir": str(project_root / "filtered_datasource" / "datasource" / "ecm-2025-12-02"), "read_only": False, "handler_configs": {}, }, "remote_datasource": { "type": "jsonl", "jsonl_dir": str(project_root / "remote_datasource" / "datasource"), "read_only": False, "handler_configs": {}, }, "persist": { "db_path": str(project_root / "test_results" / "sync_state_machine" / "pipeline_default.db"), "enable": True, "wipe_on_start": False, }, "logging": { "initialize": True, "file_path": None, "level": 20, "suppress_node_logs": True, }, } config = PipelineRunConfig.model_validate(defaults) config.strategies = _apply_strategy_overrides(config.node_types, {}) return config def apply_config_overrides(config: PipelineRunConfig, overrides: Dict[str, Any]) -> PipelineRunConfig: merged = _deep_merge(config.model_dump(), overrides) raw_strategies = merged.pop("strategies", None) resolved = PipelineRunConfig.model_validate(merged) if raw_strategies is not None: resolved.strategies = _apply_strategy_overrides(resolved.node_types, raw_strategies) else: resolved.strategies = _apply_strategy_overrides(resolved.node_types, {}) return resolved def build_config( *, project_root: Path, local_datasource: Optional[Union[DataSourceConfig, Dict[str, Any]]] = None, remote_datasource: Optional[Union[DataSourceConfig, Dict[str, Any]]] = None, overrides: Optional[Dict[str, Any]] = None, overrides_file: Optional[Union[str, Path]] = None, dotenv_file: Optional[Union[str, Path]] = None, ) -> PipelineRunConfig: if dotenv_file is not None: raise ValueError("dotenv_file is no longer supported; use overrides_file instead") config = build_default_config(project_root) if local_datasource is not None: local_payload = ( local_datasource.model_dump() if isinstance(local_datasource, BaseModel) else dict(local_datasource) ) config = apply_config_overrides(config, {"local_datasource": local_payload}) if remote_datasource is not None: remote_payload = ( remote_datasource.model_dump() if isinstance(remote_datasource, BaseModel) else dict(remote_datasource) ) config = apply_config_overrides(config, {"remote_datasource": remote_payload}) if overrides_file: override_path = Path(overrides_file) file_data = load_overrides_from_file(override_path, project_root) config = apply_config_overrides(config, file_data) if overrides: config = apply_config_overrides(config, overrides) return config def build_config_from_file( *, project_root: Path, file_path: Union[str, Path], ) -> PipelineRunConfig: return build_config(project_root=project_root, overrides_file=Path(file_path))