优化sync_system代码质量
This commit is contained in:
@@ -9,7 +9,7 @@ from ..common.registry import DomainRegistry
|
||||
from .datasource_config import DataSourceConfig
|
||||
from .pipeline_config import PipelineRunConfig
|
||||
from .run_presets import load_overrides_from_file
|
||||
from .strategy_config import ConfigPresets, StrategyRuntimeConfig
|
||||
from .strategy_config import ConfigPresets, StrategyRuntimeConfig, resolve_domain_option_config
|
||||
|
||||
|
||||
def _deep_merge(base: Mapping[str, Any], override: Mapping[str, Any]) -> Dict[str, Any]:
|
||||
@@ -28,19 +28,76 @@ def _deep_merge(base: Mapping[str, Any], override: Mapping[str, Any]) -> Dict[st
|
||||
return merged
|
||||
|
||||
|
||||
def _build_default_strategy_runtime_config(node_type: str) -> StrategyRuntimeConfig | None:
|
||||
strategy_class = DomainRegistry.get_strategy_class(node_type)
|
||||
if strategy_class is None:
|
||||
return None
|
||||
|
||||
config = strategy_class.default_config.model_copy(deep=True)
|
||||
config.domain_option = resolve_domain_option_config(
|
||||
getattr(strategy_class, "domain_option_model", None),
|
||||
config.domain_option,
|
||||
)
|
||||
|
||||
return StrategyRuntimeConfig(
|
||||
node_type=node_type,
|
||||
config=config,
|
||||
)
|
||||
|
||||
|
||||
def _default_strategy_map(node_types: list[str]) -> Dict[str, StrategyRuntimeConfig]:
|
||||
strategy_map: Dict[str, StrategyRuntimeConfig] = {}
|
||||
for node_type in node_types:
|
||||
strategy_class = DomainRegistry.get_strategy_class(node_type)
|
||||
if strategy_class is None:
|
||||
runtime_cfg = _build_default_strategy_runtime_config(node_type)
|
||||
if runtime_cfg is None:
|
||||
continue
|
||||
strategy_map[node_type] = StrategyRuntimeConfig(
|
||||
node_type=node_type,
|
||||
config=strategy_class.default_config.model_copy(deep=True),
|
||||
)
|
||||
strategy_map[node_type] = runtime_cfg
|
||||
return strategy_map
|
||||
|
||||
|
||||
def _merge_strategy_runtime_config(
|
||||
runtime_cfg: StrategyRuntimeConfig,
|
||||
raw_override: Any,
|
||||
) -> StrategyRuntimeConfig:
|
||||
override = dict(raw_override or {})
|
||||
override.pop("node_type", None)
|
||||
|
||||
deprecated_keys = [key for key in ("force_sync", "load_mode") if key in override]
|
||||
if deprecated_keys:
|
||||
joined = ", ".join(deprecated_keys)
|
||||
raise ValueError(
|
||||
f"Strategy override for {runtime_cfg.node_type!r} contains unsupported legacy keys: {joined}. "
|
||||
"Use skip_sync under strategies.<node_type> or datasource.handler_configs instead."
|
||||
)
|
||||
|
||||
preset = override.pop("preset", None)
|
||||
if preset:
|
||||
runtime_cfg.config = runtime_cfg.config.model_validate(
|
||||
_deep_merge(
|
||||
runtime_cfg.config.model_dump(),
|
||||
ConfigPresets.get_preset(str(preset)).model_dump(exclude_defaults=True),
|
||||
)
|
||||
)
|
||||
|
||||
if "config" in override and isinstance(override["config"], dict):
|
||||
runtime_cfg.config = runtime_cfg.config.model_validate(
|
||||
_deep_merge(runtime_cfg.config.model_dump(), override.pop("config"))
|
||||
)
|
||||
|
||||
if override:
|
||||
runtime_cfg.config = runtime_cfg.config.model_validate(
|
||||
_deep_merge(runtime_cfg.config.model_dump(), override)
|
||||
)
|
||||
|
||||
strategy_class = DomainRegistry.get_strategy_class(runtime_cfg.node_type)
|
||||
runtime_cfg.config.domain_option = resolve_domain_option_config(
|
||||
getattr(strategy_class, "domain_option_model", None) if strategy_class is not None else None,
|
||||
runtime_cfg.config.domain_option,
|
||||
)
|
||||
|
||||
return runtime_cfg
|
||||
|
||||
|
||||
def _apply_strategy_overrides(
|
||||
node_types: list[str],
|
||||
strategy_overrides: Any,
|
||||
@@ -49,8 +106,16 @@ def _apply_strategy_overrides(
|
||||
|
||||
if isinstance(strategy_overrides, list):
|
||||
for item in strategy_overrides:
|
||||
runtime_cfg = StrategyRuntimeConfig.model_validate(item)
|
||||
strategy_map[runtime_cfg.node_type] = runtime_cfg
|
||||
raw_item = dict(item or {})
|
||||
node_type = str(raw_item.get("node_type") or "")
|
||||
if not node_type:
|
||||
raise ValueError("Strategy override list item missing required node_type")
|
||||
|
||||
runtime_cfg = strategy_map.get(node_type)
|
||||
if runtime_cfg is None:
|
||||
runtime_cfg = _build_default_strategy_runtime_config(node_type) or StrategyRuntimeConfig(node_type=node_type)
|
||||
|
||||
strategy_map[node_type] = _merge_strategy_runtime_config(runtime_cfg, raw_item)
|
||||
return [strategy_map[n] for n in node_types if n in strategy_map]
|
||||
|
||||
if not isinstance(strategy_overrides, dict):
|
||||
@@ -58,41 +123,10 @@ def _apply_strategy_overrides(
|
||||
|
||||
for node_type, raw_override in strategy_overrides.items():
|
||||
if node_type not in strategy_map:
|
||||
strategy_map[node_type] = StrategyRuntimeConfig(node_type=node_type)
|
||||
strategy_map[node_type] = _build_default_strategy_runtime_config(node_type) or StrategyRuntimeConfig(node_type=node_type)
|
||||
|
||||
runtime_cfg = strategy_map[node_type]
|
||||
override = dict(raw_override or {})
|
||||
|
||||
deprecated_keys = [key for key in ("force_sync", "load_mode") if key in override]
|
||||
if deprecated_keys:
|
||||
joined = ", ".join(deprecated_keys)
|
||||
raise ValueError(
|
||||
f"Strategy override for {node_type!r} contains unsupported legacy keys: {joined}. "
|
||||
"Use skip_sync under strategies.<node_type> or datasource.handler_configs instead."
|
||||
)
|
||||
|
||||
preset = override.pop("preset", None)
|
||||
if preset:
|
||||
runtime_cfg.config = ConfigPresets.get_preset(str(preset))
|
||||
|
||||
skip_sync = override.pop("skip_sync", None)
|
||||
skip_post_check = override.pop("skip_post_check", None)
|
||||
|
||||
if skip_sync is not None:
|
||||
runtime_cfg.skip_sync = bool(skip_sync)
|
||||
|
||||
if skip_post_check is not None:
|
||||
runtime_cfg.skip_post_check = bool(skip_post_check)
|
||||
|
||||
if "config" in override and isinstance(override["config"], dict):
|
||||
runtime_cfg.config = runtime_cfg.config.model_validate(
|
||||
_deep_merge(runtime_cfg.config.model_dump(), override.pop("config"))
|
||||
)
|
||||
|
||||
if override:
|
||||
runtime_cfg.config = runtime_cfg.config.model_validate(
|
||||
_deep_merge(runtime_cfg.config.model_dump(), override)
|
||||
)
|
||||
strategy_map[node_type] = _merge_strategy_runtime_config(runtime_cfg, raw_override)
|
||||
|
||||
return [strategy_map[n] for n in node_types if n in strategy_map]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user