优化sync_system代码质量

This commit is contained in:
strepsiades
2026-04-01 11:52:57 +08:00
parent eb02fd32a5
commit 8f4727a772
46 changed files with 1431 additions and 890 deletions
+76 -42
View File
@@ -9,7 +9,7 @@ from ..common.registry import DomainRegistry
from .datasource_config import DataSourceConfig
from .pipeline_config import PipelineRunConfig
from .run_presets import load_overrides_from_file
from .strategy_config import ConfigPresets, StrategyRuntimeConfig
from .strategy_config import ConfigPresets, StrategyRuntimeConfig, resolve_domain_option_config
def _deep_merge(base: Mapping[str, Any], override: Mapping[str, Any]) -> Dict[str, Any]:
@@ -28,19 +28,76 @@ def _deep_merge(base: Mapping[str, Any], override: Mapping[str, Any]) -> Dict[st
return merged
def _build_default_strategy_runtime_config(node_type: str) -> StrategyRuntimeConfig | None:
strategy_class = DomainRegistry.get_strategy_class(node_type)
if strategy_class is None:
return None
config = strategy_class.default_config.model_copy(deep=True)
config.domain_option = resolve_domain_option_config(
getattr(strategy_class, "domain_option_model", None),
config.domain_option,
)
return StrategyRuntimeConfig(
node_type=node_type,
config=config,
)
def _default_strategy_map(node_types: list[str]) -> Dict[str, StrategyRuntimeConfig]:
strategy_map: Dict[str, StrategyRuntimeConfig] = {}
for node_type in node_types:
strategy_class = DomainRegistry.get_strategy_class(node_type)
if strategy_class is None:
runtime_cfg = _build_default_strategy_runtime_config(node_type)
if runtime_cfg is None:
continue
strategy_map[node_type] = StrategyRuntimeConfig(
node_type=node_type,
config=strategy_class.default_config.model_copy(deep=True),
)
strategy_map[node_type] = runtime_cfg
return strategy_map
def _merge_strategy_runtime_config(
runtime_cfg: StrategyRuntimeConfig,
raw_override: Any,
) -> StrategyRuntimeConfig:
override = dict(raw_override or {})
override.pop("node_type", None)
deprecated_keys = [key for key in ("force_sync", "load_mode") if key in override]
if deprecated_keys:
joined = ", ".join(deprecated_keys)
raise ValueError(
f"Strategy override for {runtime_cfg.node_type!r} contains unsupported legacy keys: {joined}. "
"Use skip_sync under strategies.<node_type> or datasource.handler_configs instead."
)
preset = override.pop("preset", None)
if preset:
runtime_cfg.config = runtime_cfg.config.model_validate(
_deep_merge(
runtime_cfg.config.model_dump(),
ConfigPresets.get_preset(str(preset)).model_dump(exclude_defaults=True),
)
)
if "config" in override and isinstance(override["config"], dict):
runtime_cfg.config = runtime_cfg.config.model_validate(
_deep_merge(runtime_cfg.config.model_dump(), override.pop("config"))
)
if override:
runtime_cfg.config = runtime_cfg.config.model_validate(
_deep_merge(runtime_cfg.config.model_dump(), override)
)
strategy_class = DomainRegistry.get_strategy_class(runtime_cfg.node_type)
runtime_cfg.config.domain_option = resolve_domain_option_config(
getattr(strategy_class, "domain_option_model", None) if strategy_class is not None else None,
runtime_cfg.config.domain_option,
)
return runtime_cfg
def _apply_strategy_overrides(
node_types: list[str],
strategy_overrides: Any,
@@ -49,8 +106,16 @@ def _apply_strategy_overrides(
if isinstance(strategy_overrides, list):
for item in strategy_overrides:
runtime_cfg = StrategyRuntimeConfig.model_validate(item)
strategy_map[runtime_cfg.node_type] = runtime_cfg
raw_item = dict(item or {})
node_type = str(raw_item.get("node_type") or "")
if not node_type:
raise ValueError("Strategy override list item missing required node_type")
runtime_cfg = strategy_map.get(node_type)
if runtime_cfg is None:
runtime_cfg = _build_default_strategy_runtime_config(node_type) or StrategyRuntimeConfig(node_type=node_type)
strategy_map[node_type] = _merge_strategy_runtime_config(runtime_cfg, raw_item)
return [strategy_map[n] for n in node_types if n in strategy_map]
if not isinstance(strategy_overrides, dict):
@@ -58,41 +123,10 @@ def _apply_strategy_overrides(
for node_type, raw_override in strategy_overrides.items():
if node_type not in strategy_map:
strategy_map[node_type] = StrategyRuntimeConfig(node_type=node_type)
strategy_map[node_type] = _build_default_strategy_runtime_config(node_type) or StrategyRuntimeConfig(node_type=node_type)
runtime_cfg = strategy_map[node_type]
override = dict(raw_override or {})
deprecated_keys = [key for key in ("force_sync", "load_mode") if key in override]
if deprecated_keys:
joined = ", ".join(deprecated_keys)
raise ValueError(
f"Strategy override for {node_type!r} contains unsupported legacy keys: {joined}. "
"Use skip_sync under strategies.<node_type> or datasource.handler_configs instead."
)
preset = override.pop("preset", None)
if preset:
runtime_cfg.config = ConfigPresets.get_preset(str(preset))
skip_sync = override.pop("skip_sync", None)
skip_post_check = override.pop("skip_post_check", None)
if skip_sync is not None:
runtime_cfg.skip_sync = bool(skip_sync)
if skip_post_check is not None:
runtime_cfg.skip_post_check = bool(skip_post_check)
if "config" in override and isinstance(override["config"], dict):
runtime_cfg.config = runtime_cfg.config.model_validate(
_deep_merge(runtime_cfg.config.model_dump(), override.pop("config"))
)
if override:
runtime_cfg.config = runtime_cfg.config.model_validate(
_deep_merge(runtime_cfg.config.model_dump(), override)
)
strategy_map[node_type] = _merge_strategy_runtime_config(runtime_cfg, raw_override)
return [strategy_map[n] for n in node_types if n in strategy_map]