strepsiades/ecm_sync_system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

优化sync_system代码质量

Browse Source
This commit is contained in:
strepsiades
2026-04-01 11:52:57 +08:00
parent eb02fd32a5
commit 8f4727a772
46 changed files with 1431 additions and 890 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sync_state_machine/pipeline/factory.py
+45 -78
View File
@@ -45,12 +45,14 @@ from ..config import (
is_multi_project_payload,
load_overrides_from_file,
)
from ..config.strategy_config import resolve_domain_option_config
from ..sync_system.strategy import BaseSyncStrategy
from ..logging import configure_app_logging, ensure_app_logging, get_logger
from .run_logger import init_pipeline_logger
from .full_sync_pipeline import FullSyncPipeline
logger = logging.getLogger(__name__)
logger = get_logger(__name__)
def _project_root() -> Path:
@@ -63,29 +65,6 @@ def _prepare_remote_jsonl_dir(remote_dir: Path, *, project_root: Optional[Path]
return prepare_remote_jsonl_dir(remote_dir, project_root=(project_root or _project_root()))
def _ensure_pipeline_logger(
*,
initialize_logger: bool,
logger_file_path: Optional[str],
logger_level: int,
suppress_node_logs: bool,
) -> None:
if not initialize_logger:
return
app_logger = logging.getLogger("sync_state_machine")
if app_logger.handlers:
return
init_pipeline_logger(
log_file_path=logger_file_path,
level=logger_level,
mirror_console=True,
replace_handlers=False,
suppress_node_logs=suppress_node_logs,
)
def _build_handler_config(
node_type: str,
ds_config: DataSourceConfig,
@@ -198,16 +177,28 @@ def _build_collections_and_bindings(
def _resolve_strategy_map(config: PipelineRunConfig, node_types: List[str]) -> Dict[str, StrategyRuntimeConfig]:
if config.strategies:
return {item.node_type: item for item in config.strategies}
resolved_from_config = {item.node_type: item for item in config.strategies}
for node_type, runtime_cfg in resolved_from_config.items():
strategy_class = DomainRegistry.get_strategy_class(node_type)
runtime_cfg.config.domain_option = resolve_domain_option_config(
getattr(strategy_class, "domain_option_model", None) if strategy_class is not None else None,
runtime_cfg.config.domain_option,
)
return resolved_from_config
resolved: Dict[str, StrategyRuntimeConfig] = {}
for node_type in node_types:
strategy_class = DomainRegistry.get_strategy_class(node_type)
if strategy_class is None:
continue
strategy_config = strategy_class.default_config.model_copy(deep=True)
strategy_config.domain_option = resolve_domain_option_config(
getattr(strategy_class, "domain_option_model", None),
strategy_config.domain_option,
)
resolved[node_type] = StrategyRuntimeConfig(
node_type=node_type,
config=strategy_class.default_config.model_copy(deep=True),
config=strategy_config,
)
return resolved
@@ -251,20 +242,18 @@ def _build_strategies(
raise RuntimeError(f"Missing strategy for node_type: {node_type}")
strategy = strategy_class(node_type, local_collection, remote_collection, binding_manager)
if both_jsonl:
strategy.update_config(
local_orphan_action=OrphanAction.CREATE_REMOTE,
remote_orphan_action=OrphanAction.NONE,
update_direction=UpdateDirection.PUSH,
)
if node_type == "contract":
strategy.update_config(depend_fields={"project_id": "project"})
runtime_cfg = strategy_map.get(node_type)
if runtime_cfg is not None:
strategy.set_config(runtime_cfg.config.model_copy(deep=True))
strategy.skip_sync = bool(runtime_cfg.skip_sync)
strategy.skip_post_check = bool(runtime_cfg.skip_post_check)
strategy.config = runtime_cfg.config.model_copy(deep=True)
strategy.config.log_logic_warnings(node_type=strategy.node_type, logger=logger)
if both_jsonl:
strategy.config.local_orphan_action = OrphanAction.CREATE_REMOTE
strategy.config.remote_orphan_action = OrphanAction.NONE
strategy.config.update_direction = UpdateDirection.PUSH
if node_type == "contract":
strategy.config.depend_fields = {"project_id": "project"}
strategy.config.log_logic_warnings(node_type=strategy.node_type, logger=logger)
strategies.append(strategy)
return strategies
@@ -280,15 +269,7 @@ async def create_pipeline_from_config(
local_datasource_override=None,
remote_datasource_override=None,
) -> FullSyncPipeline:
if config.logging.initialize and not config.logging.file_path:
config.logging.file_path = resolve_log_file_path(config.logging)
_ensure_pipeline_logger(
initialize_logger=config.logging.initialize,
logger_file_path=config.logging.file_path,
logger_level=config.logging.level,
suppress_node_logs=config.logging.suppress_node_logs,
)
ensure_app_logging(config.logging)
all_types = list(config.node_types)
strategy_map = _resolve_strategy_map(config, all_types)
@@ -376,15 +357,7 @@ async def run_pipeline_from_config(
local_datasource_override=None,
remote_datasource_override=None,
) -> Dict[str, Any]:
if config.logging.initialize and not config.logging.file_path:
config.logging.file_path = resolve_log_file_path(config.logging)
_ensure_pipeline_logger(
initialize_logger=config.logging.initialize,
logger_file_path=config.logging.file_path,
logger_level=config.logging.level,
suppress_node_logs=config.logging.suppress_node_logs,
)
ensure_app_logging(config.logging)
resolved_cfg = config_snapshot(config)
resolved_cfg_text = json.dumps(resolved_cfg, ensure_ascii=False, indent=2)
@@ -394,28 +367,21 @@ async def run_pipeline_from_config(
config.local_datasource.type,
config.remote_datasource.type,
)
if config.logging.file_path:
try:
with open(config.logging.file_path, "a", encoding="utf-8") as fp:
fp.write("Resolved Full Config:\n")
fp.write(resolved_cfg_text)
fp.write("\n")
except Exception as exc:
logger.warning("Failed to write resolved config into log file: %s", exc)
logger.info("Resolved Full Config:\n%s", resolved_cfg_text)
pipeline: Optional[FullSyncPipeline] = None
try:
if print_summary:
print("\n" + "=" * 80)
print(f"🚀 Creating {title}")
print(f"📝 Log file: {config.logging.file_path or '-'}")
print(
f"🔧 Resolved Full Config Summary: node_types={len(config.node_types)} "
f"local={config.local_datasource.type} "
f"remote={config.remote_datasource.type}"
logger.info("\n%s", "=" * 80)
logger.info("🚀 Creating %s", title)
logger.info("📝 Log file: %s", config.logging.file_path or "-")
logger.info(
"🔧 Resolved Full Config Summary: node_types=%d local=%s remote=%s",
len(config.node_types),
config.local_datasource.type,
config.remote_datasource.type,
)
print(f"🔧 Resolved Full Config:\n{resolved_cfg_text}")
print("=" * 80)
logger.info("=" * 80)
pipeline = await create_pipeline_from_config(
config,
@@ -428,8 +394,8 @@ async def run_pipeline_from_config(
stats = await pipeline.run()
if print_summary:
print(f"\n✅ Pipeline completed: stats={stats}")
print(f"🗃️ Persistence DB: {config.persist.db_path}")
logger.info("✅ Pipeline completed: stats=%s", stats)
logger.info("🗃️ Persistence DB: %s", config.persist.db_path)
return stats
finally:
@@ -497,12 +463,13 @@ async def run_profile_from_file(
multi_cfg = MultiProjectRunConfig.model_validate(raw)
pipeline = MultiProjectPipeline(project_root=project_root, config_path=cfg_path, config=multi_cfg)
if print_summary:
print("\n" + "=" * 80)
print(f"🚀 Creating multi-project pipeline from: {cfg_path}")
print("=" * 80)
logger.info("\n%s", "=" * 80)
logger.info("🚀 Creating multi-project pipeline from: %s", cfg_path)
logger.info("=" * 80)
return await pipeline.run()
config = build_config(project_root=project_root, overrides=raw)
configure_app_logging(config.logging, replace_handlers=False)
mode = f"{config.local_datasource.type}->{config.remote_datasource.type}"
return await run_pipeline_from_config(
config,