""" Pipeline Factory - 工厂函数，负责初始化所有组件并创建 Pipeline 用户只需提供配置参数，工厂函数负责： 1. 初始化 Persistence 2. 初始化 DataSource（并注册 Handler） 3. 初始化 Collection 和 BindingManager 4. 初始化 Strategy（并注入依赖） 5. 创建 Pipeline（注入所有组件） 6. 返回完全就绪的 Pipeline """ from __future__ import annotations import asyncio import inspect import json import logging from pathlib import Path from typing import Any, Dict, List, Optional from ..common.binding import BindingManager from ..common.collection import DataCollection from ..common.persistence import PersistenceBackend, create_persistence_backend from ..common.registry import DomainRegistry from ..datasource import ensure_builtin_datasource_types_registered from ..datasource.type_registry import get_datasource_factory from ..config import ( PipelineRunConfig, MultiProjectRunConfig, DataSourceConfig, ApiDataSourceConfig, PersistConfig, JsonlDataSourceConfig, LoggingConfig, build_config, StrategyRuntimeConfig, build_config_from_file, build_multi_project_config_from_file, apply_config_overrides, OrphanAction, UpdateDirection, resolve_log_file_path, config_snapshot, is_multi_project_payload, load_overrides_from_file, ) from ..sync_system.strategy import BaseSyncStrategy from .run_logger import init_pipeline_logger from .full_sync_pipeline import FullSyncPipeline logger = logging.getLogger(__name__) def _project_root() -> Path: return Path(__file__).resolve().parents[2] def _prepare_remote_jsonl_dir(remote_dir: Path, *, project_root: Optional[Path] = None) -> Path: from ..datasource.jsonl.path_utils import prepare_remote_jsonl_dir return prepare_remote_jsonl_dir(remote_dir, project_root=(project_root or _project_root())) def _ensure_pipeline_logger( *, initialize_logger: bool, logger_file_path: Optional[str], logger_level: int, suppress_node_logs: bool, ) -> None: if not initialize_logger: return app_logger = logging.getLogger("sync_state_machine") if app_logger.handlers: return init_pipeline_logger( log_file_path=logger_file_path, level=logger_level, mirror_console=True, replace_handlers=False, suppress_node_logs=suppress_node_logs, ) def _build_handler_config( node_type: str, ds_config: DataSourceConfig, ) -> Dict[str, Any]: return dict(ds_config.handler_configs.get(node_type, {})) def _create_handler( *, node_type: str, ds_config: DataSourceConfig, datasource, ): datasource_type = ds_config.type handler_class = DomainRegistry.get_handler(node_type, datasource_type) if handler_class is None: raise RuntimeError( f"Missing {datasource_type.upper()} handler for node_type: {node_type}" ) handler_config = _build_handler_config(node_type, ds_config) if datasource_type == "api": handler = handler_class(**handler_config) else: handler = handler_class(datasource) if hasattr(handler, "set_handler_config"): handler.set_handler_config(handler_config) return handler async def _create_datasource( ds_config: DataSourceConfig, *, datasource_override=None, endpoint_name: str, project_root: Path | None = None, ): if datasource_override is not None: return datasource_override datasource_factory = get_datasource_factory(ds_config.type) if datasource_factory is None: raise RuntimeError(f"No datasource factory registered for type: {ds_config.type}") factory_kwargs: dict[str, Any] = {} signature = inspect.signature(datasource_factory) if "project_root" in signature.parameters: factory_kwargs["project_root"] = project_root or _project_root() datasource = datasource_factory(ds_config, endpoint_name, **factory_kwargs) if inspect.isawaitable(datasource): datasource = await datasource return datasource async def _initialize_datasource(datasource, *, endpoint_name: str) -> None: try: await datasource.initialize() except Exception as exc: raise RuntimeError(f"Failed to initialize {endpoint_name} datasource: {type(exc).__name__}: {exc}") from exc ensure_builtin_datasource_types_registered() def _add_handlers_for_endpoint( *, datasource, ds_config: DataSourceConfig, node_types: List[str], ) -> None: for node_type in node_types: datasource.add_handler( _create_handler( node_type=node_type, ds_config=ds_config, datasource=datasource, ) ) def _add_handlers( config: PipelineRunConfig, local_ds, remote_ds, all_types: List[str], ) -> None: _add_handlers_for_endpoint( datasource=local_ds, ds_config=config.local_datasource, node_types=all_types, ) _add_handlers_for_endpoint( datasource=remote_ds, ds_config=config.remote_datasource, node_types=all_types, ) def _build_collections_and_bindings( *, persistence: PersistenceBackend, scope: str, ) -> tuple[DataCollection, DataCollection, BindingManager]: local_collection = DataCollection("local", scope=scope, persistence=persistence, auto_persist=False) remote_collection = DataCollection("remote", scope=scope, persistence=persistence, auto_persist=False) binding_manager = BindingManager(persistence, auto_persist=False, scope=scope) return local_collection, remote_collection, binding_manager def _resolve_strategy_map(config: PipelineRunConfig, node_types: List[str]) -> Dict[str, StrategyRuntimeConfig]: if config.strategies: return {item.node_type: item for item in config.strategies} resolved: Dict[str, StrategyRuntimeConfig] = {} for node_type in node_types: strategy_class = DomainRegistry.get_strategy_class(node_type) if strategy_class is None: continue resolved[node_type] = StrategyRuntimeConfig( node_type=node_type, config=strategy_class.default_config.model_copy(deep=True), ) return resolved def _build_pipeline_run_config( *, node_types: List[str], local_datasource: DataSourceConfig, remote_datasource: DataSourceConfig, persist: PersistConfig, logging_config: LoggingConfig, strategy_overrides: Optional[Dict[str, Dict[str, Any]]] = None, ) -> PipelineRunConfig: cfg = PipelineRunConfig( node_types=node_types, local_datasource=local_datasource, remote_datasource=remote_datasource, strategies=[], persist=persist, logging=logging_config, ) if strategy_overrides: cfg = apply_config_overrides(cfg, {"strategies": strategy_overrides}) return cfg def _build_strategies( config: PipelineRunConfig, node_types: List[str], local_collection: DataCollection, remote_collection: DataCollection, binding_manager: BindingManager, strategy_map: Dict[str, StrategyRuntimeConfig], ) -> List[BaseSyncStrategy[Any]]: strategies: List[BaseSyncStrategy[Any]] = [] both_jsonl = isinstance(config.local_datasource, JsonlDataSourceConfig) and isinstance(config.remote_datasource, JsonlDataSourceConfig) for node_type in node_types: strategy_class = DomainRegistry.get_strategy_class(node_type) if strategy_class is None: raise RuntimeError(f"Missing strategy for node_type: {node_type}") strategy = strategy_class(node_type, local_collection, remote_collection, binding_manager) if both_jsonl: strategy.update_config( local_orphan_action=OrphanAction.CREATE_REMOTE, remote_orphan_action=OrphanAction.NONE, update_direction=UpdateDirection.PUSH, ) if node_type == "contract": strategy.update_config(depend_fields={"project_id": "project"}) runtime_cfg = strategy_map.get(node_type) if runtime_cfg is not None: strategy.set_config(runtime_cfg.config.model_copy(deep=True)) strategy.skip_sync = bool(runtime_cfg.skip_sync) strategies.append(strategy) return strategies async def create_pipeline_from_config( config: PipelineRunConfig, *, scope: str = "global", pipeline_name: str | None = None, bootstrap_binding_node_types: Optional[List[str]] = None, ephemeral_node_types: Optional[List[str]] = None, local_datasource_override=None, remote_datasource_override=None, ) -> FullSyncPipeline: if config.logging.initialize and not config.logging.file_path: config.logging.file_path = resolve_log_file_path(config.logging) _ensure_pipeline_logger( initialize_logger=config.logging.initialize, logger_file_path=config.logging.file_path, logger_level=config.logging.level, suppress_node_logs=config.logging.suppress_node_logs, ) all_types = list(config.node_types) strategy_map = _resolve_strategy_map(config, all_types) db_path = Path(config.persist.db_path) if config.persist.wipe_on_start and db_path.exists(): db_path.unlink() persistence: Optional[PersistenceBackend] = None local_ds = None remote_ds = None try: persistence = create_persistence_backend(config.persist) await persistence.initialize() local_ds = await _create_datasource( config.local_datasource, datasource_override=local_datasource_override, endpoint_name="local", project_root=_project_root(), ) await _initialize_datasource(local_ds, endpoint_name="local") remote_ds = await _create_datasource( config.remote_datasource, datasource_override=remote_datasource_override, endpoint_name="remote", project_root=_project_root(), ) await _initialize_datasource(remote_ds, endpoint_name="remote") _add_handlers(config, local_ds, remote_ds, all_types) local_collection, remote_collection, binding_manager = _build_collections_and_bindings( persistence=persistence, scope=scope, ) strategies = _build_strategies( config, all_types, local_collection, remote_collection, binding_manager, strategy_map, ) pipeline = FullSyncPipeline( local_datasource=local_ds, remote_datasource=remote_ds, strategies=strategies, node_types=all_types, persistence=persistence, local_collection=local_collection, remote_collection=remote_collection, binding_manager=binding_manager, scope=scope, pipeline_name=pipeline_name, bootstrap_binding_node_types=bootstrap_binding_node_types, ephemeral_node_types=ephemeral_node_types, enable_persistence=config.persist.enable, ) return pipeline except Exception: async def _safe_close(name: str, resource) -> None: if resource is None: return try: await asyncio.wait_for(resource.close(), timeout=3.0) except Exception as close_exc: logger.warning("Failed to close %s during create cleanup: %s", name, close_exc) await _safe_close("remote_datasource", remote_ds) await _safe_close("local_datasource", local_ds) await _safe_close("persistence", persistence) raise async def run_pipeline_from_config( config: PipelineRunConfig, *, title: str = "sync_state_machine pipeline", print_summary: bool = True, scope: str = "global", bootstrap_binding_node_types: Optional[List[str]] = None, local_datasource_override=None, remote_datasource_override=None, ) -> Dict[str, Any]: if config.logging.initialize and not config.logging.file_path: config.logging.file_path = resolve_log_file_path(config.logging) _ensure_pipeline_logger( initialize_logger=config.logging.initialize, logger_file_path=config.logging.file_path, logger_level=config.logging.level, suppress_node_logs=config.logging.suppress_node_logs, ) resolved_cfg = config_snapshot(config) resolved_cfg_text = json.dumps(resolved_cfg, ensure_ascii=False, indent=2) logger.info( "Resolved Full Config Summary: node_types=%d local=%s remote=%s", len(config.node_types), config.local_datasource.type, config.remote_datasource.type, ) if config.logging.file_path: try: with open(config.logging.file_path, "a", encoding="utf-8") as fp: fp.write("Resolved Full Config:\n") fp.write(resolved_cfg_text) fp.write("\n") except Exception as exc: logger.warning("Failed to write resolved config into log file: %s", exc) pipeline: Optional[FullSyncPipeline] = None try: if print_summary: print("\n" + "=" * 80) print(f"🚀 Creating {title}") print(f"📝 Log file: {config.logging.file_path or '-'}") print( f"🔧 Resolved Full Config Summary: node_types={len(config.node_types)} " f"local={config.local_datasource.type} " f"remote={config.remote_datasource.type}" ) print(f"🔧 Resolved Full Config:\n{resolved_cfg_text}") print("=" * 80) pipeline = await create_pipeline_from_config( config, scope=scope, pipeline_name=title, bootstrap_binding_node_types=bootstrap_binding_node_types, local_datasource_override=local_datasource_override, remote_datasource_override=remote_datasource_override, ) stats = await pipeline.run() if print_summary: print(f"\n✅ Pipeline completed: stats={stats}") print(f"🗃️ Persistence DB: {config.persist.db_path}") return stats finally: if pipeline is not None: await pipeline.close() async def create_pipeline_from_file( *, project_root: Path, config_path: str | Path, scope: str = "global", pipeline_name: str | None = None, bootstrap_binding_node_types: Optional[List[str]] = None, ephemeral_node_types: Optional[List[str]] = None, local_datasource_override=None, remote_datasource_override=None, ) -> FullSyncPipeline: config = build_config_from_file(project_root=project_root, file_path=config_path) return await create_pipeline_from_config( config, scope=scope, pipeline_name=pipeline_name, bootstrap_binding_node_types=bootstrap_binding_node_types, ephemeral_node_types=ephemeral_node_types, local_datasource_override=local_datasource_override, remote_datasource_override=remote_datasource_override, ) async def run_pipeline_from_file( *, project_root: Path, config_path: str | Path, title: str = "sync_state_machine pipeline", print_summary: bool = True, scope: str = "global", bootstrap_binding_node_types: Optional[List[str]] = None, local_datasource_override=None, remote_datasource_override=None, ) -> Dict[str, Any]: config = build_config_from_file(project_root=project_root, file_path=config_path) return await run_pipeline_from_config( config, title=title, print_summary=print_summary, scope=scope, bootstrap_binding_node_types=bootstrap_binding_node_types, local_datasource_override=local_datasource_override, remote_datasource_override=remote_datasource_override, ) async def run_profile_from_file( *, project_root: Path, config_path: str | Path, print_summary: bool = True, ) -> Dict[str, Any]: cfg_path = Path(config_path) raw = load_overrides_from_file(cfg_path, project_root) if is_multi_project_payload(raw): from .multi_project_pipeline import MultiProjectPipeline multi_cfg = MultiProjectRunConfig.model_validate(raw) pipeline = MultiProjectPipeline(project_root=project_root, config_path=cfg_path, config=multi_cfg) if print_summary: print("\n" + "=" * 80) print(f"🚀 Creating multi-project pipeline from: {cfg_path}") print("=" * 80) return await pipeline.run() config = build_config(project_root=project_root, overrides=raw) mode = f"{config.local_datasource.type}->{config.remote_datasource.type}" return await run_pipeline_from_config( config, title=f"sync_state_machine pipeline ({mode})", print_summary=print_summary, ) async def create_jsonl_to_api_pipeline( *, # 路径配置 local_jsonl_dir: str, persistence_db: str, # API 配置 api_base_url: str, api_uid: str, api_secret: str, # 节点类型 node_types: List[str], strategy_overrides: Optional[Dict[str, Dict[str, Any]]] = None, # Pipeline 行为 enable_persistence: bool = True, wipe_persistence_on_start: bool = False, # API 配置 api_debug: bool = False, poll_max_retries: int = 10, poll_interval: float = 0.5, api_rate_limit_max_requests: int = 30, api_rate_limit_window_seconds: float = 10.0, local_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None, remote_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None, # Logger 配置 initialize_logger: bool = True, logger_file_path: Optional[str] = None, logger_level: int = logging.INFO, ) -> FullSyncPipeline: cfg = _build_pipeline_run_config( node_types=node_types, local_datasource=JsonlDataSourceConfig( type="jsonl", jsonl_dir=local_jsonl_dir, read_only=False, handler_configs=local_handler_configs or {}, ), remote_datasource=ApiDataSourceConfig( type="api", api_base_url=api_base_url, api_uid=api_uid, api_secret=api_secret, api_debug=api_debug, poll_max_retries=poll_max_retries, poll_interval=poll_interval, api_rate_limit_max_requests=api_rate_limit_max_requests, api_rate_limit_window_seconds=api_rate_limit_window_seconds, handler_configs=remote_handler_configs or {}, ), persist=PersistConfig( db_path=persistence_db, enable=enable_persistence, wipe_on_start=wipe_persistence_on_start, ), logging_config=LoggingConfig( initialize=initialize_logger, file_path=logger_file_path, level=logger_level, ), strategy_overrides=strategy_overrides, ) return await create_pipeline_from_config(cfg) async def create_api_to_jsonl_pipeline( *, # API 配置（本地） api_base_url: str, api_uid: str, api_secret: str, # 路径配置 remote_jsonl_dir: str, persistence_db: str, # 节点类型 node_types: List[str], strategy_overrides: Optional[Dict[str, Dict[str, Any]]] = None, # Pipeline 行为 enable_persistence: bool = True, wipe_persistence_on_start: bool = False, # API 配置 api_debug: bool = False, poll_max_retries: int = 10, poll_interval: float = 0.5, api_rate_limit_max_requests: int = 30, api_rate_limit_window_seconds: float = 10.0, local_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None, remote_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None, # Logger 配置 initialize_logger: bool = True, logger_file_path: Optional[str] = None, logger_level: int = logging.INFO, ) -> FullSyncPipeline: cfg = _build_pipeline_run_config( node_types=node_types, local_datasource=ApiDataSourceConfig( type="api", api_base_url=api_base_url, api_uid=api_uid, api_secret=api_secret, api_debug=api_debug, poll_max_retries=poll_max_retries, poll_interval=poll_interval, api_rate_limit_max_requests=api_rate_limit_max_requests, api_rate_limit_window_seconds=api_rate_limit_window_seconds, handler_configs=local_handler_configs or {}, ), remote_datasource=JsonlDataSourceConfig( type="jsonl", jsonl_dir=remote_jsonl_dir, read_only=False, handler_configs=remote_handler_configs or {}, ), persist=PersistConfig( db_path=persistence_db, enable=enable_persistence, wipe_on_start=wipe_persistence_on_start, ), logging_config=LoggingConfig( initialize=initialize_logger, file_path=logger_file_path, level=logger_level, ), strategy_overrides=strategy_overrides, ) return await create_pipeline_from_config(cfg) async def create_jsonl_to_jsonl_pipeline( *, local_jsonl_dir: str, remote_jsonl_dir: str, persistence_db: str, node_types: List[str], enable_persistence: bool = True, wipe_persistence_on_start: bool = False, strategy_overrides: Optional[Dict[str, Dict[str, Any]]] = None, local_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None, remote_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None, initialize_logger: bool = True, logger_file_path: Optional[str] = None, logger_level: int = logging.INFO, ) -> FullSyncPipeline: cfg = _build_pipeline_run_config( node_types=node_types, local_datasource=JsonlDataSourceConfig( type="jsonl", jsonl_dir=local_jsonl_dir, read_only=False, handler_configs=local_handler_configs or {}, ), remote_datasource=JsonlDataSourceConfig( type="jsonl", jsonl_dir=remote_jsonl_dir, read_only=False, handler_configs=remote_handler_configs or {}, ), persist=PersistConfig( db_path=persistence_db, enable=enable_persistence, wipe_on_start=wipe_persistence_on_start, ), logging_config=LoggingConfig( initialize=initialize_logger, file_path=logger_file_path, level=logger_level, ), strategy_overrides=strategy_overrides, ) return await create_pipeline_from_config(cfg)