整理了初始化过程，现在更容易被继承了。

sync_state_machine/pipeline/factory.py

@@ -23,19 +23,17 @@ from ..common.binding import BindingManager
 from ..common.collection import DataCollection
 from ..common.persistence import PersistenceBackend
 from ..common.registry import DomainRegistry
-from ..datasource.api.datasource import ApiDataSource
-from ..datasource.db.datasource import DbDataSource
-from ..datasource.jsonl.datasource import JsonlDataSource
-from ..datasource.type_registry import get_datasource_factory, register_datasource_factory
+from ..datasource import ensure_builtin_datasource_types_registered
+from ..datasource.type_registry import get_datasource_factory
 from ..config import (
     PipelineRunConfig,
     DataSourceConfig,
     ApiDataSourceConfig,
-    DbDataSourceConfig,
-    JsonlDataSourceConfig,
     PersistConfig,
+    JsonlDataSourceConfig,
     LoggingConfig,
     StrategyRuntimeConfig,
+    build_config_from_file,
     apply_config_overrides,
     OrphanAction,
     UpdateDirection,
@@ -55,27 +53,9 @@ def _project_root() -> Path:
 
 
 def _prepare_remote_jsonl_dir(remote_dir: Path, *, project_root: Optional[Path] = None) -> Path:
-    project_root_path = (project_root or _project_root()).resolve()
-    resolved_remote_dir = remote_dir.resolve(strict=False)
+    from ..datasource.jsonl.path_utils import prepare_remote_jsonl_dir
 
-    if resolved_remote_dir.exists():
-        if not resolved_remote_dir.is_dir():
-            raise FileNotFoundError(
-                f"remote_datasource.jsonl_dir exists but is not a directory: {resolved_remote_dir}"
-            )
-        return resolved_remote_dir
-
-    try:
-        resolved_remote_dir.relative_to(project_root_path)
-    except ValueError as exc:
-        raise FileNotFoundError(
-            "remote_datasource.jsonl_dir does not exist and auto-create is only allowed "
-            f"under project root: {resolved_remote_dir} (project_root={project_root_path})"
-        ) from exc
-
-    resolved_remote_dir.mkdir(parents=True, exist_ok=True)
-    logger.info("Created missing remote JSONL directory: %s", resolved_remote_dir)
-    return resolved_remote_dir
+    return prepare_remote_jsonl_dir(remote_dir, project_root=(project_root or _project_root()))
 
 
 def _ensure_pipeline_logger(
@@ -104,13 +84,8 @@ def _ensure_pipeline_logger(
 def _build_handler_config(
     node_type: str,
     ds_config: DataSourceConfig,
-    target_project_ids: List[str],
 ) -> Dict[str, Any]:
-    config = dict(ds_config.handler_configs.get(node_type, {}))
-
-    if node_type == "project" and target_project_ids:
-        config.setdefault("filter_project_id", target_project_ids)
-    return config
+    return dict(ds_config.handler_configs.get(node_type, {}))
 
 
 def _create_handler(
@@ -118,7 +93,6 @@ def _create_handler(
     node_type: str,
     ds_config: DataSourceConfig,
     datasource,
-    target_project_ids: List[str],
 ):
     datasource_type = ds_config.type
     handler_class = DomainRegistry.get_handler(node_type, datasource_type)
@@ -127,7 +101,7 @@ def _create_handler(
             f"Missing {datasource_type.upper()} handler for node_type: {node_type}"
         )
 
-    handler_config = _build_handler_config(node_type, ds_config, target_project_ids)
+    handler_config = _build_handler_config(node_type, ds_config)
     if datasource_type == "api":
         handler = handler_class(**handler_config)
     else:
@@ -138,22 +112,12 @@ def _create_handler(
     return handler
 
 
-def _resolve_datasource_target_project_ids(config: PipelineRunConfig) -> tuple[List[str], List[str]]:
-    fallback = [str(pid) for pid in config.target_project_ids if str(pid)]
-
-    local_cfg_ids = [str(pid) for pid in config.local_datasource.target_project_ids if str(pid)]
-    remote_cfg_ids = [str(pid) for pid in config.remote_datasource.target_project_ids if str(pid)]
-
-    local_target_ids = local_cfg_ids if local_cfg_ids else fallback
-    remote_target_ids = remote_cfg_ids if remote_cfg_ids else fallback
-    return local_target_ids, remote_target_ids
-
-
 async def _create_datasource(
     ds_config: DataSourceConfig,
     *,
     datasource_override=None,
     endpoint_name: str,
+    project_root: Path | None = None,
 ):
     if datasource_override is not None:
         return datasource_override
@@ -162,54 +126,24 @@ async def _create_datasource(
     if datasource_factory is None:
         raise RuntimeError(f"No datasource factory registered for type: {ds_config.type}")
 
-    datasource = datasource_factory(ds_config, endpoint_name)
+    factory_kwargs: dict[str, Any] = {}
+    signature = inspect.signature(datasource_factory)
+    if "project_root" in signature.parameters:
+        factory_kwargs["project_root"] = project_root or _project_root()
+
+    datasource = datasource_factory(ds_config, endpoint_name, **factory_kwargs)
     if inspect.isawaitable(datasource):
         datasource = await datasource
     return datasource
 
 
-def _build_jsonl_datasource(ds_config: JsonlDataSourceConfig, endpoint_name: str):
-    dir_path = Path(ds_config.jsonl_dir)
-    if endpoint_name == "remote":
-        dir_path = _prepare_remote_jsonl_dir(dir_path)
-    elif not dir_path.exists() or not dir_path.is_dir():
-        raise FileNotFoundError(
-            f"{endpoint_name}_datasource.jsonl_dir does not exist or is not a directory: {dir_path}"
-        )
-    return JsonlDataSource(dir_path, read_only=ds_config.read_only)
+async def _initialize_datasource(datasource, *, endpoint_name: str) -> None:
+    try:
+        await datasource.initialize()
+    except Exception as exc:
+        raise RuntimeError(f"Failed to initialize {endpoint_name} datasource: {type(exc).__name__}: {exc}") from exc
 
-
-def _build_db_datasource(ds_config: DbDataSourceConfig, endpoint_name: str):
-    del ds_config, endpoint_name
-    return DbDataSource()
-
-
-async def _build_api_datasource(ds_config: ApiDataSourceConfig, endpoint_name: str):
-    del endpoint_name
-    datasource = ApiDataSource(
-        base_url=ds_config.api_base_url,
-        uid=ds_config.api_uid,
-        secret=ds_config.api_secret,
-        debug=ds_config.api_debug,
-        poll_max_retries=ds_config.poll_max_retries,
-        poll_interval=ds_config.poll_interval,
-        rate_limit_max_requests=ds_config.api_rate_limit_max_requests,
-        rate_limit_window_seconds=ds_config.api_rate_limit_window_seconds,
-    )
-    await datasource.initialize()
-    return datasource
-
-
-def _ensure_builtin_datasource_factories_registered() -> None:
-    if get_datasource_factory("jsonl") is None:
-        register_datasource_factory("jsonl", _build_jsonl_datasource)
-    if get_datasource_factory("db") is None:
-        register_datasource_factory("db", _build_db_datasource)
-    if get_datasource_factory("api") is None:
-        register_datasource_factory("api", _build_api_datasource)
-
-
-_ensure_builtin_datasource_factories_registered()
+ensure_builtin_datasource_types_registered()
 
 
 def _register_handlers_for_endpoint(
@@ -217,7 +151,6 @@ def _register_handlers_for_endpoint(
     datasource,
     ds_config: DataSourceConfig,
     node_types: List[str],
-    target_project_ids: List[str],
 ) -> None:
     for node_type in node_types:
         datasource.register_handler(
@@ -225,7 +158,6 @@ def _register_handlers_for_endpoint(
                 node_type=node_type,
                 ds_config=ds_config,
                 datasource=datasource,
-                target_project_ids=target_project_ids,
             )
         )
 
@@ -240,13 +172,11 @@ def _register_handlers(
         datasource=local_ds,
         ds_config=config.local_datasource,
         node_types=all_types,
-        target_project_ids=[str(pid) for pid in config.local_datasource.target_project_ids if str(pid)],
     )
     _register_handlers_for_endpoint(
         datasource=remote_ds,
         ds_config=config.remote_datasource,
         node_types=all_types,
-        target_project_ids=[str(pid) for pid in config.remote_datasource.target_project_ids if str(pid)],
     )
 
 
@@ -318,7 +248,6 @@ async def create_pipeline_from_config(
 
     all_types = list(config.node_types)
     strategy_map = _resolve_strategy_map(config, all_types)
-    local_target_project_ids, remote_target_project_ids = _resolve_datasource_target_project_ids(config)
 
     if config.persist.backend != "sqlite":
         raise NotImplementedError(f"Persistence backend not supported yet: {config.persist.backend}")
@@ -338,31 +267,20 @@ async def create_pipeline_from_config(
             config.local_datasource,
             datasource_override=local_datasource_override,
             endpoint_name="local",
+            project_root=_project_root(),
         )
+        await _initialize_datasource(local_ds, endpoint_name="local")
         remote_ds = await _create_datasource(
             config.remote_datasource,
             datasource_override=remote_datasource_override,
             endpoint_name="remote",
+            project_root=_project_root(),
         )
-        local_cfg_with_target = config.local_datasource.model_copy(
-            update={"target_project_ids": list(local_target_project_ids)},
-            deep=True,
-        )
-        remote_cfg_with_target = config.remote_datasource.model_copy(
-            update={"target_project_ids": list(remote_target_project_ids)},
-            deep=True,
-        )
-        handler_target_config = config.model_copy(
-            update={
-                "local_datasource": local_cfg_with_target,
-                "remote_datasource": remote_cfg_with_target,
-            },
-            deep=True,
-        )
+        await _initialize_datasource(remote_ds, endpoint_name="remote")
+        local_ds.set_target_project_ids([str(pid) for pid in config.local_datasource.target_project_ids if str(pid)])
+        remote_ds.set_target_project_ids([str(pid) for pid in config.remote_datasource.target_project_ids if str(pid)])
 
-        _register_handlers(handler_target_config, local_ds, remote_ds, all_types)
-        local_ds.set_target_project_ids(local_target_project_ids)
-        remote_ds.set_target_project_ids(remote_target_project_ids)
+        _register_handlers(config, local_ds, remote_ds, all_types)
 
         local_collection = DataCollection("local", persistence, auto_persist=False)
         remote_collection = DataCollection("remote", persistence, auto_persist=False)
@@ -387,9 +305,6 @@ async def create_pipeline_from_config(
             remote_collection=remote_collection,
             binding_manager=binding_manager,
             enable_persistence=config.persist.enable,
-            target_project_ids=config.target_project_ids,
-            local_target_project_ids=local_target_project_ids,
-            remote_target_project_ids=remote_target_project_ids,
         )
 
         return pipeline
@@ -474,6 +389,40 @@ async def run_pipeline_from_config(
             await pipeline.close()
 
 
+async def create_pipeline_from_file(
+    *,
+    project_root: Path,
+    config_path: str | Path,
+    local_datasource_override=None,
+    remote_datasource_override=None,
+) -> FullSyncPipeline:
+    config = build_config_from_file(project_root=project_root, file_path=config_path)
+    return await create_pipeline_from_config(
+        config,
+        local_datasource_override=local_datasource_override,
+        remote_datasource_override=remote_datasource_override,
+    )
+
+
+async def run_pipeline_from_file(
+    *,
+    project_root: Path,
+    config_path: str | Path,
+    title: str = "sync_state_machine pipeline",
+    print_summary: bool = True,
+    local_datasource_override=None,
+    remote_datasource_override=None,
+) -> Dict[str, Any]:
+    config = build_config_from_file(project_root=project_root, file_path=config_path)
+    return await run_pipeline_from_config(
+        config,
+        title=title,
+        print_summary=print_summary,
+        local_datasource_override=local_datasource_override,
+        remote_datasource_override=remote_datasource_override,
+    )
+
+
 async def create_jsonl_to_api_pipeline(
     *,
     # 路径配置
@@ -506,8 +455,12 @@ async def create_jsonl_to_api_pipeline(
 ) -> FullSyncPipeline:
     cfg = PipelineRunConfig(
         node_types=node_types,
-        target_project_ids=target_project_ids or [],
-        local_datasource=JsonlDataSourceConfig(type="jsonl", jsonl_dir=local_jsonl_dir, read_only=False),
+        local_datasource=JsonlDataSourceConfig(
+            type="jsonl",
+            jsonl_dir=local_jsonl_dir,
+            read_only=False,
+            target_project_ids=target_project_ids or [],
+        ),
         remote_datasource=ApiDataSourceConfig(
             type="api",
             api_base_url=api_base_url,
@@ -570,7 +523,6 @@ async def create_api_to_jsonl_pipeline(
 ) -> FullSyncPipeline:
     cfg = PipelineRunConfig(
         node_types=node_types,
-        target_project_ids=target_project_ids or [],
         local_datasource=ApiDataSourceConfig(
             type="api",
             api_base_url=api_base_url,
@@ -584,7 +536,12 @@ async def create_api_to_jsonl_pipeline(
             target_project_ids=target_project_ids or [],
             handler_configs=handler_configs or {},
         ),
-        remote_datasource=JsonlDataSourceConfig(type="jsonl", jsonl_dir=remote_jsonl_dir, read_only=False),
+        remote_datasource=JsonlDataSourceConfig(
+            type="jsonl",
+            jsonl_dir=remote_jsonl_dir,
+            read_only=False,
+            target_project_ids=target_project_ids or [],
+        ),
         strategies=[],
         persist=PersistConfig(
             db_path=persistence_db,
@@ -618,9 +575,18 @@ async def create_jsonl_to_jsonl_pipeline(
 ) -> FullSyncPipeline:
     cfg = PipelineRunConfig(
         node_types=node_types,
-        target_project_ids=target_project_ids or [],
-        local_datasource=JsonlDataSourceConfig(type="jsonl", jsonl_dir=local_jsonl_dir, read_only=False),
-        remote_datasource=JsonlDataSourceConfig(type="jsonl", jsonl_dir=remote_jsonl_dir, read_only=False),
+        local_datasource=JsonlDataSourceConfig(
+            type="jsonl",
+            jsonl_dir=local_jsonl_dir,
+            read_only=False,
+            target_project_ids=target_project_ids or [],
+        ),
+        remote_datasource=JsonlDataSourceConfig(
+            type="jsonl",
+            jsonl_dir=remote_jsonl_dir,
+            read_only=False,
+            target_project_ids=target_project_ids or [],
+        ),
         strategies=[],
         persist=PersistConfig(
             db_path=persistence_db,