增强配置系统

sync_state_machine/pipeline/factory.py

@@ -13,6 +13,7 @@ Pipeline Factory - 工厂函数，负责初始化所有组件并创建 Pipeline
 from __future__ import annotations
 
 import asyncio
+import inspect
 import json
 import logging
 from pathlib import Path
@@ -25,6 +26,7 @@ from ..common.registry import DomainRegistry
 from ..datasource.api.datasource import ApiDataSource
 from ..datasource.db.datasource import DbDataSource
 from ..datasource.jsonl.datasource import JsonlDataSource
+from ..datasource.type_registry import get_datasource_factory, register_datasource_factory
 from ..config import (
     PipelineRunConfig,
     DataSourceConfig,
@@ -156,19 +158,34 @@ async def _create_datasource(
     if datasource_override is not None:
         return datasource_override
 
-    if isinstance(ds_config, JsonlDataSourceConfig):
-        dir_path = Path(ds_config.jsonl_dir)
-        if endpoint_name == "remote":
-            dir_path = _prepare_remote_jsonl_dir(dir_path)
-        elif not dir_path.exists() or not dir_path.is_dir():
-            raise FileNotFoundError(
-                f"{endpoint_name}_datasource.jsonl_dir does not exist or is not a directory: {dir_path}"
-            )
-        return JsonlDataSource(dir_path, read_only=ds_config.read_only)
+    datasource_factory = get_datasource_factory(ds_config.type)
+    if datasource_factory is None:
+        raise RuntimeError(f"No datasource factory registered for type: {ds_config.type}")
 
-    if isinstance(ds_config, DbDataSourceConfig):
-        return DbDataSource()
+    datasource = datasource_factory(ds_config, endpoint_name)
+    if inspect.isawaitable(datasource):
+        datasource = await datasource
+    return datasource
 
+
+def _build_jsonl_datasource(ds_config: JsonlDataSourceConfig, endpoint_name: str):
+    dir_path = Path(ds_config.jsonl_dir)
+    if endpoint_name == "remote":
+        dir_path = _prepare_remote_jsonl_dir(dir_path)
+    elif not dir_path.exists() or not dir_path.is_dir():
+        raise FileNotFoundError(
+            f"{endpoint_name}_datasource.jsonl_dir does not exist or is not a directory: {dir_path}"
+        )
+    return JsonlDataSource(dir_path, read_only=ds_config.read_only)
+
+
+def _build_db_datasource(ds_config: DbDataSourceConfig, endpoint_name: str):
+    del ds_config, endpoint_name
+    return DbDataSource()
+
+
+async def _build_api_datasource(ds_config: ApiDataSourceConfig, endpoint_name: str):
+    del endpoint_name
     datasource = ApiDataSource(
         base_url=ds_config.api_base_url,
         uid=ds_config.api_uid,
@@ -183,6 +200,18 @@ async def _create_datasource(
     return datasource
 
 
+def _ensure_builtin_datasource_factories_registered() -> None:
+    if get_datasource_factory("jsonl") is None:
+        register_datasource_factory("jsonl", _build_jsonl_datasource)
+    if get_datasource_factory("db") is None:
+        register_datasource_factory("db", _build_db_datasource)
+    if get_datasource_factory("api") is None:
+        register_datasource_factory("api", _build_api_datasource)
+
+
+_ensure_builtin_datasource_factories_registered()
+
+
 def _register_handlers_for_endpoint(
     *,
     datasource,