整理了初始化过程,现在更容易被继承了。

This commit is contained in:
strepsiades
2026-03-18 16:48:53 +08:00
parent 84b3b7652a
commit 841986740c
22 changed files with 432 additions and 277 deletions
+3 -3
View File
@@ -15,10 +15,10 @@ from .common.registry import DomainRegistry
from .config import (
ApiDataSourceConfig,
BaseDataSourceConfig,
DbDataSourceConfig,
JsonlDataSourceConfig,
PipelineRunConfig,
build_config,
build_config_from_file,
build_default_config,
get_datasource_config_model,
get_datasource_factory,
@@ -40,7 +40,7 @@ except PackageNotFoundError:
def ensure_domain_registry_loaded() -> None:
"""Ensure built-in domain registrations are imported."""
_domain # keep module import for side effects
_ = _domain
__all__ = [
@@ -50,9 +50,9 @@ __all__ = [
"PipelineRunConfig",
"ApiDataSourceConfig",
"BaseDataSourceConfig",
"DbDataSourceConfig",
"JsonlDataSourceConfig",
"build_config",
"build_config_from_file",
"build_default_config",
"get_datasource_config_model",
"get_datasource_factory",
+3 -2
View File
@@ -1,4 +1,4 @@
from .datasource_config import ApiDataSourceConfig, BaseDataSourceConfig, DbDataSourceConfig, JsonlDataSourceConfig, DataSourceConfig
from .datasource_config import ApiDataSourceConfig, BaseDataSourceConfig, JsonlDataSourceConfig, DataSourceConfig
from .persist_config import PersistConfig
from .logging_config import LoggingConfig, resolve_log_file_path
from ..datasource.type_registry import (
@@ -27,6 +27,7 @@ from .pipeline_config import PipelineRunConfig, DEFAULT_NODE_TYPES
from .builder import (
build_default_config,
build_config,
build_config_from_file,
apply_config_overrides,
apply_env_overrides,
config_snapshot,
@@ -35,7 +36,6 @@ from .builder import (
__all__ = [
"ApiDataSourceConfig",
"BaseDataSourceConfig",
"DbDataSourceConfig",
"JsonlDataSourceConfig",
"PipelineRunConfig",
"DataSourceConfig",
@@ -61,6 +61,7 @@ __all__ = [
"DEFAULT_NODE_TYPES",
"build_default_config",
"build_config",
"build_config_from_file",
"apply_config_overrides",
"apply_env_overrides",
"config_snapshot",
+8
View File
@@ -187,3 +187,11 @@ def build_config(
config = apply_config_overrides(config, overrides)
return config
def build_config_from_file(
*,
project_root: Path,
file_path: Union[str, Path],
) -> PipelineRunConfig:
return build_config(project_root=project_root, overrides_file=Path(file_path))
@@ -22,11 +22,6 @@ class JsonlDataSourceConfig(BaseDataSourceConfig):
read_only: bool = False
class DbDataSourceConfig(BaseDataSourceConfig):
type: Literal["db"] = "db"
class ApiDataSourceConfig(BaseDataSourceConfig):
type: Literal["api"] = "api"
@@ -52,5 +47,4 @@ DataSourceConfig = Annotated[
register_datasource_config_model("jsonl", JsonlDataSourceConfig)
register_datasource_config_model("db", DbDataSourceConfig)
register_datasource_config_model("api", ApiDataSourceConfig)
@@ -38,7 +38,6 @@ class PipelineRunConfig(BaseModel):
model_config = ConfigDict(extra="forbid", validate_assignment=True)
node_types: List[str] = Field(default_factory=lambda: list(DEFAULT_NODE_TYPES))
target_project_ids: List[str] = Field(default_factory=list)
local_datasource: DataSourceConfig
remote_datasource: DataSourceConfig
+2 -39
View File
@@ -1,7 +1,6 @@
from __future__ import annotations
import json
import importlib
from pathlib import Path
from typing import Any, Dict, List, Tuple
@@ -29,43 +28,6 @@ def _replace_placeholders(value: Any, project_root: Path) -> Any:
return value
def _import_from_path(import_path: str) -> Any:
module_path, sep, attr_path = import_path.partition(":")
if not module_path:
raise ValueError(f"Invalid import path: {import_path}")
module = importlib.import_module(module_path)
if not sep:
return module
obj: Any = module
for attr_name in attr_path.split("."):
obj = getattr(obj, attr_name)
return obj
def _apply_datasource_bootstraps(payload: Dict[str, Any]) -> Dict[str, Any]:
extensions = payload.pop("extensions", None)
if extensions is None:
return payload
if not isinstance(extensions, dict):
raise ValueError("extensions must be an object mapping")
bootstrap_paths = extensions.get("datasource_bootstraps") or []
if not isinstance(bootstrap_paths, list):
raise ValueError("extensions.datasource_bootstraps must be a list")
for item in bootstrap_paths:
if not isinstance(item, str) or not item.strip():
raise ValueError("extensions.datasource_bootstraps entries must be non-empty strings")
loaded = _import_from_path(item)
if not callable(loaded):
raise TypeError(f"datasource bootstrap must resolve to a callable: {item}")
loaded()
return payload
def list_preset_names() -> List[str]:
return sorted(PRESET_FILES.keys())
@@ -115,7 +77,8 @@ def load_overrides_from_file(file_path: Path, project_root: Path) -> Dict[str, A
resolved = _replace_placeholders(payload, project_root)
if not isinstance(resolved, dict):
raise ValueError(f"Preset file must contain object mapping: {file_path}")
return _apply_datasource_bootstraps(resolved)
resolved.pop("extensions", None)
return resolved
def load_named_preset_overrides(project_root: Path, preset_name: str) -> Tuple[Dict[str, Any], Path, bool]:
+3 -2
View File
@@ -3,7 +3,7 @@
Prefer importing from `sync_state_machine.config` directly.
"""
from .datasource_config import ApiDataSourceConfig, DbDataSourceConfig, JsonlDataSourceConfig, DataSourceConfig
from .datasource_config import ApiDataSourceConfig, JsonlDataSourceConfig, DataSourceConfig
from .persist_config import PersistConfig
from .logging_config import LoggingConfig
from .strategy_config import (
@@ -17,6 +17,7 @@ from .pipeline_config import PipelineRunConfig, DEFAULT_NODE_TYPES
from .builder import (
build_default_config,
build_config,
build_config_from_file,
apply_config_overrides,
apply_env_overrides,
config_snapshot,
@@ -24,7 +25,6 @@ from .builder import (
__all__ = [
"ApiDataSourceConfig",
"DbDataSourceConfig",
"JsonlDataSourceConfig",
"DataSourceConfig",
"PersistConfig",
@@ -38,6 +38,7 @@ __all__ = [
"DEFAULT_NODE_TYPES",
"build_default_config",
"build_config",
"build_config_from_file",
"apply_config_overrides",
"apply_env_overrides",
"config_snapshot",
+17 -4
View File
@@ -23,13 +23,25 @@ from .handler import (
from .task_result import TaskResult, TaskStatus
# API DataSource
from .api import ApiClient, ApiDataSource, BaseApiHandler
from .api import ApiClient, ApiDataSource, BaseApiHandler, register as register_api_datasource
# DB DataSource
# DB base helpers
from .db import DbDataSource, BaseDbHandler
# JSONL DataSource
from .jsonl import JsonlDataSource, BaseJsonlHandler
from .jsonl import JsonlDataSource, BaseJsonlHandler, register as register_jsonl_datasource
_BUILTIN_DATASOURCE_TYPES_REGISTERED = False
def ensure_builtin_datasource_types_registered(*, replace: bool = False) -> None:
global _BUILTIN_DATASOURCE_TYPES_REGISTERED
if _BUILTIN_DATASOURCE_TYPES_REGISTERED and not replace:
return
register_jsonl_datasource(replace=replace)
register_api_datasource(replace=replace)
_BUILTIN_DATASOURCE_TYPES_REGISTERED = True
__all__ = [
# 基础类
@@ -46,13 +58,14 @@ __all__ = [
"TaskResult",
"TaskStatus",
"ValidationResult",
"ensure_builtin_datasource_types_registered",
# API DataSource
"ApiClient",
"ApiDataSource",
"BaseApiHandler",
# DB DataSource
# DB base helpers
"DbDataSource",
"BaseDbHandler",
@@ -1,15 +1,50 @@
from __future__ import annotations
"""
API DataSource 模块
提供基于 HTTP API 的数据源实现。
"""
from typing import TYPE_CHECKING
from ..type_registry import register_datasource_type
from .client import ApiClient
from .datasource import ApiDataSource
from .handler import BaseApiHandler
if TYPE_CHECKING:
from ...config.datasource_config import ApiDataSourceConfig
def build_api_datasource(config: ApiDataSourceConfig, endpoint_name: str, **_: object):
del endpoint_name
return ApiDataSource(
base_url=config.api_base_url,
uid=config.api_uid,
secret=config.api_secret,
debug=config.api_debug,
poll_max_retries=config.poll_max_retries,
poll_interval=config.poll_interval,
rate_limit_max_requests=config.api_rate_limit_max_requests,
rate_limit_window_seconds=config.api_rate_limit_window_seconds,
)
def register(*, replace: bool = False) -> None:
from ...config.datasource_config import ApiDataSourceConfig
register_datasource_type(
"api",
config_model=ApiDataSourceConfig,
factory=build_api_datasource,
replace=replace,
)
__all__ = [
"ApiClient",
"ApiDataSource",
"BaseApiHandler",
"build_api_datasource",
"register",
]
+12 -1
View File
@@ -88,6 +88,13 @@ class BaseDataSource(ABC):
self._poll_interval = max(0, poll_interval)
self._sm_runtime: Optional[StateMachineRuntime] = None
def set_target_project_ids(self, target_project_ids: List[str]) -> None:
self._target_project_ids = [str(item) for item in target_project_ids if str(item)]
@property
def target_project_ids(self) -> List[str]:
return list(self._target_project_ids)
def _ensure_sm_runtime(self) -> StateMachineRuntime:
if self._sm_runtime is None:
cfg_path = Path(__file__).resolve().parents[1] / "config" / "node_state_machine.yaml"
@@ -957,6 +964,10 @@ class BaseDataSource(ABC):
return summary
async def initialize(self) -> None:
"""初始化数据源资源(子类可覆盖)"""
return None
async def close(self) -> None:
"""关闭数据源资源(子类可覆盖以实现特定的清理逻辑)"""
pass # 基类默认空实现
return None
+3 -1
View File
@@ -1,4 +1,6 @@
"""DB DataSource 模块。"""
from __future__ import annotations
"""DB DataSource 基类模块。"""
from .datasource import DbDataSource
from .handler import BaseDbHandler
@@ -1,13 +1,53 @@
from __future__ import annotations
"""
JSONL DataSource 模块
提供基于 JSONL 文件的数据源实现,用于测试和本地开发。
"""
from pathlib import Path
from typing import TYPE_CHECKING
from ..type_registry import register_datasource_type
from .datasource import JsonlDataSource
from .handler import BaseJsonlHandler
from .path_utils import prepare_remote_jsonl_dir
if TYPE_CHECKING:
from ...config.datasource_config import JsonlDataSourceConfig
def build_jsonl_datasource(
config: JsonlDataSourceConfig,
endpoint_name: str,
*,
project_root: Path | None = None,
):
dir_path = Path(config.jsonl_dir)
if endpoint_name == "remote":
dir_path = prepare_remote_jsonl_dir(dir_path, project_root=project_root or Path.cwd())
elif not dir_path.exists() or not dir_path.is_dir():
raise FileNotFoundError(
f"{endpoint_name}_datasource.jsonl_dir does not exist or is not a directory: {dir_path}"
)
return JsonlDataSource(dir_path, read_only=config.read_only)
def register(*, replace: bool = False) -> None:
from ...config.datasource_config import JsonlDataSourceConfig
register_datasource_type(
"jsonl",
config_model=JsonlDataSourceConfig,
factory=build_jsonl_datasource,
replace=replace,
)
__all__ = [
"JsonlDataSource",
"BaseJsonlHandler",
"build_jsonl_datasource",
"prepare_remote_jsonl_dir",
"register",
]
@@ -50,24 +50,22 @@ class BaseJsonlHandler(BaseNodeHandler):
):
super().__init__(node_type, node_class, schema)
self.datasource = datasource
self._target_project_ids: Optional[Set[str]] = None
self._target_project_ids: List[str] = []
def set_target_project_ids(self, target_project_ids: List[str]) -> None:
"""注入项目过滤列表,供 load 阶段预过滤使用"""
if target_project_ids:
self._target_project_ids = {str(pid) for pid in target_project_ids if str(pid)}
else:
self._target_project_ids = None
"""注入项目过滤列表的本地副本。真正的过滤来源仍然是 datasource"""
self._target_project_ids = [str(pid) for pid in target_project_ids if str(pid)]
def _should_skip_item_by_project_filter(self, item: Dict[str, Any], item_id: str) -> bool:
if not self._target_project_ids:
target_project_ids = set(self.datasource.target_project_ids)
if not target_project_ids:
return False
if self.node_type == "project":
return item_id not in self._target_project_ids
return item_id not in target_project_ids
project_id = item.get("project_id")
if project_id:
return str(project_id) not in self._target_project_ids
return str(project_id) not in target_project_ids
return False
async def load(self) -> List['SyncNode']:
@@ -0,0 +1,31 @@
from __future__ import annotations
import logging
from pathlib import Path
logger = logging.getLogger(__name__)
def prepare_remote_jsonl_dir(remote_dir: Path, *, project_root: Path) -> Path:
project_root_path = project_root.resolve()
resolved_remote_dir = remote_dir.resolve(strict=False)
if resolved_remote_dir.exists():
if not resolved_remote_dir.is_dir():
raise FileNotFoundError(
f"remote_datasource.jsonl_dir exists but is not a directory: {resolved_remote_dir}"
)
return resolved_remote_dir
try:
resolved_remote_dir.relative_to(project_root_path)
except ValueError as exc:
raise FileNotFoundError(
"remote_datasource.jsonl_dir does not exist and auto-create is only allowed "
f"under project root: {resolved_remote_dir} (project_root={project_root_path})"
) from exc
resolved_remote_dir.mkdir(parents=True, exist_ok=True)
logger.info("Created missing remote JSONL directory: %s", resolved_remote_dir)
return resolved_remote_dir
+4 -2
View File
@@ -7,7 +7,6 @@ from .full_sync_pipeline import FullSyncPipeline
from ..config import (
PipelineRunConfig,
ApiDataSourceConfig,
DbDataSourceConfig,
JsonlDataSourceConfig,
DataSourceConfig,
PersistConfig,
@@ -22,7 +21,9 @@ from .factory import (
create_jsonl_to_api_pipeline,
create_api_to_jsonl_pipeline,
create_jsonl_to_jsonl_pipeline,
create_pipeline_from_file,
create_pipeline_from_config,
run_pipeline_from_file,
run_pipeline_from_config,
)
@@ -32,11 +33,12 @@ __all__ = [
"create_jsonl_to_api_pipeline",
"create_jsonl_to_jsonl_pipeline",
"create_api_to_jsonl_pipeline",
"create_pipeline_from_file",
"create_pipeline_from_config",
"run_pipeline_from_file",
"run_pipeline_from_config",
"PipelineRunConfig",
"ApiDataSourceConfig",
"DbDataSourceConfig",
"JsonlDataSourceConfig",
"DataSourceConfig",
"PersistConfig",
+86 -120
View File
@@ -23,19 +23,17 @@ from ..common.binding import BindingManager
from ..common.collection import DataCollection
from ..common.persistence import PersistenceBackend
from ..common.registry import DomainRegistry
from ..datasource.api.datasource import ApiDataSource
from ..datasource.db.datasource import DbDataSource
from ..datasource.jsonl.datasource import JsonlDataSource
from ..datasource.type_registry import get_datasource_factory, register_datasource_factory
from ..datasource import ensure_builtin_datasource_types_registered
from ..datasource.type_registry import get_datasource_factory
from ..config import (
PipelineRunConfig,
DataSourceConfig,
ApiDataSourceConfig,
DbDataSourceConfig,
JsonlDataSourceConfig,
PersistConfig,
JsonlDataSourceConfig,
LoggingConfig,
StrategyRuntimeConfig,
build_config_from_file,
apply_config_overrides,
OrphanAction,
UpdateDirection,
@@ -55,27 +53,9 @@ def _project_root() -> Path:
def _prepare_remote_jsonl_dir(remote_dir: Path, *, project_root: Optional[Path] = None) -> Path:
project_root_path = (project_root or _project_root()).resolve()
resolved_remote_dir = remote_dir.resolve(strict=False)
from ..datasource.jsonl.path_utils import prepare_remote_jsonl_dir
if resolved_remote_dir.exists():
if not resolved_remote_dir.is_dir():
raise FileNotFoundError(
f"remote_datasource.jsonl_dir exists but is not a directory: {resolved_remote_dir}"
)
return resolved_remote_dir
try:
resolved_remote_dir.relative_to(project_root_path)
except ValueError as exc:
raise FileNotFoundError(
"remote_datasource.jsonl_dir does not exist and auto-create is only allowed "
f"under project root: {resolved_remote_dir} (project_root={project_root_path})"
) from exc
resolved_remote_dir.mkdir(parents=True, exist_ok=True)
logger.info("Created missing remote JSONL directory: %s", resolved_remote_dir)
return resolved_remote_dir
return prepare_remote_jsonl_dir(remote_dir, project_root=(project_root or _project_root()))
def _ensure_pipeline_logger(
@@ -104,13 +84,8 @@ def _ensure_pipeline_logger(
def _build_handler_config(
node_type: str,
ds_config: DataSourceConfig,
target_project_ids: List[str],
) -> Dict[str, Any]:
config = dict(ds_config.handler_configs.get(node_type, {}))
if node_type == "project" and target_project_ids:
config.setdefault("filter_project_id", target_project_ids)
return config
return dict(ds_config.handler_configs.get(node_type, {}))
def _create_handler(
@@ -118,7 +93,6 @@ def _create_handler(
node_type: str,
ds_config: DataSourceConfig,
datasource,
target_project_ids: List[str],
):
datasource_type = ds_config.type
handler_class = DomainRegistry.get_handler(node_type, datasource_type)
@@ -127,7 +101,7 @@ def _create_handler(
f"Missing {datasource_type.upper()} handler for node_type: {node_type}"
)
handler_config = _build_handler_config(node_type, ds_config, target_project_ids)
handler_config = _build_handler_config(node_type, ds_config)
if datasource_type == "api":
handler = handler_class(**handler_config)
else:
@@ -138,22 +112,12 @@ def _create_handler(
return handler
def _resolve_datasource_target_project_ids(config: PipelineRunConfig) -> tuple[List[str], List[str]]:
fallback = [str(pid) for pid in config.target_project_ids if str(pid)]
local_cfg_ids = [str(pid) for pid in config.local_datasource.target_project_ids if str(pid)]
remote_cfg_ids = [str(pid) for pid in config.remote_datasource.target_project_ids if str(pid)]
local_target_ids = local_cfg_ids if local_cfg_ids else fallback
remote_target_ids = remote_cfg_ids if remote_cfg_ids else fallback
return local_target_ids, remote_target_ids
async def _create_datasource(
ds_config: DataSourceConfig,
*,
datasource_override=None,
endpoint_name: str,
project_root: Path | None = None,
):
if datasource_override is not None:
return datasource_override
@@ -162,54 +126,24 @@ async def _create_datasource(
if datasource_factory is None:
raise RuntimeError(f"No datasource factory registered for type: {ds_config.type}")
datasource = datasource_factory(ds_config, endpoint_name)
factory_kwargs: dict[str, Any] = {}
signature = inspect.signature(datasource_factory)
if "project_root" in signature.parameters:
factory_kwargs["project_root"] = project_root or _project_root()
datasource = datasource_factory(ds_config, endpoint_name, **factory_kwargs)
if inspect.isawaitable(datasource):
datasource = await datasource
return datasource
def _build_jsonl_datasource(ds_config: JsonlDataSourceConfig, endpoint_name: str):
dir_path = Path(ds_config.jsonl_dir)
if endpoint_name == "remote":
dir_path = _prepare_remote_jsonl_dir(dir_path)
elif not dir_path.exists() or not dir_path.is_dir():
raise FileNotFoundError(
f"{endpoint_name}_datasource.jsonl_dir does not exist or is not a directory: {dir_path}"
)
return JsonlDataSource(dir_path, read_only=ds_config.read_only)
async def _initialize_datasource(datasource, *, endpoint_name: str) -> None:
try:
await datasource.initialize()
except Exception as exc:
raise RuntimeError(f"Failed to initialize {endpoint_name} datasource: {type(exc).__name__}: {exc}") from exc
def _build_db_datasource(ds_config: DbDataSourceConfig, endpoint_name: str):
del ds_config, endpoint_name
return DbDataSource()
async def _build_api_datasource(ds_config: ApiDataSourceConfig, endpoint_name: str):
del endpoint_name
datasource = ApiDataSource(
base_url=ds_config.api_base_url,
uid=ds_config.api_uid,
secret=ds_config.api_secret,
debug=ds_config.api_debug,
poll_max_retries=ds_config.poll_max_retries,
poll_interval=ds_config.poll_interval,
rate_limit_max_requests=ds_config.api_rate_limit_max_requests,
rate_limit_window_seconds=ds_config.api_rate_limit_window_seconds,
)
await datasource.initialize()
return datasource
def _ensure_builtin_datasource_factories_registered() -> None:
if get_datasource_factory("jsonl") is None:
register_datasource_factory("jsonl", _build_jsonl_datasource)
if get_datasource_factory("db") is None:
register_datasource_factory("db", _build_db_datasource)
if get_datasource_factory("api") is None:
register_datasource_factory("api", _build_api_datasource)
_ensure_builtin_datasource_factories_registered()
ensure_builtin_datasource_types_registered()
def _register_handlers_for_endpoint(
@@ -217,7 +151,6 @@ def _register_handlers_for_endpoint(
datasource,
ds_config: DataSourceConfig,
node_types: List[str],
target_project_ids: List[str],
) -> None:
for node_type in node_types:
datasource.register_handler(
@@ -225,7 +158,6 @@ def _register_handlers_for_endpoint(
node_type=node_type,
ds_config=ds_config,
datasource=datasource,
target_project_ids=target_project_ids,
)
)
@@ -240,13 +172,11 @@ def _register_handlers(
datasource=local_ds,
ds_config=config.local_datasource,
node_types=all_types,
target_project_ids=[str(pid) for pid in config.local_datasource.target_project_ids if str(pid)],
)
_register_handlers_for_endpoint(
datasource=remote_ds,
ds_config=config.remote_datasource,
node_types=all_types,
target_project_ids=[str(pid) for pid in config.remote_datasource.target_project_ids if str(pid)],
)
@@ -318,7 +248,6 @@ async def create_pipeline_from_config(
all_types = list(config.node_types)
strategy_map = _resolve_strategy_map(config, all_types)
local_target_project_ids, remote_target_project_ids = _resolve_datasource_target_project_ids(config)
if config.persist.backend != "sqlite":
raise NotImplementedError(f"Persistence backend not supported yet: {config.persist.backend}")
@@ -338,31 +267,20 @@ async def create_pipeline_from_config(
config.local_datasource,
datasource_override=local_datasource_override,
endpoint_name="local",
project_root=_project_root(),
)
await _initialize_datasource(local_ds, endpoint_name="local")
remote_ds = await _create_datasource(
config.remote_datasource,
datasource_override=remote_datasource_override,
endpoint_name="remote",
project_root=_project_root(),
)
local_cfg_with_target = config.local_datasource.model_copy(
update={"target_project_ids": list(local_target_project_ids)},
deep=True,
)
remote_cfg_with_target = config.remote_datasource.model_copy(
update={"target_project_ids": list(remote_target_project_ids)},
deep=True,
)
handler_target_config = config.model_copy(
update={
"local_datasource": local_cfg_with_target,
"remote_datasource": remote_cfg_with_target,
},
deep=True,
)
await _initialize_datasource(remote_ds, endpoint_name="remote")
local_ds.set_target_project_ids([str(pid) for pid in config.local_datasource.target_project_ids if str(pid)])
remote_ds.set_target_project_ids([str(pid) for pid in config.remote_datasource.target_project_ids if str(pid)])
_register_handlers(handler_target_config, local_ds, remote_ds, all_types)
local_ds.set_target_project_ids(local_target_project_ids)
remote_ds.set_target_project_ids(remote_target_project_ids)
_register_handlers(config, local_ds, remote_ds, all_types)
local_collection = DataCollection("local", persistence, auto_persist=False)
remote_collection = DataCollection("remote", persistence, auto_persist=False)
@@ -387,9 +305,6 @@ async def create_pipeline_from_config(
remote_collection=remote_collection,
binding_manager=binding_manager,
enable_persistence=config.persist.enable,
target_project_ids=config.target_project_ids,
local_target_project_ids=local_target_project_ids,
remote_target_project_ids=remote_target_project_ids,
)
return pipeline
@@ -474,6 +389,40 @@ async def run_pipeline_from_config(
await pipeline.close()
async def create_pipeline_from_file(
*,
project_root: Path,
config_path: str | Path,
local_datasource_override=None,
remote_datasource_override=None,
) -> FullSyncPipeline:
config = build_config_from_file(project_root=project_root, file_path=config_path)
return await create_pipeline_from_config(
config,
local_datasource_override=local_datasource_override,
remote_datasource_override=remote_datasource_override,
)
async def run_pipeline_from_file(
*,
project_root: Path,
config_path: str | Path,
title: str = "sync_state_machine pipeline",
print_summary: bool = True,
local_datasource_override=None,
remote_datasource_override=None,
) -> Dict[str, Any]:
config = build_config_from_file(project_root=project_root, file_path=config_path)
return await run_pipeline_from_config(
config,
title=title,
print_summary=print_summary,
local_datasource_override=local_datasource_override,
remote_datasource_override=remote_datasource_override,
)
async def create_jsonl_to_api_pipeline(
*,
# 路径配置
@@ -506,8 +455,12 @@ async def create_jsonl_to_api_pipeline(
) -> FullSyncPipeline:
cfg = PipelineRunConfig(
node_types=node_types,
target_project_ids=target_project_ids or [],
local_datasource=JsonlDataSourceConfig(type="jsonl", jsonl_dir=local_jsonl_dir, read_only=False),
local_datasource=JsonlDataSourceConfig(
type="jsonl",
jsonl_dir=local_jsonl_dir,
read_only=False,
target_project_ids=target_project_ids or [],
),
remote_datasource=ApiDataSourceConfig(
type="api",
api_base_url=api_base_url,
@@ -570,7 +523,6 @@ async def create_api_to_jsonl_pipeline(
) -> FullSyncPipeline:
cfg = PipelineRunConfig(
node_types=node_types,
target_project_ids=target_project_ids or [],
local_datasource=ApiDataSourceConfig(
type="api",
api_base_url=api_base_url,
@@ -584,7 +536,12 @@ async def create_api_to_jsonl_pipeline(
target_project_ids=target_project_ids or [],
handler_configs=handler_configs or {},
),
remote_datasource=JsonlDataSourceConfig(type="jsonl", jsonl_dir=remote_jsonl_dir, read_only=False),
remote_datasource=JsonlDataSourceConfig(
type="jsonl",
jsonl_dir=remote_jsonl_dir,
read_only=False,
target_project_ids=target_project_ids or [],
),
strategies=[],
persist=PersistConfig(
db_path=persistence_db,
@@ -618,9 +575,18 @@ async def create_jsonl_to_jsonl_pipeline(
) -> FullSyncPipeline:
cfg = PipelineRunConfig(
node_types=node_types,
target_project_ids=target_project_ids or [],
local_datasource=JsonlDataSourceConfig(type="jsonl", jsonl_dir=local_jsonl_dir, read_only=False),
remote_datasource=JsonlDataSourceConfig(type="jsonl", jsonl_dir=remote_jsonl_dir, read_only=False),
local_datasource=JsonlDataSourceConfig(
type="jsonl",
jsonl_dir=local_jsonl_dir,
read_only=False,
target_project_ids=target_project_ids or [],
),
remote_datasource=JsonlDataSourceConfig(
type="jsonl",
jsonl_dir=remote_jsonl_dir,
read_only=False,
target_project_ids=target_project_ids or [],
),
strategies=[],
persist=PersistConfig(
db_path=persistence_db,
@@ -51,10 +51,6 @@ class FullSyncPipeline:
load_order: Optional[List[str]] = None,
sync_order: Optional[List[str]] = None,
enable_persistence: bool = True,
# 项目过滤
target_project_ids: Optional[List[str]] = None,
local_target_project_ids: Optional[List[str]] = None,
remote_target_project_ids: Optional[List[str]] = None,
):
# 验证必需参数(由工厂函数创建)
if strategies is None:
@@ -70,17 +66,6 @@ class FullSyncPipeline:
self.sync_order = sync_order or self.node_types
self.persistence = persistence
self.enable_persistence = enable_persistence
self.target_project_ids = target_project_ids or []
self.local_target_project_ids = (
list(local_target_project_ids)
if local_target_project_ids is not None
else list(self.target_project_ids)
)
self.remote_target_project_ids = (
list(remote_target_project_ids)
if remote_target_project_ids is not None
else list(self.target_project_ids)
)
self.local_collection = local_collection
self.remote_collection = remote_collection
@@ -187,12 +172,6 @@ class FullSyncPipeline:
await self._load_from_datasource()
self._logger.info("✅ Bootstrap: datasource fetched")
if self.local_target_project_ids or self.remote_target_project_ids:
local_deleted = await self.local_collection.filter_by_project_ids(self.local_target_project_ids)
remote_deleted = await self.remote_collection.filter_by_project_ids(self.remote_target_project_ids)
if local_deleted > 0 or remote_deleted > 0:
self._logger.info(f"🔍 Project filtering: deleted {local_deleted} local, {remote_deleted} remote nodes")
async def phase_bind(self) -> None:
strategy_map = {s.node_type: s for s in self.strategies}
for node_type in self.sync_order:
@@ -1,28 +1,23 @@
from __future__ import annotations
from pydantic import ValidationError
import pytest
from sync_state_machine.config import (
PipelineRunConfig,
JsonlDataSourceConfig,
ApiDataSourceConfig,
DbDataSourceConfig,
PersistConfig,
LoggingConfig,
)
from sync_state_machine.pipeline.factory import _resolve_datasource_target_project_ids
def _build_config(
*,
top_level: list[str],
local_ids: list[str] | None = None,
remote_ids: list[str] | None = None,
) -> PipelineRunConfig:
return PipelineRunConfig(
node_types=["project"],
target_project_ids=top_level,
local_datasource=JsonlDataSourceConfig(
type="jsonl",
jsonl_dir="./filtered_datasource/datasource/filtered",
@@ -39,49 +34,22 @@ def _build_config(
)
def test_datasource_target_ids_take_precedence_over_top_level() -> None:
cfg = _build_config(top_level=["p_top"], local_ids=["p_local"], remote_ids=["p_remote"])
def test_pipeline_config_has_no_top_level_target_project_ids() -> None:
cfg = _build_config(local_ids=["p_local"], remote_ids=["p_remote"])
local_ids, remote_ids = _resolve_datasource_target_project_ids(cfg)
assert local_ids == ["p_local"]
assert remote_ids == ["p_remote"]
assert cfg.local_datasource.target_project_ids == ["p_local"]
assert cfg.remote_datasource.target_project_ids == ["p_remote"]
with pytest.raises(AttributeError):
_ = cfg.target_project_ids
def test_datasource_target_ids_fallback_to_top_level_when_missing() -> None:
cfg = _build_config(top_level=["p_top"], local_ids=[], remote_ids=[])
def test_api_datasource_allows_empty_target_project_ids() -> None:
cfg = ApiDataSourceConfig(
type="api",
api_base_url="https://example.com",
)
local_ids, remote_ids = _resolve_datasource_target_project_ids(cfg)
assert local_ids == ["p_top"]
assert remote_ids == ["p_top"]
def test_datasource_target_ids_support_mixed_override_and_fallback() -> None:
cfg = _build_config(top_level=["p_top"], local_ids=["p_local"], remote_ids=[])
local_ids, remote_ids = _resolve_datasource_target_project_ids(cfg)
assert local_ids == ["p_local"]
assert remote_ids == ["p_top"]
def test_api_datasource_requires_target_project_ids() -> None:
with pytest.raises(ValidationError) as exc_info:
ApiDataSourceConfig(
type="api",
api_base_url="https://example.com",
target_project_ids=[], # Empty list should fail
)
assert "List should have at least 1 item" in str(exc_info.value)
with pytest.raises(ValidationError) as exc_info:
ApiDataSourceConfig(
type="api",
api_base_url="https://example.com",
# Missing target_project_ids should fail
)
assert "Field required" in str(exc_info.value) or "missing" in str(exc_info.value).lower()
assert cfg.target_project_ids == []
def test_api_datasource_rate_limit_config_fields() -> None:
@@ -102,11 +70,3 @@ def test_api_datasource_rate_limit_config_fields() -> None:
)
assert overridden.api_rate_limit_max_requests == 5
assert overridden.api_rate_limit_window_seconds == 2.5
def test_db_datasource_accepts_minimal_config() -> None:
cfg = DbDataSourceConfig(type="db")
assert cfg.type == "db"
assert cfg.target_project_ids == []
assert cfg.handler_configs == {}
+17 -6
View File
@@ -9,14 +9,15 @@ from sync_state_machine.common.registry import DomainRegistry
from sync_state_machine.common.sync_node import SyncNode
from sync_state_machine.config import (
BaseDataSourceConfig,
DbDataSourceConfig,
LoggingConfig,
PersistConfig,
PipelineRunConfig,
build_config,
get_datasource_factory,
register_datasource_type,
)
from sync_state_machine.datasource import BaseDataSource
from sync_state_machine.datasource import ensure_builtin_datasource_types_registered
from sync_state_machine.datasource.db import BaseDbHandler, DbDataSource
from sync_state_machine.pipeline.factory import create_pipeline_from_config
from sync_state_machine.sync_system.strategy import DefaultSyncStrategy
@@ -118,6 +119,13 @@ def _memory_factory(config: MemoryConfig, endpoint_name: str) -> MemoryDataSourc
return MemoryDataSource(namespace=f"{endpoint_name}:{config.namespace}")
def test_builtin_datasource_factories_are_registered() -> None:
ensure_builtin_datasource_types_registered(replace=True)
assert get_datasource_factory("jsonl") is not None
assert get_datasource_factory("api") is not None
@pytest.mark.asyncio
async def test_create_pipeline_from_config_supports_registered_custom_datasource_type(tmp_path: Path) -> None:
register_datasource_type("memory", config_model=MemoryConfig, factory=_memory_factory, replace=True)
@@ -126,7 +134,7 @@ async def test_create_pipeline_from_config_supports_registered_custom_datasource
config = PipelineRunConfig(
node_types=["memory_demo"],
local_datasource=MemoryConfig(type="memory", namespace="alpha"),
remote_datasource=DbDataSourceConfig(type="db"),
remote_datasource=MemoryConfig(type="memory", namespace="omega"),
strategies=[],
persist=PersistConfig(db_path=str(tmp_path / "memory-registry.db")),
logging=LoggingConfig(initialize=False),
@@ -136,8 +144,10 @@ async def test_create_pipeline_from_config_supports_registered_custom_datasource
try:
assert isinstance(pipeline.local_datasource, MemoryDataSource)
assert pipeline.local_datasource.namespace == "local:alpha"
assert isinstance(pipeline.remote_datasource, MemoryDataSource)
assert pipeline.remote_datasource.namespace == "remote:omega"
assert isinstance(pipeline.local_datasource.get_handler("memory_demo"), MemoryHandler)
assert isinstance(pipeline.remote_datasource.get_handler("memory_demo"), MemoryRemoteDbHandler)
assert isinstance(pipeline.remote_datasource.get_handler("memory_demo"), MemoryHandler)
finally:
await pipeline.close()
@@ -176,14 +186,15 @@ def test_build_config_supports_same_file_datasource_bootstrap(tmp_path: Path, mo
profile_path = tmp_path / "custom-memory.yaml"
profile_path.write_text(
"extensions:\n"
" datasource_bootstraps:\n"
" bootstraps:\n"
" - memory_bootstrap_demo:register\n"
"node_types: []\n"
"local_datasource:\n"
" type: memory_bootstrap\n"
" namespace: local-demo\n"
"remote_datasource:\n"
" type: db\n"
" type: jsonl\n"
" jsonl_dir: ${PROJECT_ROOT}/remote_datasource/datasource\n"
"persist:\n"
" db_path: ${PROJECT_ROOT}/runtime/test-memory-bootstrap.db\n"
"logging:\n"
@@ -195,4 +206,4 @@ def test_build_config_supports_same_file_datasource_bootstrap(tmp_path: Path, mo
assert config.local_datasource.type == "memory_bootstrap"
assert getattr(config.local_datasource, "namespace") == "local-demo"
assert config.remote_datasource.type == "db"
assert config.remote_datasource.type == "jsonl"
@@ -6,13 +6,25 @@ from pydantic import BaseModel
from sync_state_machine.common.registry import DomainRegistry
from sync_state_machine.common.sync_node import SyncNode
from sync_state_machine.config import DbDataSourceConfig, JsonlDataSourceConfig, LoggingConfig, PersistConfig, PipelineRunConfig
from sync_state_machine.config import (
BaseDataSourceConfig,
JsonlDataSourceConfig,
LoggingConfig,
PersistConfig,
PipelineRunConfig,
register_datasource_type,
)
from sync_state_machine.datasource.db import BaseDbHandler, DbDataSource
from sync_state_machine.datasource.jsonl import BaseJsonlHandler, JsonlDataSource
from sync_state_machine.pipeline.factory import _register_handlers
from sync_state_machine.sync_system.strategy import DefaultSyncStrategy
class TestRegisteredDbConfig(BaseDataSourceConfig):
__test__ = False
type: str = "test_db"
class TestDbSchema(BaseModel):
__test__ = False
id: str
@@ -56,7 +68,18 @@ class TestJsonlHandler(BaseJsonlHandler):
super().__init__(datasource, "test_db_handler", TestDbNode, TestDbSchema)
def _register_test_db_datasource_type() -> None:
register_datasource_type(
"test_db",
config_model=TestRegisteredDbConfig,
factory=lambda config, endpoint_name: DbDataSource(),
replace=True,
)
def test_register_handlers_uses_db_handler_for_db_datasource(tmp_path: Path) -> None:
_register_test_db_datasource_type()
if not DomainRegistry.is_registered("test_db_handler"):
DomainRegistry.register(
node_type="test_db_handler",
@@ -65,11 +88,11 @@ def test_register_handlers_uses_db_handler_for_db_datasource(tmp_path: Path) ->
strategy_class=TestDbStrategy,
jsonl_handler_class=TestJsonlHandler,
)
DomainRegistry.register_db_handler("test_db_handler", TestDbHandler)
DomainRegistry.register_handler("test_db_handler", "test_db", TestDbHandler)
config = PipelineRunConfig(
node_types=["test_db_handler"],
local_datasource=DbDataSourceConfig(type="db"),
local_datasource=TestRegisteredDbConfig(type="test_db"),
remote_datasource=JsonlDataSourceConfig(
type="jsonl",
jsonl_dir=str(tmp_path),
@@ -7,12 +7,23 @@ from pydantic import BaseModel
from sync_state_machine.common.registry import DomainRegistry
from sync_state_machine.common.sync_node import SyncNode
from sync_state_machine.config import DbDataSourceConfig, LoggingConfig, PersistConfig, PipelineRunConfig
from sync_state_machine.config import (
BaseDataSourceConfig,
LoggingConfig,
PersistConfig,
PipelineRunConfig,
register_datasource_type,
)
from sync_state_machine.datasource.db import BaseDbHandler, DbDataSource
from sync_state_machine.pipeline.factory import create_pipeline_from_config
from sync_state_machine.sync_system.strategy import DefaultSyncStrategy
class EndpointInitDbConfig(BaseDataSourceConfig):
__test__ = False
type: str = "endpoint_init_db"
class EndpointInitSchema(BaseModel):
__test__ = False
id: str
@@ -57,6 +68,7 @@ def _ensure_registered() -> None:
reg.node_class = EndpointInitNode
reg.strategy_class = EndpointInitStrategy
reg.db_handler_class = EndpointInitDbHandler
reg.handler_classes["endpoint_init_db"] = EndpointInitDbHandler
return
DomainRegistry.register(
@@ -66,6 +78,16 @@ def _ensure_registered() -> None:
strategy_class=EndpointInitStrategy,
db_handler_class=EndpointInitDbHandler,
)
DomainRegistry.register_handler(node_type, "endpoint_init_db", EndpointInitDbHandler)
def _register_endpoint_init_datasource_type() -> None:
register_datasource_type(
"endpoint_init_db",
config_model=EndpointInitDbConfig,
factory=lambda config, endpoint_name: DbDataSource(),
replace=True,
)
@pytest.mark.asyncio
@@ -74,16 +96,17 @@ async def test_factory_initializes_same_datasource_type_with_distinct_handler_co
monkeypatch: pytest.MonkeyPatch,
) -> None:
_ensure_registered()
_register_endpoint_init_datasource_type()
config = PipelineRunConfig(
node_types=["factory_endpoint_demo"],
local_datasource=DbDataSourceConfig(
type="db",
local_datasource=EndpointInitDbConfig(
type="endpoint_init_db",
handler_configs={"factory_endpoint_demo": {"side": "local", "batch_size": 10}},
target_project_ids=["project_local"],
),
remote_datasource=DbDataSourceConfig(
type="db",
remote_datasource=EndpointInitDbConfig(
type="endpoint_init_db",
handler_configs={"factory_endpoint_demo": {"side": "remote", "batch_size": 20}},
target_project_ids=["project_remote"],
),
@@ -4,6 +4,7 @@ from pathlib import Path
import pytest
from sync_state_machine.config import build_config_from_file
from sync_state_machine.config.run_presets import load_overrides_from_file
@@ -29,3 +30,97 @@ local_datasource:
assert overrides["logging"]["suppress_node_logs"] is True
assert str(tmp_path) in overrides["local_datasource"]["jsonl_dir"]
def test_load_overrides_executes_generic_extensions_bootstraps(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
bootstrap_module = tmp_path / "profile_bootstrap_demo.py"
bootstrap_module.write_text(
"BOOTSTRAP_CALLED = False\n"
"\n"
"def register():\n"
" global BOOTSTRAP_CALLED\n"
" BOOTSTRAP_CALLED = True\n",
encoding="utf-8",
)
monkeypatch.syspath_prepend(str(tmp_path))
yaml_file = tmp_path / "profile_with_bootstrap.yaml"
yaml_file.write_text(
"""
extensions:
bootstraps:
- profile_bootstrap_demo:register
local_datasource:
type: jsonl
jsonl_dir: ${PROJECT_ROOT}/filtered_datasource/datasource/filtered
remote_datasource:
type: jsonl
jsonl_dir: ${PROJECT_ROOT}/remote_datasource/datasource
persist:
db_path: ${PROJECT_ROOT}/runtime/test.db
logging:
initialize: false
""".strip(),
encoding="utf-8",
)
load_overrides_from_file(yaml_file, project_root=tmp_path)
import profile_bootstrap_demo
assert profile_bootstrap_demo.BOOTSTRAP_CALLED is True
def test_build_config_from_file_uses_registered_bootstraps(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
bootstrap_module = tmp_path / "config_from_file_bootstrap.py"
bootstrap_module.write_text(
"from pydantic import ConfigDict\n"
"from sync_state_machine.config import BaseDataSourceConfig, register_datasource_type\n"
"from sync_state_machine.datasource import BaseDataSource\n"
"\n"
"class FileBackedConfig(BaseDataSourceConfig):\n"
" model_config = ConfigDict(extra=\"forbid\", validate_assignment=True)\n"
" type: str = \"file_backed\"\n"
" namespace: str = \"demo\"\n"
"\n"
"class FileBackedDatasource(BaseDataSource):\n"
" pass\n"
"\n"
"def build_datasource(config, endpoint_name):\n"
" return FileBackedDatasource()\n"
"\n"
"def register():\n"
" register_datasource_type(\n"
" \"file_backed\",\n"
" config_model=FileBackedConfig,\n"
" factory=build_datasource,\n"
" replace=True,\n"
" )\n",
encoding="utf-8",
)
monkeypatch.syspath_prepend(str(tmp_path))
profile = tmp_path / "profile.yaml"
profile.write_text(
"""
extensions:
bootstraps:
- config_from_file_bootstrap:register
local_datasource:
type: file_backed
namespace: local-demo
remote_datasource:
type: jsonl
jsonl_dir: ${PROJECT_ROOT}/remote_datasource/datasource
persist:
db_path: ${PROJECT_ROOT}/runtime/test.db
logging:
initialize: false
""".strip(),
encoding="utf-8",
)
config = build_config_from_file(project_root=tmp_path, file_path=profile)
assert config.local_datasource.type == "file_backed"
assert getattr(config.local_datasource, "namespace") == "local-demo"