strepsiades
649 lines
22 KiB
Python
649 lines
22 KiB
Python
"""
|
||
Pipeline Factory - 工厂函数，负责初始化所有组件并创建 Pipeline
|
||
|
||
用户只需提供配置参数，工厂函数负责：
|
||
1. 初始化 Persistence
|
||
2. 初始化 DataSource（并注册 Handler）
|
||
3. 初始化 Collection 和 BindingManager
|
||
4. 初始化 Strategy（并注入依赖）
|
||
5. 创建 Pipeline（注入所有组件）
|
||
6. 返回完全就绪的 Pipeline
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import inspect
|
||
import json
|
||
import logging
|
||
from pathlib import Path
|
||
from typing import Any, Dict, List, Optional
|
||
|
||
from ..common.binding import BindingManager
|
||
from ..common.collection import DataCollection
|
||
from ..common.persistence import PersistenceBackend, create_persistence_backend
|
||
from ..common.registry import DomainRegistry
|
||
from ..datasource import ensure_builtin_datasource_types_registered
|
||
from ..datasource.type_registry import get_datasource_factory
|
||
from ..config import (
|
||
PipelineRunConfig,
|
||
MultiProjectRunConfig,
|
||
DataSourceConfig,
|
||
ApiDataSourceConfig,
|
||
PersistConfig,
|
||
JsonlDataSourceConfig,
|
||
LoggingConfig,
|
||
build_config,
|
||
StrategyRuntimeConfig,
|
||
build_config_from_file,
|
||
build_multi_project_config_from_file,
|
||
apply_config_overrides,
|
||
OrphanAction,
|
||
UpdateDirection,
|
||
resolve_log_file_path,
|
||
config_snapshot,
|
||
is_multi_project_payload,
|
||
load_overrides_from_file,
|
||
)
|
||
from ..config.strategy_config import resolve_domain_option_config
|
||
from ..sync_system.strategy import BaseSyncStrategy
|
||
from ..logging import configure_app_logging, ensure_app_logging, get_logger
|
||
from .run_logger import init_pipeline_logger
|
||
from .full_sync_pipeline import FullSyncPipeline
|
||
|
||
|
||
logger = get_logger(__name__)
|
||
|
||
|
||
def _project_root() -> Path:
|
||
return Path(__file__).resolve().parents[2]
|
||
|
||
|
||
def _prepare_remote_jsonl_dir(remote_dir: Path, *, project_root: Optional[Path] = None) -> Path:
|
||
from ..datasource.jsonl.path_utils import prepare_remote_jsonl_dir
|
||
|
||
return prepare_remote_jsonl_dir(remote_dir, project_root=(project_root or _project_root()))
|
||
|
||
|
||
def _build_handler_config(
|
||
node_type: str,
|
||
ds_config: DataSourceConfig,
|
||
) -> Dict[str, Any]:
|
||
return dict(ds_config.handler_configs.get(node_type, {}))
|
||
|
||
|
||
def _create_handler(
|
||
*,
|
||
node_type: str,
|
||
ds_config: DataSourceConfig,
|
||
datasource,
|
||
):
|
||
datasource_type = ds_config.type
|
||
handler_class = DomainRegistry.get_handler(node_type, datasource_type)
|
||
if handler_class is None:
|
||
raise RuntimeError(
|
||
f"Missing {datasource_type.upper()} handler for node_type: {node_type}"
|
||
)
|
||
|
||
handler_config = _build_handler_config(node_type, ds_config)
|
||
if datasource_type == "api":
|
||
handler = handler_class(**handler_config)
|
||
else:
|
||
handler = handler_class(datasource)
|
||
|
||
if hasattr(handler, "set_handler_config"):
|
||
handler.set_handler_config(handler_config)
|
||
return handler
|
||
|
||
|
||
async def _create_datasource(
|
||
ds_config: DataSourceConfig,
|
||
*,
|
||
datasource_override=None,
|
||
endpoint_name: str,
|
||
project_root: Path | None = None,
|
||
):
|
||
if datasource_override is not None:
|
||
return datasource_override
|
||
|
||
datasource_factory = get_datasource_factory(ds_config.type)
|
||
if datasource_factory is None:
|
||
raise RuntimeError(f"No datasource factory registered for type: {ds_config.type}")
|
||
|
||
factory_kwargs: dict[str, Any] = {}
|
||
signature = inspect.signature(datasource_factory)
|
||
if "project_root" in signature.parameters:
|
||
factory_kwargs["project_root"] = project_root or _project_root()
|
||
|
||
datasource = datasource_factory(ds_config, endpoint_name, **factory_kwargs)
|
||
if inspect.isawaitable(datasource):
|
||
datasource = await datasource
|
||
return datasource
|
||
|
||
|
||
async def _initialize_datasource(datasource, *, endpoint_name: str) -> None:
|
||
try:
|
||
await datasource.initialize()
|
||
except Exception as exc:
|
||
raise RuntimeError(f"Failed to initialize {endpoint_name} datasource: {type(exc).__name__}: {exc}") from exc
|
||
|
||
ensure_builtin_datasource_types_registered()
|
||
|
||
|
||
def _add_handlers_for_endpoint(
|
||
*,
|
||
datasource,
|
||
ds_config: DataSourceConfig,
|
||
node_types: List[str],
|
||
) -> None:
|
||
for node_type in node_types:
|
||
datasource.add_handler(
|
||
_create_handler(
|
||
node_type=node_type,
|
||
ds_config=ds_config,
|
||
datasource=datasource,
|
||
)
|
||
)
|
||
|
||
|
||
def _add_handlers(
|
||
config: PipelineRunConfig,
|
||
local_ds,
|
||
remote_ds,
|
||
all_types: List[str],
|
||
) -> None:
|
||
_add_handlers_for_endpoint(
|
||
datasource=local_ds,
|
||
ds_config=config.local_datasource,
|
||
node_types=all_types,
|
||
)
|
||
_add_handlers_for_endpoint(
|
||
datasource=remote_ds,
|
||
ds_config=config.remote_datasource,
|
||
node_types=all_types,
|
||
)
|
||
|
||
|
||
def _build_collections_and_bindings(
|
||
*,
|
||
persistence: PersistenceBackend,
|
||
scope: str,
|
||
) -> tuple[DataCollection, DataCollection, BindingManager]:
|
||
local_collection = DataCollection("local", scope=scope, persistence=persistence, auto_persist=False)
|
||
remote_collection = DataCollection("remote", scope=scope, persistence=persistence, auto_persist=False)
|
||
binding_manager = BindingManager(persistence, auto_persist=False, scope=scope)
|
||
return local_collection, remote_collection, binding_manager
|
||
|
||
|
||
def _resolve_strategy_map(config: PipelineRunConfig, node_types: List[str]) -> Dict[str, StrategyRuntimeConfig]:
|
||
if config.strategies:
|
||
resolved_from_config = {item.node_type: item for item in config.strategies}
|
||
for node_type, runtime_cfg in resolved_from_config.items():
|
||
strategy_class = DomainRegistry.get_strategy_class(node_type)
|
||
runtime_cfg.config.domain_option = resolve_domain_option_config(
|
||
getattr(strategy_class, "domain_option_model", None) if strategy_class is not None else None,
|
||
runtime_cfg.config.domain_option,
|
||
)
|
||
return resolved_from_config
|
||
|
||
resolved: Dict[str, StrategyRuntimeConfig] = {}
|
||
for node_type in node_types:
|
||
strategy_class = DomainRegistry.get_strategy_class(node_type)
|
||
if strategy_class is None:
|
||
continue
|
||
strategy_config = strategy_class.default_config.model_copy(deep=True)
|
||
strategy_config.domain_option = resolve_domain_option_config(
|
||
getattr(strategy_class, "domain_option_model", None),
|
||
strategy_config.domain_option,
|
||
)
|
||
resolved[node_type] = StrategyRuntimeConfig(
|
||
node_type=node_type,
|
||
config=strategy_config,
|
||
)
|
||
return resolved
|
||
|
||
|
||
def _build_pipeline_run_config(
|
||
*,
|
||
node_types: List[str],
|
||
local_datasource: DataSourceConfig,
|
||
remote_datasource: DataSourceConfig,
|
||
persist: PersistConfig,
|
||
logging_config: LoggingConfig,
|
||
strategy_overrides: Optional[Dict[str, Dict[str, Any]]] = None,
|
||
) -> PipelineRunConfig:
|
||
cfg = PipelineRunConfig(
|
||
node_types=node_types,
|
||
local_datasource=local_datasource,
|
||
remote_datasource=remote_datasource,
|
||
strategies=[],
|
||
persist=persist,
|
||
logging=logging_config,
|
||
)
|
||
if strategy_overrides:
|
||
cfg = apply_config_overrides(cfg, {"strategies": strategy_overrides})
|
||
return cfg
|
||
|
||
|
||
def _build_strategies(
|
||
config: PipelineRunConfig,
|
||
node_types: List[str],
|
||
local_collection: DataCollection,
|
||
remote_collection: DataCollection,
|
||
binding_manager: BindingManager,
|
||
strategy_map: Dict[str, StrategyRuntimeConfig],
|
||
) -> List[BaseSyncStrategy[Any]]:
|
||
strategies: List[BaseSyncStrategy[Any]] = []
|
||
both_jsonl = isinstance(config.local_datasource, JsonlDataSourceConfig) and isinstance(config.remote_datasource, JsonlDataSourceConfig)
|
||
|
||
for node_type in node_types:
|
||
strategy_class = DomainRegistry.get_strategy_class(node_type)
|
||
if strategy_class is None:
|
||
raise RuntimeError(f"Missing strategy for node_type: {node_type}")
|
||
strategy = strategy_class(node_type, local_collection, remote_collection, binding_manager)
|
||
|
||
runtime_cfg = strategy_map.get(node_type)
|
||
if runtime_cfg is not None:
|
||
strategy.config = runtime_cfg.config.model_copy(deep=True)
|
||
strategy.config.log_logic_warnings(node_type=strategy.node_type, logger=logger)
|
||
|
||
if both_jsonl:
|
||
strategy.config.local_orphan_action = OrphanAction.CREATE_REMOTE
|
||
strategy.config.remote_orphan_action = OrphanAction.NONE
|
||
strategy.config.update_direction = UpdateDirection.PUSH
|
||
if node_type == "contract":
|
||
strategy.config.depend_fields = {"project_id": "project"}
|
||
strategy.config.log_logic_warnings(node_type=strategy.node_type, logger=logger)
|
||
|
||
strategies.append(strategy)
|
||
return strategies
|
||
|
||
|
||
async def create_pipeline_from_config(
|
||
config: PipelineRunConfig,
|
||
*,
|
||
scope: str = "global",
|
||
pipeline_name: str | None = None,
|
||
bootstrap_binding_node_types: Optional[List[str]] = None,
|
||
ephemeral_node_types: Optional[List[str]] = None,
|
||
local_datasource_override=None,
|
||
remote_datasource_override=None,
|
||
) -> FullSyncPipeline:
|
||
ensure_app_logging(config.logging)
|
||
|
||
all_types = list(config.node_types)
|
||
strategy_map = _resolve_strategy_map(config, all_types)
|
||
|
||
db_path = Path(config.persist.db_path)
|
||
if config.persist.wipe_on_start and db_path.exists():
|
||
db_path.unlink()
|
||
|
||
persistence: Optional[PersistenceBackend] = None
|
||
local_ds = None
|
||
remote_ds = None
|
||
try:
|
||
persistence = create_persistence_backend(config.persist)
|
||
await persistence.initialize()
|
||
|
||
local_ds = await _create_datasource(
|
||
config.local_datasource,
|
||
datasource_override=local_datasource_override,
|
||
endpoint_name="local",
|
||
project_root=_project_root(),
|
||
)
|
||
await _initialize_datasource(local_ds, endpoint_name="local")
|
||
remote_ds = await _create_datasource(
|
||
config.remote_datasource,
|
||
datasource_override=remote_datasource_override,
|
||
endpoint_name="remote",
|
||
project_root=_project_root(),
|
||
)
|
||
await _initialize_datasource(remote_ds, endpoint_name="remote")
|
||
_add_handlers(config, local_ds, remote_ds, all_types)
|
||
|
||
local_collection, remote_collection, binding_manager = _build_collections_and_bindings(
|
||
persistence=persistence,
|
||
scope=scope,
|
||
)
|
||
|
||
strategies = _build_strategies(
|
||
config,
|
||
all_types,
|
||
local_collection,
|
||
remote_collection,
|
||
binding_manager,
|
||
strategy_map,
|
||
)
|
||
|
||
pipeline = FullSyncPipeline(
|
||
local_datasource=local_ds,
|
||
remote_datasource=remote_ds,
|
||
strategies=strategies,
|
||
node_types=all_types,
|
||
persistence=persistence,
|
||
local_collection=local_collection,
|
||
remote_collection=remote_collection,
|
||
binding_manager=binding_manager,
|
||
scope=scope,
|
||
pipeline_name=pipeline_name,
|
||
bootstrap_binding_node_types=bootstrap_binding_node_types,
|
||
ephemeral_node_types=ephemeral_node_types,
|
||
enable_persistence=config.persist.enable,
|
||
)
|
||
|
||
return pipeline
|
||
except Exception:
|
||
async def _safe_close(name: str, resource) -> None:
|
||
if resource is None:
|
||
return
|
||
try:
|
||
await asyncio.wait_for(resource.close(), timeout=3.0)
|
||
except Exception as close_exc:
|
||
logger.warning("Failed to close %s during create cleanup: %s", name, close_exc)
|
||
|
||
await _safe_close("remote_datasource", remote_ds)
|
||
await _safe_close("local_datasource", local_ds)
|
||
await _safe_close("persistence", persistence)
|
||
raise
|
||
|
||
|
||
async def run_pipeline_from_config(
|
||
config: PipelineRunConfig,
|
||
*,
|
||
title: str = "sync_state_machine pipeline",
|
||
print_summary: bool = True,
|
||
scope: str = "global",
|
||
bootstrap_binding_node_types: Optional[List[str]] = None,
|
||
local_datasource_override=None,
|
||
remote_datasource_override=None,
|
||
) -> Dict[str, Any]:
|
||
ensure_app_logging(config.logging)
|
||
|
||
resolved_cfg = config_snapshot(config)
|
||
resolved_cfg_text = json.dumps(resolved_cfg, ensure_ascii=False, indent=2)
|
||
logger.info(
|
||
"Resolved Full Config Summary: node_types=%d local=%s remote=%s",
|
||
len(config.node_types),
|
||
config.local_datasource.type,
|
||
config.remote_datasource.type,
|
||
)
|
||
logger.info("Resolved Full Config:\n%s", resolved_cfg_text)
|
||
|
||
pipeline: Optional[FullSyncPipeline] = None
|
||
try:
|
||
if print_summary:
|
||
logger.info("\n%s", "=" * 80)
|
||
logger.info("🚀 Creating %s", title)
|
||
logger.info("📝 Log file: %s", config.logging.file_path or "-")
|
||
logger.info(
|
||
"🔧 Resolved Full Config Summary: node_types=%d local=%s remote=%s",
|
||
len(config.node_types),
|
||
config.local_datasource.type,
|
||
config.remote_datasource.type,
|
||
)
|
||
logger.info("=" * 80)
|
||
|
||
pipeline = await create_pipeline_from_config(
|
||
config,
|
||
scope=scope,
|
||
pipeline_name=title,
|
||
bootstrap_binding_node_types=bootstrap_binding_node_types,
|
||
local_datasource_override=local_datasource_override,
|
||
remote_datasource_override=remote_datasource_override,
|
||
)
|
||
stats = await pipeline.run()
|
||
|
||
if print_summary:
|
||
logger.info("✅ Pipeline completed: stats=%s", stats)
|
||
logger.info("🗃️ Persistence DB: %s", config.persist.db_path)
|
||
|
||
return stats
|
||
finally:
|
||
if pipeline is not None:
|
||
await pipeline.close()
|
||
|
||
|
||
async def create_pipeline_from_file(
|
||
*,
|
||
project_root: Path,
|
||
config_path: str | Path,
|
||
scope: str = "global",
|
||
pipeline_name: str | None = None,
|
||
bootstrap_binding_node_types: Optional[List[str]] = None,
|
||
ephemeral_node_types: Optional[List[str]] = None,
|
||
local_datasource_override=None,
|
||
remote_datasource_override=None,
|
||
) -> FullSyncPipeline:
|
||
config = build_config_from_file(project_root=project_root, file_path=config_path)
|
||
return await create_pipeline_from_config(
|
||
config,
|
||
scope=scope,
|
||
pipeline_name=pipeline_name,
|
||
bootstrap_binding_node_types=bootstrap_binding_node_types,
|
||
ephemeral_node_types=ephemeral_node_types,
|
||
local_datasource_override=local_datasource_override,
|
||
remote_datasource_override=remote_datasource_override,
|
||
)
|
||
|
||
|
||
async def run_pipeline_from_file(
|
||
*,
|
||
project_root: Path,
|
||
config_path: str | Path,
|
||
title: str = "sync_state_machine pipeline",
|
||
print_summary: bool = True,
|
||
scope: str = "global",
|
||
bootstrap_binding_node_types: Optional[List[str]] = None,
|
||
local_datasource_override=None,
|
||
remote_datasource_override=None,
|
||
) -> Dict[str, Any]:
|
||
config = build_config_from_file(project_root=project_root, file_path=config_path)
|
||
return await run_pipeline_from_config(
|
||
config,
|
||
title=title,
|
||
print_summary=print_summary,
|
||
scope=scope,
|
||
bootstrap_binding_node_types=bootstrap_binding_node_types,
|
||
local_datasource_override=local_datasource_override,
|
||
remote_datasource_override=remote_datasource_override,
|
||
)
|
||
|
||
|
||
async def run_profile_from_file(
|
||
*,
|
||
project_root: Path,
|
||
config_path: str | Path,
|
||
print_summary: bool = True,
|
||
) -> Dict[str, Any]:
|
||
cfg_path = Path(config_path)
|
||
raw = load_overrides_from_file(cfg_path, project_root)
|
||
if is_multi_project_payload(raw):
|
||
from .multi_project_pipeline import MultiProjectPipeline
|
||
|
||
multi_cfg = MultiProjectRunConfig.model_validate(raw)
|
||
pipeline = MultiProjectPipeline(project_root=project_root, config_path=cfg_path, config=multi_cfg)
|
||
if print_summary:
|
||
logger.info("\n%s", "=" * 80)
|
||
logger.info("🚀 Creating multi-project pipeline from: %s", cfg_path)
|
||
logger.info("=" * 80)
|
||
return await pipeline.run()
|
||
|
||
config = build_config(project_root=project_root, overrides=raw)
|
||
configure_app_logging(config.logging, replace_handlers=False)
|
||
mode = f"{config.local_datasource.type}->{config.remote_datasource.type}"
|
||
return await run_pipeline_from_config(
|
||
config,
|
||
title=f"sync_state_machine pipeline ({mode})",
|
||
print_summary=print_summary,
|
||
)
|
||
|
||
|
||
async def create_jsonl_to_api_pipeline(
|
||
*,
|
||
# 路径配置
|
||
local_jsonl_dir: str,
|
||
persistence_db: str,
|
||
# API 配置
|
||
api_base_url: str,
|
||
api_uid: str,
|
||
api_secret: str,
|
||
# 节点类型
|
||
node_types: List[str],
|
||
strategy_overrides: Optional[Dict[str, Dict[str, Any]]] = None,
|
||
# Pipeline 行为
|
||
enable_persistence: bool = True,
|
||
wipe_persistence_on_start: bool = False,
|
||
# API 配置
|
||
api_debug: bool = False,
|
||
poll_max_retries: int = 10,
|
||
poll_interval: float = 0.5,
|
||
api_rate_limit_max_requests: int = 30,
|
||
api_rate_limit_window_seconds: float = 10.0,
|
||
local_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
|
||
remote_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
|
||
# Logger 配置
|
||
initialize_logger: bool = True,
|
||
logger_file_path: Optional[str] = None,
|
||
logger_level: int = logging.INFO,
|
||
) -> FullSyncPipeline:
|
||
cfg = _build_pipeline_run_config(
|
||
node_types=node_types,
|
||
local_datasource=JsonlDataSourceConfig(
|
||
type="jsonl",
|
||
jsonl_dir=local_jsonl_dir,
|
||
read_only=False,
|
||
handler_configs=local_handler_configs or {},
|
||
),
|
||
remote_datasource=ApiDataSourceConfig(
|
||
type="api",
|
||
api_base_url=api_base_url,
|
||
api_uid=api_uid,
|
||
api_secret=api_secret,
|
||
api_debug=api_debug,
|
||
poll_max_retries=poll_max_retries,
|
||
poll_interval=poll_interval,
|
||
api_rate_limit_max_requests=api_rate_limit_max_requests,
|
||
api_rate_limit_window_seconds=api_rate_limit_window_seconds,
|
||
handler_configs=remote_handler_configs or {},
|
||
),
|
||
persist=PersistConfig(
|
||
db_path=persistence_db,
|
||
enable=enable_persistence,
|
||
wipe_on_start=wipe_persistence_on_start,
|
||
),
|
||
logging_config=LoggingConfig(
|
||
initialize=initialize_logger,
|
||
file_path=logger_file_path,
|
||
level=logger_level,
|
||
),
|
||
strategy_overrides=strategy_overrides,
|
||
)
|
||
return await create_pipeline_from_config(cfg)
|
||
|
||
|
||
async def create_api_to_jsonl_pipeline(
|
||
*,
|
||
# API 配置（本地）
|
||
api_base_url: str,
|
||
api_uid: str,
|
||
api_secret: str,
|
||
# 路径配置
|
||
remote_jsonl_dir: str,
|
||
persistence_db: str,
|
||
# 节点类型
|
||
node_types: List[str],
|
||
strategy_overrides: Optional[Dict[str, Dict[str, Any]]] = None,
|
||
# Pipeline 行为
|
||
enable_persistence: bool = True,
|
||
wipe_persistence_on_start: bool = False,
|
||
# API 配置
|
||
api_debug: bool = False,
|
||
poll_max_retries: int = 10,
|
||
poll_interval: float = 0.5,
|
||
api_rate_limit_max_requests: int = 30,
|
||
api_rate_limit_window_seconds: float = 10.0,
|
||
local_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
|
||
remote_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
|
||
# Logger 配置
|
||
initialize_logger: bool = True,
|
||
logger_file_path: Optional[str] = None,
|
||
logger_level: int = logging.INFO,
|
||
) -> FullSyncPipeline:
|
||
cfg = _build_pipeline_run_config(
|
||
node_types=node_types,
|
||
local_datasource=ApiDataSourceConfig(
|
||
type="api",
|
||
api_base_url=api_base_url,
|
||
api_uid=api_uid,
|
||
api_secret=api_secret,
|
||
api_debug=api_debug,
|
||
poll_max_retries=poll_max_retries,
|
||
poll_interval=poll_interval,
|
||
api_rate_limit_max_requests=api_rate_limit_max_requests,
|
||
api_rate_limit_window_seconds=api_rate_limit_window_seconds,
|
||
handler_configs=local_handler_configs or {},
|
||
),
|
||
remote_datasource=JsonlDataSourceConfig(
|
||
type="jsonl",
|
||
jsonl_dir=remote_jsonl_dir,
|
||
read_only=False,
|
||
handler_configs=remote_handler_configs or {},
|
||
),
|
||
persist=PersistConfig(
|
||
db_path=persistence_db,
|
||
enable=enable_persistence,
|
||
wipe_on_start=wipe_persistence_on_start,
|
||
),
|
||
logging_config=LoggingConfig(
|
||
initialize=initialize_logger,
|
||
file_path=logger_file_path,
|
||
level=logger_level,
|
||
),
|
||
strategy_overrides=strategy_overrides,
|
||
)
|
||
return await create_pipeline_from_config(cfg)
|
||
|
||
|
||
async def create_jsonl_to_jsonl_pipeline(
|
||
*,
|
||
local_jsonl_dir: str,
|
||
remote_jsonl_dir: str,
|
||
persistence_db: str,
|
||
node_types: List[str],
|
||
enable_persistence: bool = True,
|
||
wipe_persistence_on_start: bool = False,
|
||
strategy_overrides: Optional[Dict[str, Dict[str, Any]]] = None,
|
||
local_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
|
||
remote_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
|
||
initialize_logger: bool = True,
|
||
logger_file_path: Optional[str] = None,
|
||
logger_level: int = logging.INFO,
|
||
) -> FullSyncPipeline:
|
||
cfg = _build_pipeline_run_config(
|
||
node_types=node_types,
|
||
local_datasource=JsonlDataSourceConfig(
|
||
type="jsonl",
|
||
jsonl_dir=local_jsonl_dir,
|
||
read_only=False,
|
||
handler_configs=local_handler_configs or {},
|
||
),
|
||
remote_datasource=JsonlDataSourceConfig(
|
||
type="jsonl",
|
||
jsonl_dir=remote_jsonl_dir,
|
||
read_only=False,
|
||
handler_configs=remote_handler_configs or {},
|
||
),
|
||
persist=PersistConfig(
|
||
db_path=persistence_db,
|
||
enable=enable_persistence,
|
||
wipe_on_start=wipe_persistence_on_start,
|
||
),
|
||
logging_config=LoggingConfig(
|
||
initialize=initialize_logger,
|
||
file_path=logger_file_path,
|
||
level=logger_level,
|
||
),
|
||
strategy_overrides=strategy_overrides,
|
||
)
|
||
return await create_pipeline_from_config(cfg)
|