strepsiades
684 lines
22 KiB
Python
684 lines
22 KiB
Python
"""
|
||
Pipeline Factory - 工厂函数，负责初始化所有组件并创建 Pipeline
|
||
|
||
用户只需提供配置参数，工厂函数负责：
|
||
1. 初始化 Persistence
|
||
2. 初始化 DataSource（并注册 Handler）
|
||
3. 初始化 Collection 和 BindingManager
|
||
4. 初始化 Strategy（并注入依赖）
|
||
5. 创建 Pipeline（注入所有组件）
|
||
6. 返回完全就绪的 Pipeline
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import inspect
|
||
import json
|
||
import logging
|
||
from pathlib import Path
|
||
from typing import Any, Dict, List, Optional
|
||
|
||
from ..common.binding import BindingManager
|
||
from ..common.collection import DataCollection
|
||
from ..common.persistence import PersistenceBackend
|
||
from ..common.registry import DomainRegistry
|
||
from ..datasource import ensure_builtin_datasource_types_registered
|
||
from ..datasource.type_registry import get_datasource_factory
|
||
from ..config import (
|
||
PipelineRunConfig,
|
||
MultiProjectRunConfig,
|
||
DataSourceConfig,
|
||
ApiDataSourceConfig,
|
||
PersistConfig,
|
||
JsonlDataSourceConfig,
|
||
LoggingConfig,
|
||
build_config,
|
||
StrategyRuntimeConfig,
|
||
build_config_from_file,
|
||
build_multi_project_config_from_file,
|
||
apply_config_overrides,
|
||
OrphanAction,
|
||
UpdateDirection,
|
||
resolve_log_file_path,
|
||
config_snapshot,
|
||
is_multi_project_payload,
|
||
load_overrides_from_file,
|
||
)
|
||
from ..sync_system.strategy import BaseSyncStrategy
|
||
from .run_logger import init_pipeline_logger
|
||
from .full_sync_pipeline import FullSyncPipeline
|
||
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
def _project_root() -> Path:
|
||
return Path(__file__).resolve().parents[2]
|
||
|
||
|
||
def _prepare_remote_jsonl_dir(remote_dir: Path, *, project_root: Optional[Path] = None) -> Path:
|
||
from ..datasource.jsonl.path_utils import prepare_remote_jsonl_dir
|
||
|
||
return prepare_remote_jsonl_dir(remote_dir, project_root=(project_root or _project_root()))
|
||
|
||
|
||
def _ensure_pipeline_logger(
|
||
*,
|
||
initialize_logger: bool,
|
||
logger_file_path: Optional[str],
|
||
logger_level: int,
|
||
suppress_node_logs: bool,
|
||
) -> None:
|
||
if not initialize_logger:
|
||
return
|
||
|
||
app_logger = logging.getLogger("sync_state_machine")
|
||
if app_logger.handlers:
|
||
return
|
||
|
||
init_pipeline_logger(
|
||
log_file_path=logger_file_path,
|
||
level=logger_level,
|
||
mirror_console=True,
|
||
replace_handlers=False,
|
||
suppress_node_logs=suppress_node_logs,
|
||
)
|
||
|
||
|
||
def _build_handler_config(
|
||
node_type: str,
|
||
ds_config: DataSourceConfig,
|
||
) -> Dict[str, Any]:
|
||
return dict(ds_config.handler_configs.get(node_type, {}))
|
||
|
||
|
||
def _create_handler(
|
||
*,
|
||
node_type: str,
|
||
ds_config: DataSourceConfig,
|
||
datasource,
|
||
):
|
||
datasource_type = ds_config.type
|
||
handler_class = DomainRegistry.get_handler(node_type, datasource_type)
|
||
if handler_class is None:
|
||
raise RuntimeError(
|
||
f"Missing {datasource_type.upper()} handler for node_type: {node_type}"
|
||
)
|
||
|
||
handler_config = _build_handler_config(node_type, ds_config)
|
||
if datasource_type == "api":
|
||
handler = handler_class(**handler_config)
|
||
else:
|
||
handler = handler_class(datasource)
|
||
|
||
if hasattr(handler, "set_handler_config"):
|
||
handler.set_handler_config(handler_config)
|
||
return handler
|
||
|
||
|
||
async def _create_datasource(
|
||
ds_config: DataSourceConfig,
|
||
*,
|
||
datasource_override=None,
|
||
endpoint_name: str,
|
||
project_root: Path | None = None,
|
||
):
|
||
if datasource_override is not None:
|
||
return datasource_override
|
||
|
||
datasource_factory = get_datasource_factory(ds_config.type)
|
||
if datasource_factory is None:
|
||
raise RuntimeError(f"No datasource factory registered for type: {ds_config.type}")
|
||
|
||
factory_kwargs: dict[str, Any] = {}
|
||
signature = inspect.signature(datasource_factory)
|
||
if "project_root" in signature.parameters:
|
||
factory_kwargs["project_root"] = project_root or _project_root()
|
||
|
||
datasource = datasource_factory(ds_config, endpoint_name, **factory_kwargs)
|
||
if inspect.isawaitable(datasource):
|
||
datasource = await datasource
|
||
return datasource
|
||
|
||
|
||
async def _initialize_datasource(datasource, *, endpoint_name: str) -> None:
|
||
try:
|
||
await datasource.initialize()
|
||
except Exception as exc:
|
||
raise RuntimeError(f"Failed to initialize {endpoint_name} datasource: {type(exc).__name__}: {exc}") from exc
|
||
|
||
ensure_builtin_datasource_types_registered()
|
||
|
||
|
||
def _add_handlers_for_endpoint(
|
||
*,
|
||
datasource,
|
||
ds_config: DataSourceConfig,
|
||
node_types: List[str],
|
||
) -> None:
|
||
for node_type in node_types:
|
||
datasource.add_handler(
|
||
_create_handler(
|
||
node_type=node_type,
|
||
ds_config=ds_config,
|
||
datasource=datasource,
|
||
)
|
||
)
|
||
|
||
|
||
def _add_handlers(
|
||
config: PipelineRunConfig,
|
||
local_ds,
|
||
remote_ds,
|
||
all_types: List[str],
|
||
) -> None:
|
||
_add_handlers_for_endpoint(
|
||
datasource=local_ds,
|
||
ds_config=config.local_datasource,
|
||
node_types=all_types,
|
||
)
|
||
_add_handlers_for_endpoint(
|
||
datasource=remote_ds,
|
||
ds_config=config.remote_datasource,
|
||
node_types=all_types,
|
||
)
|
||
|
||
|
||
def _build_collections_and_bindings(
|
||
*,
|
||
persistence: PersistenceBackend,
|
||
scope: str,
|
||
) -> tuple[DataCollection, DataCollection, BindingManager]:
|
||
local_collection = DataCollection("local", scope=scope, persistence=persistence, auto_persist=False)
|
||
remote_collection = DataCollection("remote", scope=scope, persistence=persistence, auto_persist=False)
|
||
binding_manager = BindingManager(persistence, auto_persist=False, scope=scope)
|
||
return local_collection, remote_collection, binding_manager
|
||
|
||
|
||
def _resolve_strategy_map(config: PipelineRunConfig, node_types: List[str]) -> Dict[str, StrategyRuntimeConfig]:
|
||
if config.strategies:
|
||
return {item.node_type: item for item in config.strategies}
|
||
|
||
resolved: Dict[str, StrategyRuntimeConfig] = {}
|
||
for node_type in node_types:
|
||
strategy_class = DomainRegistry.get_strategy_class(node_type)
|
||
if strategy_class is None:
|
||
continue
|
||
resolved[node_type] = StrategyRuntimeConfig(
|
||
node_type=node_type,
|
||
config=strategy_class.default_config.model_copy(deep=True),
|
||
)
|
||
return resolved
|
||
|
||
|
||
def _build_pipeline_run_config(
|
||
*,
|
||
node_types: List[str],
|
||
local_datasource: DataSourceConfig,
|
||
remote_datasource: DataSourceConfig,
|
||
persist: PersistConfig,
|
||
logging_config: LoggingConfig,
|
||
strategy_overrides: Optional[Dict[str, Dict[str, Any]]] = None,
|
||
) -> PipelineRunConfig:
|
||
cfg = PipelineRunConfig(
|
||
node_types=node_types,
|
||
local_datasource=local_datasource,
|
||
remote_datasource=remote_datasource,
|
||
strategies=[],
|
||
persist=persist,
|
||
logging=logging_config,
|
||
)
|
||
if strategy_overrides:
|
||
cfg = apply_config_overrides(cfg, {"strategies": strategy_overrides})
|
||
return cfg
|
||
|
||
|
||
def _build_strategies(
|
||
config: PipelineRunConfig,
|
||
node_types: List[str],
|
||
local_collection: DataCollection,
|
||
remote_collection: DataCollection,
|
||
binding_manager: BindingManager,
|
||
strategy_map: Dict[str, StrategyRuntimeConfig],
|
||
) -> List[BaseSyncStrategy[Any]]:
|
||
strategies: List[BaseSyncStrategy[Any]] = []
|
||
both_jsonl = isinstance(config.local_datasource, JsonlDataSourceConfig) and isinstance(config.remote_datasource, JsonlDataSourceConfig)
|
||
|
||
for node_type in node_types:
|
||
strategy_class = DomainRegistry.get_strategy_class(node_type)
|
||
if strategy_class is None:
|
||
raise RuntimeError(f"Missing strategy for node_type: {node_type}")
|
||
strategy = strategy_class(node_type, local_collection, remote_collection, binding_manager)
|
||
|
||
if both_jsonl:
|
||
strategy.update_config(
|
||
local_orphan_action=OrphanAction.CREATE_REMOTE,
|
||
remote_orphan_action=OrphanAction.NONE,
|
||
update_direction=UpdateDirection.PUSH,
|
||
)
|
||
if node_type == "contract":
|
||
strategy.update_config(depend_fields={"project_id": "project"})
|
||
|
||
runtime_cfg = strategy_map.get(node_type)
|
||
if runtime_cfg is not None:
|
||
strategy.set_config(runtime_cfg.config.model_copy(deep=True))
|
||
strategy.skip_sync = bool(runtime_cfg.skip_sync)
|
||
|
||
strategies.append(strategy)
|
||
return strategies
|
||
|
||
|
||
async def create_pipeline_from_config(
|
||
config: PipelineRunConfig,
|
||
*,
|
||
scope: str = "global",
|
||
pipeline_name: str | None = None,
|
||
bootstrap_binding_node_types: Optional[List[str]] = None,
|
||
ephemeral_node_types: Optional[List[str]] = None,
|
||
local_datasource_override=None,
|
||
remote_datasource_override=None,
|
||
) -> FullSyncPipeline:
|
||
if config.logging.initialize and not config.logging.file_path:
|
||
config.logging.file_path = resolve_log_file_path(config.logging)
|
||
|
||
_ensure_pipeline_logger(
|
||
initialize_logger=config.logging.initialize,
|
||
logger_file_path=config.logging.file_path,
|
||
logger_level=config.logging.level,
|
||
suppress_node_logs=config.logging.suppress_node_logs,
|
||
)
|
||
|
||
all_types = list(config.node_types)
|
||
strategy_map = _resolve_strategy_map(config, all_types)
|
||
|
||
if config.persist.backend != "sqlite":
|
||
raise NotImplementedError(f"Persistence backend not supported yet: {config.persist.backend}")
|
||
|
||
db_path = Path(config.persist.db_path)
|
||
if config.persist.wipe_on_start and db_path.exists():
|
||
db_path.unlink()
|
||
|
||
persistence: Optional[PersistenceBackend] = None
|
||
local_ds = None
|
||
remote_ds = None
|
||
try:
|
||
persistence = PersistenceBackend(str(config.persist.db_path), backend=config.persist.backend)
|
||
await persistence.initialize()
|
||
|
||
local_ds = await _create_datasource(
|
||
config.local_datasource,
|
||
datasource_override=local_datasource_override,
|
||
endpoint_name="local",
|
||
project_root=_project_root(),
|
||
)
|
||
await _initialize_datasource(local_ds, endpoint_name="local")
|
||
remote_ds = await _create_datasource(
|
||
config.remote_datasource,
|
||
datasource_override=remote_datasource_override,
|
||
endpoint_name="remote",
|
||
project_root=_project_root(),
|
||
)
|
||
await _initialize_datasource(remote_ds, endpoint_name="remote")
|
||
_add_handlers(config, local_ds, remote_ds, all_types)
|
||
|
||
local_collection, remote_collection, binding_manager = _build_collections_and_bindings(
|
||
persistence=persistence,
|
||
scope=scope,
|
||
)
|
||
|
||
strategies = _build_strategies(
|
||
config,
|
||
all_types,
|
||
local_collection,
|
||
remote_collection,
|
||
binding_manager,
|
||
strategy_map,
|
||
)
|
||
|
||
pipeline = FullSyncPipeline(
|
||
local_datasource=local_ds,
|
||
remote_datasource=remote_ds,
|
||
strategies=strategies,
|
||
node_types=all_types,
|
||
persistence=persistence,
|
||
local_collection=local_collection,
|
||
remote_collection=remote_collection,
|
||
binding_manager=binding_manager,
|
||
scope=scope,
|
||
pipeline_name=pipeline_name,
|
||
bootstrap_binding_node_types=bootstrap_binding_node_types,
|
||
ephemeral_node_types=ephemeral_node_types,
|
||
enable_persistence=config.persist.enable,
|
||
)
|
||
|
||
return pipeline
|
||
except Exception:
|
||
async def _safe_close(name: str, resource) -> None:
|
||
if resource is None:
|
||
return
|
||
try:
|
||
await asyncio.wait_for(resource.close(), timeout=3.0)
|
||
except Exception as close_exc:
|
||
logger.warning("Failed to close %s during create cleanup: %s", name, close_exc)
|
||
|
||
await _safe_close("remote_datasource", remote_ds)
|
||
await _safe_close("local_datasource", local_ds)
|
||
await _safe_close("persistence", persistence)
|
||
raise
|
||
|
||
|
||
async def run_pipeline_from_config(
|
||
config: PipelineRunConfig,
|
||
*,
|
||
title: str = "sync_state_machine pipeline",
|
||
print_summary: bool = True,
|
||
scope: str = "global",
|
||
bootstrap_binding_node_types: Optional[List[str]] = None,
|
||
local_datasource_override=None,
|
||
remote_datasource_override=None,
|
||
) -> Dict[str, Any]:
|
||
if config.logging.initialize and not config.logging.file_path:
|
||
config.logging.file_path = resolve_log_file_path(config.logging)
|
||
|
||
_ensure_pipeline_logger(
|
||
initialize_logger=config.logging.initialize,
|
||
logger_file_path=config.logging.file_path,
|
||
logger_level=config.logging.level,
|
||
suppress_node_logs=config.logging.suppress_node_logs,
|
||
)
|
||
|
||
resolved_cfg = config_snapshot(config)
|
||
resolved_cfg_text = json.dumps(resolved_cfg, ensure_ascii=False, indent=2)
|
||
logger.info(
|
||
"Resolved Full Config Summary: node_types=%d local=%s remote=%s",
|
||
len(config.node_types),
|
||
config.local_datasource.type,
|
||
config.remote_datasource.type,
|
||
)
|
||
if config.logging.file_path:
|
||
try:
|
||
with open(config.logging.file_path, "a", encoding="utf-8") as fp:
|
||
fp.write("Resolved Full Config:\n")
|
||
fp.write(resolved_cfg_text)
|
||
fp.write("\n")
|
||
except Exception as exc:
|
||
logger.warning("Failed to write resolved config into log file: %s", exc)
|
||
|
||
pipeline: Optional[FullSyncPipeline] = None
|
||
try:
|
||
if print_summary:
|
||
print("\n" + "=" * 80)
|
||
print(f"🚀 Creating {title}")
|
||
print(f"📝 Log file: {config.logging.file_path or '-'}")
|
||
print(
|
||
f"🔧 Resolved Full Config Summary: node_types={len(config.node_types)} "
|
||
f"local={config.local_datasource.type} "
|
||
f"remote={config.remote_datasource.type}"
|
||
)
|
||
print(f"🔧 Resolved Full Config:\n{resolved_cfg_text}")
|
||
print("=" * 80)
|
||
|
||
pipeline = await create_pipeline_from_config(
|
||
config,
|
||
scope=scope,
|
||
pipeline_name=title,
|
||
bootstrap_binding_node_types=bootstrap_binding_node_types,
|
||
local_datasource_override=local_datasource_override,
|
||
remote_datasource_override=remote_datasource_override,
|
||
)
|
||
stats = await pipeline.run()
|
||
|
||
if print_summary:
|
||
print(f"\n✅ Pipeline completed: stats={stats}")
|
||
print(f"🗃️ Persistence DB: {config.persist.db_path}")
|
||
|
||
return stats
|
||
finally:
|
||
if pipeline is not None:
|
||
await pipeline.close()
|
||
|
||
|
||
async def create_pipeline_from_file(
|
||
*,
|
||
project_root: Path,
|
||
config_path: str | Path,
|
||
scope: str = "global",
|
||
pipeline_name: str | None = None,
|
||
bootstrap_binding_node_types: Optional[List[str]] = None,
|
||
ephemeral_node_types: Optional[List[str]] = None,
|
||
local_datasource_override=None,
|
||
remote_datasource_override=None,
|
||
) -> FullSyncPipeline:
|
||
config = build_config_from_file(project_root=project_root, file_path=config_path)
|
||
return await create_pipeline_from_config(
|
||
config,
|
||
scope=scope,
|
||
pipeline_name=pipeline_name,
|
||
bootstrap_binding_node_types=bootstrap_binding_node_types,
|
||
ephemeral_node_types=ephemeral_node_types,
|
||
local_datasource_override=local_datasource_override,
|
||
remote_datasource_override=remote_datasource_override,
|
||
)
|
||
|
||
|
||
async def run_pipeline_from_file(
|
||
*,
|
||
project_root: Path,
|
||
config_path: str | Path,
|
||
title: str = "sync_state_machine pipeline",
|
||
print_summary: bool = True,
|
||
scope: str = "global",
|
||
bootstrap_binding_node_types: Optional[List[str]] = None,
|
||
local_datasource_override=None,
|
||
remote_datasource_override=None,
|
||
) -> Dict[str, Any]:
|
||
config = build_config_from_file(project_root=project_root, file_path=config_path)
|
||
return await run_pipeline_from_config(
|
||
config,
|
||
title=title,
|
||
print_summary=print_summary,
|
||
scope=scope,
|
||
bootstrap_binding_node_types=bootstrap_binding_node_types,
|
||
local_datasource_override=local_datasource_override,
|
||
remote_datasource_override=remote_datasource_override,
|
||
)
|
||
|
||
|
||
async def run_profile_from_file(
|
||
*,
|
||
project_root: Path,
|
||
config_path: str | Path,
|
||
print_summary: bool = True,
|
||
) -> Dict[str, Any]:
|
||
cfg_path = Path(config_path)
|
||
raw = load_overrides_from_file(cfg_path, project_root)
|
||
if is_multi_project_payload(raw):
|
||
from .multi_project_pipeline import MultiProjectPipeline
|
||
|
||
multi_cfg = MultiProjectRunConfig.model_validate(raw)
|
||
pipeline = MultiProjectPipeline(project_root=project_root, config_path=cfg_path, config=multi_cfg)
|
||
if print_summary:
|
||
print("\n" + "=" * 80)
|
||
print(f"🚀 Creating multi-project pipeline from: {cfg_path}")
|
||
print("=" * 80)
|
||
return await pipeline.run()
|
||
|
||
config = build_config(project_root=project_root, overrides=raw)
|
||
mode = f"{config.local_datasource.type}->{config.remote_datasource.type}"
|
||
return await run_pipeline_from_config(
|
||
config,
|
||
title=f"sync_state_machine pipeline ({mode})",
|
||
print_summary=print_summary,
|
||
)
|
||
|
||
|
||
async def create_jsonl_to_api_pipeline(
|
||
*,
|
||
# 路径配置
|
||
local_jsonl_dir: str,
|
||
persistence_db: str,
|
||
# API 配置
|
||
api_base_url: str,
|
||
api_uid: str,
|
||
api_secret: str,
|
||
# 节点类型
|
||
node_types: List[str],
|
||
strategy_overrides: Optional[Dict[str, Dict[str, Any]]] = None,
|
||
# Pipeline 行为
|
||
enable_persistence: bool = True,
|
||
wipe_persistence_on_start: bool = False,
|
||
# API 配置
|
||
api_debug: bool = False,
|
||
poll_max_retries: int = 10,
|
||
poll_interval: float = 0.5,
|
||
api_rate_limit_max_requests: int = 30,
|
||
api_rate_limit_window_seconds: float = 10.0,
|
||
local_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
|
||
remote_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
|
||
# Logger 配置
|
||
initialize_logger: bool = True,
|
||
logger_file_path: Optional[str] = None,
|
||
logger_level: int = logging.INFO,
|
||
) -> FullSyncPipeline:
|
||
cfg = _build_pipeline_run_config(
|
||
node_types=node_types,
|
||
local_datasource=JsonlDataSourceConfig(
|
||
type="jsonl",
|
||
jsonl_dir=local_jsonl_dir,
|
||
read_only=False,
|
||
handler_configs=local_handler_configs or {},
|
||
),
|
||
remote_datasource=ApiDataSourceConfig(
|
||
type="api",
|
||
api_base_url=api_base_url,
|
||
api_uid=api_uid,
|
||
api_secret=api_secret,
|
||
api_debug=api_debug,
|
||
poll_max_retries=poll_max_retries,
|
||
poll_interval=poll_interval,
|
||
api_rate_limit_max_requests=api_rate_limit_max_requests,
|
||
api_rate_limit_window_seconds=api_rate_limit_window_seconds,
|
||
handler_configs=remote_handler_configs or {},
|
||
),
|
||
persist=PersistConfig(
|
||
db_path=persistence_db,
|
||
enable=enable_persistence,
|
||
wipe_on_start=wipe_persistence_on_start,
|
||
),
|
||
logging_config=LoggingConfig(
|
||
initialize=initialize_logger,
|
||
file_path=logger_file_path,
|
||
level=logger_level,
|
||
),
|
||
strategy_overrides=strategy_overrides,
|
||
)
|
||
return await create_pipeline_from_config(cfg)
|
||
|
||
|
||
async def create_api_to_jsonl_pipeline(
|
||
*,
|
||
# API 配置（本地）
|
||
api_base_url: str,
|
||
api_uid: str,
|
||
api_secret: str,
|
||
# 路径配置
|
||
remote_jsonl_dir: str,
|
||
persistence_db: str,
|
||
# 节点类型
|
||
node_types: List[str],
|
||
strategy_overrides: Optional[Dict[str, Dict[str, Any]]] = None,
|
||
# Pipeline 行为
|
||
enable_persistence: bool = True,
|
||
wipe_persistence_on_start: bool = False,
|
||
# API 配置
|
||
api_debug: bool = False,
|
||
poll_max_retries: int = 10,
|
||
poll_interval: float = 0.5,
|
||
api_rate_limit_max_requests: int = 30,
|
||
api_rate_limit_window_seconds: float = 10.0,
|
||
local_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
|
||
remote_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
|
||
# Logger 配置
|
||
initialize_logger: bool = True,
|
||
logger_file_path: Optional[str] = None,
|
||
logger_level: int = logging.INFO,
|
||
) -> FullSyncPipeline:
|
||
cfg = _build_pipeline_run_config(
|
||
node_types=node_types,
|
||
local_datasource=ApiDataSourceConfig(
|
||
type="api",
|
||
api_base_url=api_base_url,
|
||
api_uid=api_uid,
|
||
api_secret=api_secret,
|
||
api_debug=api_debug,
|
||
poll_max_retries=poll_max_retries,
|
||
poll_interval=poll_interval,
|
||
api_rate_limit_max_requests=api_rate_limit_max_requests,
|
||
api_rate_limit_window_seconds=api_rate_limit_window_seconds,
|
||
handler_configs=local_handler_configs or {},
|
||
),
|
||
remote_datasource=JsonlDataSourceConfig(
|
||
type="jsonl",
|
||
jsonl_dir=remote_jsonl_dir,
|
||
read_only=False,
|
||
handler_configs=remote_handler_configs or {},
|
||
),
|
||
persist=PersistConfig(
|
||
db_path=persistence_db,
|
||
enable=enable_persistence,
|
||
wipe_on_start=wipe_persistence_on_start,
|
||
),
|
||
logging_config=LoggingConfig(
|
||
initialize=initialize_logger,
|
||
file_path=logger_file_path,
|
||
level=logger_level,
|
||
),
|
||
strategy_overrides=strategy_overrides,
|
||
)
|
||
return await create_pipeline_from_config(cfg)
|
||
|
||
|
||
async def create_jsonl_to_jsonl_pipeline(
|
||
*,
|
||
local_jsonl_dir: str,
|
||
remote_jsonl_dir: str,
|
||
persistence_db: str,
|
||
node_types: List[str],
|
||
enable_persistence: bool = True,
|
||
wipe_persistence_on_start: bool = False,
|
||
strategy_overrides: Optional[Dict[str, Dict[str, Any]]] = None,
|
||
local_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
|
||
remote_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
|
||
initialize_logger: bool = True,
|
||
logger_file_path: Optional[str] = None,
|
||
logger_level: int = logging.INFO,
|
||
) -> FullSyncPipeline:
|
||
cfg = _build_pipeline_run_config(
|
||
node_types=node_types,
|
||
local_datasource=JsonlDataSourceConfig(
|
||
type="jsonl",
|
||
jsonl_dir=local_jsonl_dir,
|
||
read_only=False,
|
||
handler_configs=local_handler_configs or {},
|
||
),
|
||
remote_datasource=JsonlDataSourceConfig(
|
||
type="jsonl",
|
||
jsonl_dir=remote_jsonl_dir,
|
||
read_only=False,
|
||
handler_configs=remote_handler_configs or {},
|
||
),
|
||
persist=PersistConfig(
|
||
db_path=persistence_db,
|
||
enable=enable_persistence,
|
||
wipe_on_start=wipe_persistence_on_start,
|
||
),
|
||
logging_config=LoggingConfig(
|
||
initialize=initialize_logger,
|
||
file_path=logger_file_path,
|
||
level=logger_level,
|
||
),
|
||
strategy_overrides=strategy_overrides,
|
||
)
|
||
return await create_pipeline_from_config(cfg)
|