first commit
This commit is contained in:
strepsiades
@@ -0,0 +1,578 @@
|
||||
"""
|
||||
Pipeline Factory - 工厂函数，负责初始化所有组件并创建 Pipeline
|
||||
|
||||
用户只需提供配置参数，工厂函数负责：
|
||||
1. 初始化 Persistence
|
||||
2. 初始化 DataSource（并注册 Handler）
|
||||
3. 初始化 Collection 和 BindingManager
|
||||
4. 初始化 Strategy（并注入依赖）
|
||||
5. 创建 Pipeline（注入所有组件）
|
||||
6. 返回完全就绪的 Pipeline
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from ..common.binding import BindingManager
|
||||
from ..common.collection import DataCollection
|
||||
from ..common.persistence import PersistenceBackend
|
||||
from ..common.registry import DomainRegistry
|
||||
from ..datasource.api.datasource import ApiDataSource
|
||||
from ..datasource.jsonl.datasource import JsonlDataSource
|
||||
from ..config import (
|
||||
PipelineRunConfig,
|
||||
DataSourceConfig,
|
||||
ApiDataSourceConfig,
|
||||
JsonlDataSourceConfig,
|
||||
PersistConfig,
|
||||
LoggingConfig,
|
||||
StrategyRuntimeConfig,
|
||||
apply_config_overrides,
|
||||
OrphanAction,
|
||||
UpdateDirection,
|
||||
resolve_log_file_path,
|
||||
config_snapshot,
|
||||
)
|
||||
from ..sync_system.strategy import BaseSyncStrategy
|
||||
from .run_logger import init_pipeline_logger
|
||||
from .full_sync_pipeline import FullSyncPipeline
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _project_root() -> Path:
|
||||
return Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def _prepare_remote_jsonl_dir(remote_dir: Path, *, project_root: Optional[Path] = None) -> Path:
|
||||
project_root_path = (project_root or _project_root()).resolve()
|
||||
resolved_remote_dir = remote_dir.resolve(strict=False)
|
||||
|
||||
if resolved_remote_dir.exists():
|
||||
if not resolved_remote_dir.is_dir():
|
||||
raise FileNotFoundError(
|
||||
f"remote_datasource.jsonl_dir exists but is not a directory: {resolved_remote_dir}"
|
||||
)
|
||||
return resolved_remote_dir
|
||||
|
||||
try:
|
||||
resolved_remote_dir.relative_to(project_root_path)
|
||||
except ValueError as exc:
|
||||
raise FileNotFoundError(
|
||||
"remote_datasource.jsonl_dir does not exist and auto-create is only allowed "
|
||||
f"under project root: {resolved_remote_dir} (project_root={project_root_path})"
|
||||
) from exc
|
||||
|
||||
resolved_remote_dir.mkdir(parents=True, exist_ok=True)
|
||||
logger.info("Created missing remote JSONL directory: %s", resolved_remote_dir)
|
||||
return resolved_remote_dir
|
||||
|
||||
|
||||
def _ensure_pipeline_logger(
|
||||
*,
|
||||
initialize_logger: bool,
|
||||
logger_file_path: Optional[str],
|
||||
logger_level: int,
|
||||
suppress_node_logs: bool,
|
||||
) -> None:
|
||||
if not initialize_logger:
|
||||
return
|
||||
|
||||
app_logger = logging.getLogger("sync_state_machine")
|
||||
if app_logger.handlers:
|
||||
return
|
||||
|
||||
init_pipeline_logger(
|
||||
log_file_path=logger_file_path,
|
||||
level=logger_level,
|
||||
mirror_console=True,
|
||||
replace_handlers=False,
|
||||
suppress_node_logs=suppress_node_logs,
|
||||
)
|
||||
|
||||
|
||||
def _api_handler_kwargs(
|
||||
node_type: str,
|
||||
ds_config: DataSourceConfig,
|
||||
target_project_ids: List[str],
|
||||
) -> Dict[str, Any]:
|
||||
kwargs = dict(ds_config.handler_configs.get(node_type, {}))
|
||||
|
||||
if node_type == "project" and target_project_ids:
|
||||
kwargs.setdefault("filter_project_id", target_project_ids)
|
||||
return kwargs
|
||||
|
||||
|
||||
def _resolve_datasource_target_project_ids(config: PipelineRunConfig) -> tuple[List[str], List[str]]:
|
||||
fallback = [str(pid) for pid in config.target_project_ids if str(pid)]
|
||||
|
||||
local_cfg_ids = [str(pid) for pid in config.local_datasource.target_project_ids if str(pid)]
|
||||
remote_cfg_ids = [str(pid) for pid in config.remote_datasource.target_project_ids if str(pid)]
|
||||
|
||||
local_target_ids = local_cfg_ids if local_cfg_ids else fallback
|
||||
remote_target_ids = remote_cfg_ids if remote_cfg_ids else fallback
|
||||
return local_target_ids, remote_target_ids
|
||||
|
||||
|
||||
async def _create_datasources(config: PipelineRunConfig):
|
||||
local_cfg = config.local_datasource
|
||||
remote_cfg = config.remote_datasource
|
||||
|
||||
if isinstance(local_cfg, JsonlDataSourceConfig):
|
||||
local_dir = Path(local_cfg.jsonl_dir)
|
||||
if not local_dir.exists() or not local_dir.is_dir():
|
||||
raise FileNotFoundError(
|
||||
f"local_datasource.jsonl_dir does not exist or is not a directory: {local_dir}"
|
||||
)
|
||||
local_ds = JsonlDataSource(local_dir, read_only=local_cfg.read_only)
|
||||
else:
|
||||
local_ds = ApiDataSource(
|
||||
base_url=local_cfg.api_base_url,
|
||||
uid=local_cfg.api_uid,
|
||||
secret=local_cfg.api_secret,
|
||||
debug=local_cfg.api_debug,
|
||||
poll_max_retries=local_cfg.poll_max_retries,
|
||||
poll_interval=local_cfg.poll_interval,
|
||||
rate_limit_max_requests=local_cfg.api_rate_limit_max_requests,
|
||||
rate_limit_window_seconds=local_cfg.api_rate_limit_window_seconds,
|
||||
)
|
||||
await local_ds.initialize()
|
||||
|
||||
if isinstance(remote_cfg, JsonlDataSourceConfig):
|
||||
remote_dir = Path(remote_cfg.jsonl_dir)
|
||||
remote_dir = _prepare_remote_jsonl_dir(remote_dir)
|
||||
remote_ds = JsonlDataSource(remote_dir, read_only=remote_cfg.read_only)
|
||||
else:
|
||||
remote_ds = ApiDataSource(
|
||||
base_url=remote_cfg.api_base_url,
|
||||
uid=remote_cfg.api_uid,
|
||||
secret=remote_cfg.api_secret,
|
||||
debug=remote_cfg.api_debug,
|
||||
poll_max_retries=remote_cfg.poll_max_retries,
|
||||
poll_interval=remote_cfg.poll_interval,
|
||||
rate_limit_max_requests=remote_cfg.api_rate_limit_max_requests,
|
||||
rate_limit_window_seconds=remote_cfg.api_rate_limit_window_seconds,
|
||||
)
|
||||
await remote_ds.initialize()
|
||||
|
||||
return local_ds, remote_ds
|
||||
|
||||
|
||||
def _register_handlers(
|
||||
config: PipelineRunConfig,
|
||||
local_ds,
|
||||
remote_ds,
|
||||
all_types: List[str],
|
||||
) -> None:
|
||||
for node_type in all_types:
|
||||
if isinstance(config.local_datasource, JsonlDataSourceConfig):
|
||||
jsonl_handler_class = DomainRegistry.get_jsonl_handler(node_type)
|
||||
if jsonl_handler_class is None:
|
||||
raise RuntimeError(f"Missing JSONL handler for node_type: {node_type}")
|
||||
local_ds.register_handler(jsonl_handler_class(local_ds))
|
||||
else:
|
||||
api_handler_class = DomainRegistry.get_api_handler(node_type)
|
||||
if api_handler_class is None:
|
||||
raise RuntimeError(f"Missing API handler for node_type: {node_type}")
|
||||
local_ds.register_handler(
|
||||
api_handler_class(
|
||||
**_api_handler_kwargs(
|
||||
node_type,
|
||||
config.local_datasource,
|
||||
config.target_project_ids,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
if isinstance(config.remote_datasource, JsonlDataSourceConfig):
|
||||
jsonl_handler_class = DomainRegistry.get_jsonl_handler(node_type)
|
||||
if jsonl_handler_class is None:
|
||||
raise RuntimeError(f"Missing JSONL handler for node_type: {node_type}")
|
||||
remote_ds.register_handler(jsonl_handler_class(remote_ds))
|
||||
else:
|
||||
api_handler_class = DomainRegistry.get_api_handler(node_type)
|
||||
if api_handler_class is None:
|
||||
raise RuntimeError(f"Missing API handler for node_type: {node_type}")
|
||||
remote_ds.register_handler(
|
||||
api_handler_class(
|
||||
**_api_handler_kwargs(
|
||||
node_type,
|
||||
config.remote_datasource,
|
||||
config.target_project_ids,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _resolve_strategy_map(config: PipelineRunConfig, node_types: List[str]) -> Dict[str, StrategyRuntimeConfig]:
|
||||
if config.strategies:
|
||||
return {item.node_type: item for item in config.strategies}
|
||||
|
||||
resolved: Dict[str, StrategyRuntimeConfig] = {}
|
||||
for node_type in node_types:
|
||||
strategy_class = DomainRegistry.get_strategy_class(node_type)
|
||||
if strategy_class is None:
|
||||
continue
|
||||
resolved[node_type] = StrategyRuntimeConfig(
|
||||
node_type=node_type,
|
||||
config=strategy_class.default_config.model_copy(deep=True),
|
||||
)
|
||||
return resolved
|
||||
|
||||
|
||||
def _build_strategies(
|
||||
config: PipelineRunConfig,
|
||||
node_types: List[str],
|
||||
local_collection: DataCollection,
|
||||
remote_collection: DataCollection,
|
||||
binding_manager: BindingManager,
|
||||
strategy_map: Dict[str, StrategyRuntimeConfig],
|
||||
) -> List[BaseSyncStrategy[Any]]:
|
||||
strategies: List[BaseSyncStrategy[Any]] = []
|
||||
both_jsonl = isinstance(config.local_datasource, JsonlDataSourceConfig) and isinstance(config.remote_datasource, JsonlDataSourceConfig)
|
||||
|
||||
for node_type in node_types:
|
||||
strategy_class = DomainRegistry.get_strategy_class(node_type)
|
||||
if strategy_class is None:
|
||||
raise RuntimeError(f"Missing strategy for node_type: {node_type}")
|
||||
strategy = strategy_class(node_type, local_collection, remote_collection, binding_manager)
|
||||
|
||||
if both_jsonl:
|
||||
strategy.update_config(
|
||||
local_orphan_action=OrphanAction.CREATE_REMOTE,
|
||||
remote_orphan_action=OrphanAction.NONE,
|
||||
update_direction=UpdateDirection.PUSH,
|
||||
)
|
||||
if node_type == "contract":
|
||||
strategy.update_config(depend_fields={"project_id": "project"})
|
||||
|
||||
runtime_cfg = strategy_map.get(node_type)
|
||||
if runtime_cfg is not None:
|
||||
strategy.set_config(runtime_cfg.config.model_copy(deep=True))
|
||||
|
||||
strategies.append(strategy)
|
||||
return strategies
|
||||
|
||||
|
||||
async def create_pipeline_from_config(config: PipelineRunConfig) -> FullSyncPipeline:
|
||||
if config.logging.initialize and not config.logging.file_path:
|
||||
config.logging.file_path = resolve_log_file_path(config.logging)
|
||||
|
||||
_ensure_pipeline_logger(
|
||||
initialize_logger=config.logging.initialize,
|
||||
logger_file_path=config.logging.file_path,
|
||||
logger_level=config.logging.level,
|
||||
suppress_node_logs=config.logging.suppress_node_logs,
|
||||
)
|
||||
|
||||
all_types = list(config.node_types)
|
||||
strategy_map = _resolve_strategy_map(config, all_types)
|
||||
local_target_project_ids, remote_target_project_ids = _resolve_datasource_target_project_ids(config)
|
||||
|
||||
if config.persist.backend != "sqlite":
|
||||
raise NotImplementedError(f"Persistence backend not supported yet: {config.persist.backend}")
|
||||
|
||||
db_path = Path(config.persist.db_path)
|
||||
if config.persist.wipe_on_start and db_path.exists():
|
||||
db_path.unlink()
|
||||
|
||||
persistence: Optional[PersistenceBackend] = None
|
||||
local_ds = None
|
||||
remote_ds = None
|
||||
try:
|
||||
persistence = PersistenceBackend(str(config.persist.db_path), backend=config.persist.backend)
|
||||
await persistence.initialize()
|
||||
|
||||
local_ds, remote_ds = await _create_datasources(config)
|
||||
local_cfg_with_target = config.local_datasource.model_copy(
|
||||
update={"target_project_ids": list(local_target_project_ids)},
|
||||
deep=True,
|
||||
)
|
||||
remote_cfg_with_target = config.remote_datasource.model_copy(
|
||||
update={"target_project_ids": list(remote_target_project_ids)},
|
||||
deep=True,
|
||||
)
|
||||
handler_target_config = config.model_copy(
|
||||
update={
|
||||
"local_datasource": local_cfg_with_target,
|
||||
"remote_datasource": remote_cfg_with_target,
|
||||
},
|
||||
deep=True,
|
||||
)
|
||||
|
||||
_register_handlers(handler_target_config, local_ds, remote_ds, all_types)
|
||||
local_ds.set_target_project_ids(local_target_project_ids)
|
||||
remote_ds.set_target_project_ids(remote_target_project_ids)
|
||||
|
||||
local_collection = DataCollection("local", persistence, auto_persist=False)
|
||||
remote_collection = DataCollection("remote", persistence, auto_persist=False)
|
||||
binding_manager = BindingManager(persistence, auto_persist=False)
|
||||
|
||||
strategies = _build_strategies(
|
||||
config,
|
||||
all_types,
|
||||
local_collection,
|
||||
remote_collection,
|
||||
binding_manager,
|
||||
strategy_map,
|
||||
)
|
||||
|
||||
pipeline = FullSyncPipeline(
|
||||
local_datasource=local_ds,
|
||||
remote_datasource=remote_ds,
|
||||
strategies=strategies,
|
||||
node_types=all_types,
|
||||
persistence=persistence,
|
||||
local_collection=local_collection,
|
||||
remote_collection=remote_collection,
|
||||
binding_manager=binding_manager,
|
||||
enable_persistence=config.persist.enable,
|
||||
target_project_ids=config.target_project_ids,
|
||||
local_target_project_ids=local_target_project_ids,
|
||||
remote_target_project_ids=remote_target_project_ids,
|
||||
)
|
||||
|
||||
return pipeline
|
||||
except Exception:
|
||||
async def _safe_close(name: str, resource: ApiDataSource | JsonlDataSource | PersistenceBackend | None) -> None:
|
||||
if resource is None:
|
||||
return
|
||||
try:
|
||||
await asyncio.wait_for(resource.close(), timeout=3.0)
|
||||
except Exception as close_exc:
|
||||
logger.warning("Failed to close %s during create cleanup: %s", name, close_exc)
|
||||
|
||||
await _safe_close("remote_datasource", remote_ds)
|
||||
await _safe_close("local_datasource", local_ds)
|
||||
await _safe_close("persistence", persistence)
|
||||
raise
|
||||
|
||||
|
||||
async def run_pipeline_from_config(
|
||||
config: PipelineRunConfig,
|
||||
*,
|
||||
title: str = "sync_state_machine pipeline",
|
||||
print_summary: bool = True,
|
||||
) -> Dict[str, Any]:
|
||||
if config.logging.initialize and not config.logging.file_path:
|
||||
config.logging.file_path = resolve_log_file_path(config.logging)
|
||||
|
||||
_ensure_pipeline_logger(
|
||||
initialize_logger=config.logging.initialize,
|
||||
logger_file_path=config.logging.file_path,
|
||||
logger_level=config.logging.level,
|
||||
suppress_node_logs=config.logging.suppress_node_logs,
|
||||
)
|
||||
|
||||
resolved_cfg = config_snapshot(config)
|
||||
resolved_cfg_text = json.dumps(resolved_cfg, ensure_ascii=False, indent=2)
|
||||
logger.info(
|
||||
"Resolved Config Summary: node_types=%d local=%s remote=%s",
|
||||
len(config.node_types),
|
||||
config.local_datasource.type,
|
||||
config.remote_datasource.type,
|
||||
)
|
||||
if config.logging.file_path:
|
||||
try:
|
||||
with open(config.logging.file_path, "a", encoding="utf-8") as fp:
|
||||
fp.write("Resolved Config:\n")
|
||||
fp.write(resolved_cfg_text)
|
||||
fp.write("\n")
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to write resolved config into log file: %s", exc)
|
||||
|
||||
pipeline: Optional[FullSyncPipeline] = None
|
||||
try:
|
||||
if print_summary:
|
||||
print("\n" + "=" * 80)
|
||||
print(f"🚀 Creating {title}")
|
||||
print(f"📝 Log file: {config.logging.file_path or '-'}")
|
||||
print(
|
||||
f"🔧 Resolved Config Summary: node_types={len(config.node_types)} "
|
||||
f"local={config.local_datasource.type} "
|
||||
f"remote={config.remote_datasource.type}"
|
||||
)
|
||||
print(f"🔧 Resolved Config:\n{resolved_cfg_text}")
|
||||
print("=" * 80)
|
||||
|
||||
pipeline = await create_pipeline_from_config(config)
|
||||
stats = await pipeline.run()
|
||||
|
||||
if print_summary:
|
||||
print(f"\n✅ Pipeline completed: stats={stats}")
|
||||
print(f"🗃️ Persistence DB: {config.persist.db_path}")
|
||||
|
||||
return stats
|
||||
finally:
|
||||
if pipeline is not None:
|
||||
await pipeline.close()
|
||||
|
||||
|
||||
async def create_jsonl_to_api_pipeline(
|
||||
*,
|
||||
# 路径配置
|
||||
local_jsonl_dir: str,
|
||||
persistence_db: str,
|
||||
# API 配置
|
||||
api_base_url: str,
|
||||
api_uid: str,
|
||||
api_secret: str,
|
||||
# 节点类型
|
||||
node_types: List[str],
|
||||
# 项目过滤
|
||||
target_project_ids: Optional[List[str]] = None,
|
||||
strategy_overrides: Optional[Dict[str, Dict[str, Any]]] = None,
|
||||
# Pipeline 行为
|
||||
enable_persistence: bool = True,
|
||||
wipe_persistence_on_start: bool = False,
|
||||
# API 配置
|
||||
api_debug: bool = False,
|
||||
poll_max_retries: int = 10,
|
||||
poll_interval: float = 0.5,
|
||||
api_rate_limit_max_requests: int = 30,
|
||||
api_rate_limit_window_seconds: float = 10.0,
|
||||
# Handler 自定义配置（可选，作用于 API 端）
|
||||
handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
|
||||
# Logger 配置
|
||||
initialize_logger: bool = True,
|
||||
logger_file_path: Optional[str] = None,
|
||||
logger_level: int = logging.INFO,
|
||||
) -> FullSyncPipeline:
|
||||
cfg = PipelineRunConfig(
|
||||
node_types=node_types,
|
||||
target_project_ids=target_project_ids or [],
|
||||
local_datasource=JsonlDataSourceConfig(type="jsonl", jsonl_dir=local_jsonl_dir, read_only=False),
|
||||
remote_datasource=ApiDataSourceConfig(
|
||||
type="api",
|
||||
api_base_url=api_base_url,
|
||||
api_uid=api_uid,
|
||||
api_secret=api_secret,
|
||||
api_debug=api_debug,
|
||||
poll_max_retries=poll_max_retries,
|
||||
poll_interval=poll_interval,
|
||||
api_rate_limit_max_requests=api_rate_limit_max_requests,
|
||||
api_rate_limit_window_seconds=api_rate_limit_window_seconds,
|
||||
target_project_ids=target_project_ids or [],
|
||||
handler_configs=handler_configs or {},
|
||||
),
|
||||
strategies=[],
|
||||
persist=PersistConfig(
|
||||
db_path=persistence_db,
|
||||
enable=enable_persistence,
|
||||
wipe_on_start=wipe_persistence_on_start,
|
||||
),
|
||||
logging=LoggingConfig(
|
||||
initialize=initialize_logger,
|
||||
file_path=logger_file_path,
|
||||
level=logger_level,
|
||||
),
|
||||
)
|
||||
if strategy_overrides:
|
||||
cfg = apply_config_overrides(cfg, {"strategies": strategy_overrides})
|
||||
return await create_pipeline_from_config(cfg)
|
||||
|
||||
|
||||
async def create_api_to_jsonl_pipeline(
|
||||
*,
|
||||
# API 配置（本地）
|
||||
api_base_url: str,
|
||||
api_uid: str,
|
||||
api_secret: str,
|
||||
# 路径配置
|
||||
remote_jsonl_dir: str,
|
||||
persistence_db: str,
|
||||
# 节点类型
|
||||
node_types: List[str],
|
||||
# 项目过滤
|
||||
target_project_ids: Optional[List[str]] = None,
|
||||
strategy_overrides: Optional[Dict[str, Dict[str, Any]]] = None,
|
||||
# Pipeline 行为
|
||||
enable_persistence: bool = True,
|
||||
wipe_persistence_on_start: bool = False,
|
||||
# API 配置
|
||||
api_debug: bool = False,
|
||||
poll_max_retries: int = 10,
|
||||
poll_interval: float = 0.5,
|
||||
api_rate_limit_max_requests: int = 30,
|
||||
api_rate_limit_window_seconds: float = 10.0,
|
||||
# Handler 自定义配置（可选，作用于 API 端）
|
||||
handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
|
||||
# Logger 配置
|
||||
initialize_logger: bool = True,
|
||||
logger_file_path: Optional[str] = None,
|
||||
logger_level: int = logging.INFO,
|
||||
) -> FullSyncPipeline:
|
||||
cfg = PipelineRunConfig(
|
||||
node_types=node_types,
|
||||
target_project_ids=target_project_ids or [],
|
||||
local_datasource=ApiDataSourceConfig(
|
||||
type="api",
|
||||
api_base_url=api_base_url,
|
||||
api_uid=api_uid,
|
||||
api_secret=api_secret,
|
||||
api_debug=api_debug,
|
||||
poll_max_retries=poll_max_retries,
|
||||
poll_interval=poll_interval,
|
||||
api_rate_limit_max_requests=api_rate_limit_max_requests,
|
||||
api_rate_limit_window_seconds=api_rate_limit_window_seconds,
|
||||
target_project_ids=target_project_ids or [],
|
||||
handler_configs=handler_configs or {},
|
||||
),
|
||||
remote_datasource=JsonlDataSourceConfig(type="jsonl", jsonl_dir=remote_jsonl_dir, read_only=False),
|
||||
strategies=[],
|
||||
persist=PersistConfig(
|
||||
db_path=persistence_db,
|
||||
enable=enable_persistence,
|
||||
wipe_on_start=wipe_persistence_on_start,
|
||||
),
|
||||
logging=LoggingConfig(
|
||||
initialize=initialize_logger,
|
||||
file_path=logger_file_path,
|
||||
level=logger_level,
|
||||
),
|
||||
)
|
||||
if strategy_overrides:
|
||||
cfg = apply_config_overrides(cfg, {"strategies": strategy_overrides})
|
||||
return await create_pipeline_from_config(cfg)
|
||||
|
||||
|
||||
async def create_jsonl_to_jsonl_pipeline(
|
||||
*,
|
||||
local_jsonl_dir: str,
|
||||
remote_jsonl_dir: str,
|
||||
persistence_db: str,
|
||||
node_types: List[str],
|
||||
target_project_ids: Optional[List[str]] = None,
|
||||
enable_persistence: bool = True,
|
||||
wipe_persistence_on_start: bool = False,
|
||||
strategy_overrides: Optional[Dict[str, Dict[str, Any]]] = None,
|
||||
initialize_logger: bool = True,
|
||||
logger_file_path: Optional[str] = None,
|
||||
logger_level: int = logging.INFO,
|
||||
) -> FullSyncPipeline:
|
||||
cfg = PipelineRunConfig(
|
||||
node_types=node_types,
|
||||
target_project_ids=target_project_ids or [],
|
||||
local_datasource=JsonlDataSourceConfig(type="jsonl", jsonl_dir=local_jsonl_dir, read_only=False),
|
||||
remote_datasource=JsonlDataSourceConfig(type="jsonl", jsonl_dir=remote_jsonl_dir, read_only=False),
|
||||
strategies=[],
|
||||
persist=PersistConfig(
|
||||
db_path=persistence_db,
|
||||
enable=enable_persistence,
|
||||
wipe_on_start=wipe_persistence_on_start,
|
||||
),
|
||||
logging=LoggingConfig(
|
||||
initialize=initialize_logger,
|
||||
file_path=logger_file_path,
|
||||
level=logger_level,
|
||||
),
|
||||
)
|
||||
if strategy_overrides:
|
||||
cfg = apply_config_overrides(cfg, {"strategies": strategy_overrides})
|
||||
return await create_pipeline_from_config(cfg)
|
||||
Reference in New Issue
Block a user