strepsiades/ecm_sync_system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

多pipeline同步支持

Browse Source
This commit is contained in:
strepsiades
2026-03-23 21:02:52 +08:00
parent 0812489f9d
commit b51ed0175b
49 changed files with 2110 additions and 792 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sync_state_machine/pipeline/factory.py
+111 -24
View File
@@ -27,18 +27,23 @@ from ..datasource import ensure_builtin_datasource_types_registered
from ..datasource.type_registry import get_datasource_factory
from ..config import (
PipelineRunConfig,
MultiProjectRunConfig,
DataSourceConfig,
ApiDataSourceConfig,
PersistConfig,
JsonlDataSourceConfig,
LoggingConfig,
build_config,
StrategyRuntimeConfig,
build_config_from_file,
build_multi_project_config_from_file,
apply_config_overrides,
OrphanAction,
UpdateDirection,
resolve_log_file_path,
config_snapshot,
is_multi_project_payload,
load_overrides_from_file,
)
from ..sync_system.strategy import BaseSyncStrategy
from .run_logger import init_pipeline_logger
@@ -180,6 +185,17 @@ def _register_handlers(
)
def _build_collections_and_bindings(
*,
persistence: PersistenceBackend,
scope: str,
) -> tuple[DataCollection, DataCollection, BindingManager]:
local_collection = DataCollection("local", scope=scope, persistence=persistence, auto_persist=False)
remote_collection = DataCollection("remote", scope=scope, persistence=persistence, auto_persist=False)
binding_manager = BindingManager(persistence, auto_persist=False, scope=scope)
return local_collection, remote_collection, binding_manager
def _resolve_strategy_map(config: PipelineRunConfig, node_types: List[str]) -> Dict[str, StrategyRuntimeConfig]:
if config.strategies:
return {item.node_type: item for item in config.strategies}
@@ -196,6 +212,28 @@ def _resolve_strategy_map(config: PipelineRunConfig, node_types: List[str]) -> D
return resolved
def _build_pipeline_run_config(
*,
node_types: List[str],
local_datasource: DataSourceConfig,
remote_datasource: DataSourceConfig,
persist: PersistConfig,
logging_config: LoggingConfig,
strategy_overrides: Optional[Dict[str, Dict[str, Any]]] = None,
) -> PipelineRunConfig:
cfg = PipelineRunConfig(
node_types=node_types,
local_datasource=local_datasource,
remote_datasource=remote_datasource,
strategies=[],
persist=persist,
logging=logging_config,
)
if strategy_overrides:
cfg = apply_config_overrides(cfg, {"strategies": strategy_overrides})
return cfg
def _build_strategies(
config: PipelineRunConfig,
node_types: List[str],
@@ -225,6 +263,7 @@ def _build_strategies(
runtime_cfg = strategy_map.get(node_type)
if runtime_cfg is not None:
strategy.set_config(runtime_cfg.config.model_copy(deep=True))
strategy.skip_sync = bool(runtime_cfg.skip_sync)
strategies.append(strategy)
return strategies
@@ -233,6 +272,10 @@ def _build_strategies(
async def create_pipeline_from_config(
config: PipelineRunConfig,
*,
scope: str = "global",
pipeline_name: str | None = None,
bootstrap_binding_node_types: Optional[List[str]] = None,
ephemeral_node_types: Optional[List[str]] = None,
local_datasource_override=None,
remote_datasource_override=None,
) -> FullSyncPipeline:
@@ -279,9 +322,10 @@ async def create_pipeline_from_config(
await _initialize_datasource(remote_ds, endpoint_name="remote")
_register_handlers(config, local_ds, remote_ds, all_types)
local_collection = DataCollection("local", persistence, auto_persist=False)
remote_collection = DataCollection("remote", persistence, auto_persist=False)
binding_manager = BindingManager(persistence, auto_persist=False)
local_collection, remote_collection, binding_manager = _build_collections_and_bindings(
persistence=persistence,
scope=scope,
)
strategies = _build_strategies(
config,
@@ -301,6 +345,10 @@ async def create_pipeline_from_config(
local_collection=local_collection,
remote_collection=remote_collection,
binding_manager=binding_manager,
scope=scope,
pipeline_name=pipeline_name,
bootstrap_binding_node_types=bootstrap_binding_node_types,
ephemeral_node_types=ephemeral_node_types,
enable_persistence=config.persist.enable,
)
@@ -325,6 +373,8 @@ async def run_pipeline_from_config(
*,
title: str = "sync_state_machine pipeline",
print_summary: bool = True,
scope: str = "global",
bootstrap_binding_node_types: Optional[List[str]] = None,
local_datasource_override=None,
remote_datasource_override=None,
) -> Dict[str, Any]:
@@ -341,7 +391,7 @@ async def run_pipeline_from_config(
resolved_cfg = config_snapshot(config)
resolved_cfg_text = json.dumps(resolved_cfg, ensure_ascii=False, indent=2)
logger.info(
"Resolved Config Summary: node_types=%d local=%s remote=%s",
"Resolved Full Config Summary: node_types=%d local=%s remote=%s",
len(config.node_types),
config.local_datasource.type,
config.remote_datasource.type,
@@ -349,7 +399,7 @@ async def run_pipeline_from_config(
if config.logging.file_path:
try:
with open(config.logging.file_path, "a", encoding="utf-8") as fp:
fp.write("Resolved Config:\n")
fp.write("Resolved Full Config:\n")
fp.write(resolved_cfg_text)
fp.write("\n")
except Exception as exc:
@@ -362,15 +412,18 @@ async def run_pipeline_from_config(
print(f"🚀 Creating {title}")
print(f"📝 Log file: {config.logging.file_path or '-'}")
print(
f"🔧 Resolved Config Summary: node_types={len(config.node_types)} "
f"🔧 Resolved Full Config Summary: node_types={len(config.node_types)} "
f"local={config.local_datasource.type} "
f"remote={config.remote_datasource.type}"
)
print(f"🔧 Resolved Config:\n{resolved_cfg_text}")
print(f"🔧 Resolved Full Config:\n{resolved_cfg_text}")
print("=" * 80)
pipeline = await create_pipeline_from_config(
config,
scope=scope,
pipeline_name=title,
bootstrap_binding_node_types=bootstrap_binding_node_types,
local_datasource_override=local_datasource_override,
remote_datasource_override=remote_datasource_override,
)
@@ -390,12 +443,20 @@ async def create_pipeline_from_file(
*,
project_root: Path,
config_path: str | Path,
scope: str = "global",
pipeline_name: str | None = None,
bootstrap_binding_node_types: Optional[List[str]] = None,
ephemeral_node_types: Optional[List[str]] = None,
local_datasource_override=None,
remote_datasource_override=None,
) -> FullSyncPipeline:
config = build_config_from_file(project_root=project_root, file_path=config_path)
return await create_pipeline_from_config(
config,
scope=scope,
pipeline_name=pipeline_name,
bootstrap_binding_node_types=bootstrap_binding_node_types,
ephemeral_node_types=ephemeral_node_types,
local_datasource_override=local_datasource_override,
remote_datasource_override=remote_datasource_override,
)
@@ -407,6 +468,8 @@ async def run_pipeline_from_file(
config_path: str | Path,
title: str = "sync_state_machine pipeline",
print_summary: bool = True,
scope: str = "global",
bootstrap_binding_node_types: Optional[List[str]] = None,
local_datasource_override=None,
remote_datasource_override=None,
) -> Dict[str, Any]:
@@ -415,11 +478,41 @@ async def run_pipeline_from_file(
config,
title=title,
print_summary=print_summary,
scope=scope,
bootstrap_binding_node_types=bootstrap_binding_node_types,
local_datasource_override=local_datasource_override,
remote_datasource_override=remote_datasource_override,
)
async def run_profile_from_file(
*,
project_root: Path,
config_path: str | Path,
print_summary: bool = True,
) -> Dict[str, Any]:
cfg_path = Path(config_path)
raw = load_overrides_from_file(cfg_path, project_root)
if is_multi_project_payload(raw):
from .multi_project_pipeline import MultiProjectPipeline
multi_cfg = MultiProjectRunConfig.model_validate(raw)
pipeline = MultiProjectPipeline(project_root=project_root, config_path=cfg_path, config=multi_cfg)
if print_summary:
print("\n" + "=" * 80)
print(f"🚀 Creating multi-project pipeline from: {cfg_path}")
print("=" * 80)
return await pipeline.run()
config = build_config(project_root=project_root, overrides=raw)
mode = f"{config.local_datasource.type}->{config.remote_datasource.type}"
return await run_pipeline_from_config(
config,
title=f"sync_state_machine pipeline ({mode})",
print_summary=print_summary,
)
async def create_jsonl_to_api_pipeline(
*,
# 路径配置
@@ -438,7 +531,7 @@ async def create_jsonl_to_api_pipeline(
# API 配置
api_debug: bool = False,
poll_max_retries: int = 10,
poll_interval: float = 0.5,
poll_interval: float = 0.5,
api_rate_limit_max_requests: int = 30,
api_rate_limit_window_seconds: float = 10.0,
local_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
@@ -448,7 +541,7 @@ async def create_jsonl_to_api_pipeline(
logger_file_path: Optional[str] = None,
logger_level: int = logging.INFO,
) -> FullSyncPipeline:
cfg = PipelineRunConfig(
cfg = _build_pipeline_run_config(
node_types=node_types,
local_datasource=JsonlDataSourceConfig(
type="jsonl",
@@ -468,20 +561,18 @@ async def create_jsonl_to_api_pipeline(
api_rate_limit_window_seconds=api_rate_limit_window_seconds,
handler_configs=remote_handler_configs or {},
),
strategies=[],
persist=PersistConfig(
db_path=persistence_db,
enable=enable_persistence,
wipe_on_start=wipe_persistence_on_start,
),
logging=LoggingConfig(
logging_config=LoggingConfig(
initialize=initialize_logger,
file_path=logger_file_path,
level=logger_level,
),
strategy_overrides=strategy_overrides,
)
if strategy_overrides:
cfg = apply_config_overrides(cfg, {"strategies": strategy_overrides})
return await create_pipeline_from_config(cfg)
@@ -503,7 +594,7 @@ async def create_api_to_jsonl_pipeline(
# API 配置
api_debug: bool = False,
poll_max_retries: int = 10,
poll_interval: float = 0.5,
poll_interval: float = 0.5,
api_rate_limit_max_requests: int = 30,
api_rate_limit_window_seconds: float = 10.0,
local_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
@@ -513,7 +604,7 @@ async def create_api_to_jsonl_pipeline(
logger_file_path: Optional[str] = None,
logger_level: int = logging.INFO,
) -> FullSyncPipeline:
cfg = PipelineRunConfig(
cfg = _build_pipeline_run_config(
node_types=node_types,
local_datasource=ApiDataSourceConfig(
type="api",
@@ -533,20 +624,18 @@ async def create_api_to_jsonl_pipeline(
read_only=False,
handler_configs=remote_handler_configs or {},
),
strategies=[],
persist=PersistConfig(
db_path=persistence_db,
enable=enable_persistence,
wipe_on_start=wipe_persistence_on_start,
),
logging=LoggingConfig(
logging_config=LoggingConfig(
initialize=initialize_logger,
file_path=logger_file_path,
level=logger_level,
),
strategy_overrides=strategy_overrides,
)
if strategy_overrides:
cfg = apply_config_overrides(cfg, {"strategies": strategy_overrides})
return await create_pipeline_from_config(cfg)
@@ -565,7 +654,7 @@ async def create_jsonl_to_jsonl_pipeline(
logger_file_path: Optional[str] = None,
logger_level: int = logging.INFO,
) -> FullSyncPipeline:
cfg = PipelineRunConfig(
cfg = _build_pipeline_run_config(
node_types=node_types,
local_datasource=JsonlDataSourceConfig(
type="jsonl",
@@ -579,18 +668,16 @@ async def create_jsonl_to_jsonl_pipeline(
read_only=False,
handler_configs=remote_handler_configs or {},
),
strategies=[],
persist=PersistConfig(
db_path=persistence_db,
enable=enable_persistence,
wipe_on_start=wipe_persistence_on_start,
),
logging=LoggingConfig(
logging_config=LoggingConfig(
initialize=initialize_logger,
file_path=logger_file_path,
level=logger_level,
),
strategy_overrides=strategy_overrides,
)
if strategy_overrides:
cfg = apply_config_overrides(cfg, {"strategies": strategy_overrides})
return await create_pipeline_from_config(cfg)