first commit
This commit is contained in:
strepsiades
@@ -0,0 +1,47 @@
|
||||
"""
|
||||
Pipeline Layer - Pipeline orchestration and workflow management
|
||||
"""
|
||||
|
||||
from .copy_data_pipeline import CopyDataPipeline
|
||||
from .full_sync_pipeline import FullSyncPipeline
|
||||
from ..config import (
|
||||
PipelineRunConfig,
|
||||
ApiDataSourceConfig,
|
||||
JsonlDataSourceConfig,
|
||||
DataSourceConfig,
|
||||
PersistConfig,
|
||||
LoggingConfig,
|
||||
build_default_config,
|
||||
build_config,
|
||||
apply_config_overrides,
|
||||
apply_env_overrides,
|
||||
config_snapshot,
|
||||
)
|
||||
from .factory import (
|
||||
create_jsonl_to_api_pipeline,
|
||||
create_api_to_jsonl_pipeline,
|
||||
create_jsonl_to_jsonl_pipeline,
|
||||
create_pipeline_from_config,
|
||||
run_pipeline_from_config,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"CopyDataPipeline",
|
||||
"FullSyncPipeline",
|
||||
"create_jsonl_to_api_pipeline",
|
||||
"create_jsonl_to_jsonl_pipeline",
|
||||
"create_api_to_jsonl_pipeline",
|
||||
"create_pipeline_from_config",
|
||||
"run_pipeline_from_config",
|
||||
"PipelineRunConfig",
|
||||
"ApiDataSourceConfig",
|
||||
"JsonlDataSourceConfig",
|
||||
"DataSourceConfig",
|
||||
"PersistConfig",
|
||||
"LoggingConfig",
|
||||
"build_default_config",
|
||||
"build_config",
|
||||
"apply_config_overrides",
|
||||
"apply_env_overrides",
|
||||
"config_snapshot",
|
||||
]
|
||||
@@ -0,0 +1,247 @@
|
||||
"""
|
||||
Copy Data Pipeline - Simple data copying without strategy execution
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import List, Any, Optional
|
||||
|
||||
from ..common.collection import DataCollection
|
||||
from ..common.types import SyncAction, SyncStatus
|
||||
from ..common.registry import DomainRegistry
|
||||
from ..datasource import JsonlDataSource
|
||||
from ..engine import StateMachineConfig, StateMachineRuntime, apply_state_to_node
|
||||
|
||||
|
||||
class CopyDataPipeline:
|
||||
"""
|
||||
通用数据复制 Pipeline，支持任意数据源之间的复制。
|
||||
|
||||
流程：
|
||||
1. 从 source_datasource 加载所有数据到 source_collection
|
||||
2. 将每个节点复制到 target_collection，设置 action=CREATE
|
||||
3. 使用 target_datasource 保存所有节点
|
||||
|
||||
用途：
|
||||
- 数据迁移 (JSONL → JSONL)
|
||||
- API 数据下载 (API → JSONL)
|
||||
- 测试数据准备
|
||||
- 初始化同步
|
||||
|
||||
支持两种初始化方式：
|
||||
1. 传入数据源实例 (推荐): source_datasource + target_datasource
|
||||
2. 传入目录路径 (兼容): source_dir + target_dir (自动创建 JsonlDataSource)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
source_datasource: Any = None,
|
||||
target_datasource: Any = None,
|
||||
source_dir: Optional[Path | str] = None,
|
||||
target_dir: Optional[Path | str] = None,
|
||||
node_types: List[str] | None = None,
|
||||
load_order: List[str] | None = None,
|
||||
save_order: List[str] | None = None,
|
||||
):
|
||||
"""
|
||||
初始化 CopyDataPipeline
|
||||
|
||||
Args:
|
||||
source_datasource: 源数据源实例 (ApiDataSource, JsonlDataSource 等)
|
||||
target_datasource: 目标数据源实例
|
||||
source_dir: 源数据目录 (兼容模式，会创建 JsonlDataSource)
|
||||
target_dir: 目标数据目录 (兼容模式，会创建 JsonlDataSource)
|
||||
node_types: 要复制的节点类型列表，None 表示复制所有注册的类型
|
||||
load_order: 数据加载顺序，None 表示使用 node_types 顺序
|
||||
save_order: 数据保存顺序，None 表示使用 node_types 顺序
|
||||
"""
|
||||
self._supported_types: Optional[List[str]] = None
|
||||
|
||||
# 数据源处理 - 优先使用实例，其次使用目录
|
||||
if source_datasource is not None:
|
||||
self.source_datasource = source_datasource
|
||||
self._owns_source = False
|
||||
elif source_dir is not None:
|
||||
from ..domain.registry import JSONL_HANDLERS
|
||||
self.source_datasource = JsonlDataSource(dir_path=Path(source_dir), read_only=True)
|
||||
# 自动注册所有 JSONL handlers，并收集支持的类型
|
||||
supported_types = []
|
||||
for handler_class in JSONL_HANDLERS:
|
||||
handler = handler_class(self.source_datasource)
|
||||
self.source_datasource.register_handler(handler)
|
||||
supported_types.append(handler.node_type)
|
||||
self._owns_source = True
|
||||
self._supported_types = supported_types
|
||||
else:
|
||||
raise ValueError("Must provide either source_datasource or source_dir")
|
||||
|
||||
if target_datasource is not None:
|
||||
self.target_datasource = target_datasource
|
||||
self._owns_target = False
|
||||
elif target_dir is not None:
|
||||
from ..domain.registry import JSONL_HANDLERS
|
||||
self.target_datasource = JsonlDataSource(dir_path=Path(target_dir), read_only=False)
|
||||
# 自动注册所有 JSONL handlers
|
||||
for handler_class in JSONL_HANDLERS:
|
||||
self.target_datasource.register_handler(handler_class(self.target_datasource))
|
||||
self._owns_target = True
|
||||
else:
|
||||
raise ValueError("Must provide either target_datasource or target_dir")
|
||||
|
||||
# 当使用 source_dir 创建时，如果 node_types 未指定，只使用支持的类型
|
||||
if node_types is None and self._supported_types is not None:
|
||||
self.node_types = self._supported_types
|
||||
else:
|
||||
self.node_types = node_types or DomainRegistry.get_registered_types()
|
||||
self.load_order = load_order or self.node_types
|
||||
self.save_order = save_order or self.node_types
|
||||
cfg_path = Path(__file__).resolve().parents[1] / "config" / "node_state_machine.yaml"
|
||||
self._sm_runtime = StateMachineRuntime(StateMachineConfig.load(cfg_path))
|
||||
|
||||
# 统计信息
|
||||
self.stats = {
|
||||
"loaded": 0,
|
||||
"copied": 0,
|
||||
"saved": 0,
|
||||
"failed": 0,
|
||||
}
|
||||
|
||||
async def run(self):
|
||||
"""
|
||||
执行完整的复制流程
|
||||
|
||||
Returns:
|
||||
dict: 统计信息 {"loaded": int, "copied": int, "saved": int, "failed": int}
|
||||
"""
|
||||
print(f"[CopyDataPipeline] Starting...")
|
||||
print(f" Source: {self.source_datasource.__class__.__name__}")
|
||||
print(f" Target: {self.target_datasource.__class__.__name__}")
|
||||
print(f" Node types: {self.node_types}")
|
||||
|
||||
# Stage 1: 加载源数据
|
||||
print("\n[Stage 1] Loading source data...")
|
||||
source_collection = DataCollection(collection_id="source")
|
||||
await self._load_source_data(source_collection)
|
||||
|
||||
# Stage 2: 复制节点
|
||||
print("\n[Stage 2] Copying nodes...")
|
||||
target_collection = DataCollection(collection_id="target")
|
||||
await self._copy_nodes(source_collection, target_collection)
|
||||
|
||||
# Stage 3: 保存到目标
|
||||
print("\n[Stage 3] Saving to target...")
|
||||
await self._save_target_data(target_collection)
|
||||
|
||||
# 打印统计信息
|
||||
self._print_summary()
|
||||
|
||||
return self.stats
|
||||
|
||||
async def _load_source_data(self, collection: DataCollection):
|
||||
"""从源数据源加载所有节点"""
|
||||
# 如果是 JsonlDataSource 且没有注册 handlers，自动注册
|
||||
if isinstance(self.source_datasource, JsonlDataSource) and not self.source_datasource._handlers:
|
||||
self._auto_register_jsonl_handlers(self.source_datasource)
|
||||
|
||||
self.source_datasource.set_collection(collection)
|
||||
await self.source_datasource.load_all(order=self.load_order)
|
||||
|
||||
# 统计加载的节点数
|
||||
for node_type in self.node_types:
|
||||
count = len(collection.filter(node_type=node_type))
|
||||
if count > 0:
|
||||
print(f" [LOAD] {node_type}: {count} nodes")
|
||||
self.stats["loaded"] += count
|
||||
|
||||
async def _copy_nodes(
|
||||
self,
|
||||
source_collection: DataCollection,
|
||||
target_collection: DataCollection
|
||||
):
|
||||
"""
|
||||
复制所有节点到目标 collection，设置 action=CREATE, status=PENDING
|
||||
|
||||
注意：
|
||||
- 保持相同的 node_id（用于追踪）
|
||||
- data_id 设为空字符串（待创建）
|
||||
- 复制 data 和 depend_ids
|
||||
"""
|
||||
for node in source_collection.filter(): # filter() 不带参数返回所有节点
|
||||
# 获取节点类
|
||||
node_class = DomainRegistry.get_node_class(node.node_type)
|
||||
|
||||
# 创建新节点
|
||||
new_node = node_class(
|
||||
node_id=node.node_id, # 保持相同的 node_id
|
||||
data_id="", # CREATE 节点的 data_id 初始为空
|
||||
data=node.data, # 复制数据
|
||||
depend_ids=node.depend_ids.copy() if node.depend_ids else [],
|
||||
)
|
||||
|
||||
apply_state_to_node(
|
||||
self._sm_runtime,
|
||||
new_node,
|
||||
state_id="S06",
|
||||
reason="copy pipeline init",
|
||||
fields={"binding_status", "action", "status"},
|
||||
)
|
||||
apply_state_to_node(
|
||||
self._sm_runtime,
|
||||
new_node,
|
||||
state_id="S06",
|
||||
reason="copy pipeline init",
|
||||
fields={"data_id"},
|
||||
data_id="",
|
||||
)
|
||||
|
||||
await target_collection.add(new_node)
|
||||
self.stats["copied"] += 1
|
||||
|
||||
print(f" Copied {self.stats['copied']} nodes")
|
||||
|
||||
async def _save_target_data(self, collection: DataCollection):
|
||||
"""保存所有节点到目标数据源"""
|
||||
# 如果是 JsonlDataSource 且没有注册 handlers，自动注册
|
||||
if isinstance(self.target_datasource, JsonlDataSource) and not self.target_datasource._handlers:
|
||||
self._auto_register_jsonl_handlers(self.target_datasource)
|
||||
|
||||
self.target_datasource.set_collection(collection)
|
||||
await self.target_datasource.sync_all(order=self.save_order)
|
||||
|
||||
# 统计保存结果
|
||||
for node_type in self.node_types:
|
||||
nodes = collection.filter(node_type=node_type)
|
||||
success_count = sum(1 for n in nodes if n.status == SyncStatus.SUCCESS)
|
||||
failed_count = sum(1 for n in nodes if n.status == SyncStatus.FAILED)
|
||||
|
||||
if success_count > 0 or failed_count > 0:
|
||||
print(f" [SAVE] {node_type}: {success_count} success, {failed_count} failed")
|
||||
|
||||
self.stats["saved"] += success_count
|
||||
self.stats["failed"] += failed_count
|
||||
|
||||
def _auto_register_jsonl_handlers(self, datasource: JsonlDataSource):
|
||||
"""自动为 JsonlDataSource 注册需要的 handlers（兼容模式）"""
|
||||
for node_type in self.node_types:
|
||||
if not DomainRegistry.is_registered(node_type):
|
||||
continue
|
||||
|
||||
handler_class = DomainRegistry.get_jsonl_handler(node_type)
|
||||
if handler_class:
|
||||
handler = handler_class(datasource)
|
||||
datasource.register_handler(handler)
|
||||
|
||||
def _print_summary(self):
|
||||
"""打印执行摘要"""
|
||||
print("\n" + "=" * 60)
|
||||
print("[CopyDataPipeline] Execution Summary")
|
||||
print("=" * 60)
|
||||
print(f" Loaded: {self.stats['loaded']} nodes")
|
||||
print(f" Copied: {self.stats['copied']} nodes")
|
||||
print(f" Saved: {self.stats['saved']} nodes")
|
||||
print(f" Failed: {self.stats['failed']} nodes")
|
||||
print("=" * 60)
|
||||
|
||||
if self.stats["failed"] > 0:
|
||||
print(f"\n⚠️ {self.stats['failed']} nodes failed to save")
|
||||
else:
|
||||
print("\n✅ All nodes saved successfully")
|
||||
@@ -0,0 +1,578 @@
|
||||
"""
|
||||
Pipeline Factory - 工厂函数，负责初始化所有组件并创建 Pipeline
|
||||
|
||||
用户只需提供配置参数，工厂函数负责：
|
||||
1. 初始化 Persistence
|
||||
2. 初始化 DataSource（并注册 Handler）
|
||||
3. 初始化 Collection 和 BindingManager
|
||||
4. 初始化 Strategy（并注入依赖）
|
||||
5. 创建 Pipeline（注入所有组件）
|
||||
6. 返回完全就绪的 Pipeline
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from ..common.binding import BindingManager
|
||||
from ..common.collection import DataCollection
|
||||
from ..common.persistence import PersistenceBackend
|
||||
from ..common.registry import DomainRegistry
|
||||
from ..datasource.api.datasource import ApiDataSource
|
||||
from ..datasource.jsonl.datasource import JsonlDataSource
|
||||
from ..config import (
|
||||
PipelineRunConfig,
|
||||
DataSourceConfig,
|
||||
ApiDataSourceConfig,
|
||||
JsonlDataSourceConfig,
|
||||
PersistConfig,
|
||||
LoggingConfig,
|
||||
StrategyRuntimeConfig,
|
||||
apply_config_overrides,
|
||||
OrphanAction,
|
||||
UpdateDirection,
|
||||
resolve_log_file_path,
|
||||
config_snapshot,
|
||||
)
|
||||
from ..sync_system.strategy import BaseSyncStrategy
|
||||
from .run_logger import init_pipeline_logger
|
||||
from .full_sync_pipeline import FullSyncPipeline
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _project_root() -> Path:
|
||||
return Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def _prepare_remote_jsonl_dir(remote_dir: Path, *, project_root: Optional[Path] = None) -> Path:
|
||||
project_root_path = (project_root or _project_root()).resolve()
|
||||
resolved_remote_dir = remote_dir.resolve(strict=False)
|
||||
|
||||
if resolved_remote_dir.exists():
|
||||
if not resolved_remote_dir.is_dir():
|
||||
raise FileNotFoundError(
|
||||
f"remote_datasource.jsonl_dir exists but is not a directory: {resolved_remote_dir}"
|
||||
)
|
||||
return resolved_remote_dir
|
||||
|
||||
try:
|
||||
resolved_remote_dir.relative_to(project_root_path)
|
||||
except ValueError as exc:
|
||||
raise FileNotFoundError(
|
||||
"remote_datasource.jsonl_dir does not exist and auto-create is only allowed "
|
||||
f"under project root: {resolved_remote_dir} (project_root={project_root_path})"
|
||||
) from exc
|
||||
|
||||
resolved_remote_dir.mkdir(parents=True, exist_ok=True)
|
||||
logger.info("Created missing remote JSONL directory: %s", resolved_remote_dir)
|
||||
return resolved_remote_dir
|
||||
|
||||
|
||||
def _ensure_pipeline_logger(
|
||||
*,
|
||||
initialize_logger: bool,
|
||||
logger_file_path: Optional[str],
|
||||
logger_level: int,
|
||||
suppress_node_logs: bool,
|
||||
) -> None:
|
||||
if not initialize_logger:
|
||||
return
|
||||
|
||||
app_logger = logging.getLogger("sync_state_machine")
|
||||
if app_logger.handlers:
|
||||
return
|
||||
|
||||
init_pipeline_logger(
|
||||
log_file_path=logger_file_path,
|
||||
level=logger_level,
|
||||
mirror_console=True,
|
||||
replace_handlers=False,
|
||||
suppress_node_logs=suppress_node_logs,
|
||||
)
|
||||
|
||||
|
||||
def _api_handler_kwargs(
|
||||
node_type: str,
|
||||
ds_config: DataSourceConfig,
|
||||
target_project_ids: List[str],
|
||||
) -> Dict[str, Any]:
|
||||
kwargs = dict(ds_config.handler_configs.get(node_type, {}))
|
||||
|
||||
if node_type == "project" and target_project_ids:
|
||||
kwargs.setdefault("filter_project_id", target_project_ids)
|
||||
return kwargs
|
||||
|
||||
|
||||
def _resolve_datasource_target_project_ids(config: PipelineRunConfig) -> tuple[List[str], List[str]]:
|
||||
fallback = [str(pid) for pid in config.target_project_ids if str(pid)]
|
||||
|
||||
local_cfg_ids = [str(pid) for pid in config.local_datasource.target_project_ids if str(pid)]
|
||||
remote_cfg_ids = [str(pid) for pid in config.remote_datasource.target_project_ids if str(pid)]
|
||||
|
||||
local_target_ids = local_cfg_ids if local_cfg_ids else fallback
|
||||
remote_target_ids = remote_cfg_ids if remote_cfg_ids else fallback
|
||||
return local_target_ids, remote_target_ids
|
||||
|
||||
|
||||
async def _create_datasources(config: PipelineRunConfig):
|
||||
local_cfg = config.local_datasource
|
||||
remote_cfg = config.remote_datasource
|
||||
|
||||
if isinstance(local_cfg, JsonlDataSourceConfig):
|
||||
local_dir = Path(local_cfg.jsonl_dir)
|
||||
if not local_dir.exists() or not local_dir.is_dir():
|
||||
raise FileNotFoundError(
|
||||
f"local_datasource.jsonl_dir does not exist or is not a directory: {local_dir}"
|
||||
)
|
||||
local_ds = JsonlDataSource(local_dir, read_only=local_cfg.read_only)
|
||||
else:
|
||||
local_ds = ApiDataSource(
|
||||
base_url=local_cfg.api_base_url,
|
||||
uid=local_cfg.api_uid,
|
||||
secret=local_cfg.api_secret,
|
||||
debug=local_cfg.api_debug,
|
||||
poll_max_retries=local_cfg.poll_max_retries,
|
||||
poll_interval=local_cfg.poll_interval,
|
||||
rate_limit_max_requests=local_cfg.api_rate_limit_max_requests,
|
||||
rate_limit_window_seconds=local_cfg.api_rate_limit_window_seconds,
|
||||
)
|
||||
await local_ds.initialize()
|
||||
|
||||
if isinstance(remote_cfg, JsonlDataSourceConfig):
|
||||
remote_dir = Path(remote_cfg.jsonl_dir)
|
||||
remote_dir = _prepare_remote_jsonl_dir(remote_dir)
|
||||
remote_ds = JsonlDataSource(remote_dir, read_only=remote_cfg.read_only)
|
||||
else:
|
||||
remote_ds = ApiDataSource(
|
||||
base_url=remote_cfg.api_base_url,
|
||||
uid=remote_cfg.api_uid,
|
||||
secret=remote_cfg.api_secret,
|
||||
debug=remote_cfg.api_debug,
|
||||
poll_max_retries=remote_cfg.poll_max_retries,
|
||||
poll_interval=remote_cfg.poll_interval,
|
||||
rate_limit_max_requests=remote_cfg.api_rate_limit_max_requests,
|
||||
rate_limit_window_seconds=remote_cfg.api_rate_limit_window_seconds,
|
||||
)
|
||||
await remote_ds.initialize()
|
||||
|
||||
return local_ds, remote_ds
|
||||
|
||||
|
||||
def _register_handlers(
|
||||
config: PipelineRunConfig,
|
||||
local_ds,
|
||||
remote_ds,
|
||||
all_types: List[str],
|
||||
) -> None:
|
||||
for node_type in all_types:
|
||||
if isinstance(config.local_datasource, JsonlDataSourceConfig):
|
||||
jsonl_handler_class = DomainRegistry.get_jsonl_handler(node_type)
|
||||
if jsonl_handler_class is None:
|
||||
raise RuntimeError(f"Missing JSONL handler for node_type: {node_type}")
|
||||
local_ds.register_handler(jsonl_handler_class(local_ds))
|
||||
else:
|
||||
api_handler_class = DomainRegistry.get_api_handler(node_type)
|
||||
if api_handler_class is None:
|
||||
raise RuntimeError(f"Missing API handler for node_type: {node_type}")
|
||||
local_ds.register_handler(
|
||||
api_handler_class(
|
||||
**_api_handler_kwargs(
|
||||
node_type,
|
||||
config.local_datasource,
|
||||
config.target_project_ids,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
if isinstance(config.remote_datasource, JsonlDataSourceConfig):
|
||||
jsonl_handler_class = DomainRegistry.get_jsonl_handler(node_type)
|
||||
if jsonl_handler_class is None:
|
||||
raise RuntimeError(f"Missing JSONL handler for node_type: {node_type}")
|
||||
remote_ds.register_handler(jsonl_handler_class(remote_ds))
|
||||
else:
|
||||
api_handler_class = DomainRegistry.get_api_handler(node_type)
|
||||
if api_handler_class is None:
|
||||
raise RuntimeError(f"Missing API handler for node_type: {node_type}")
|
||||
remote_ds.register_handler(
|
||||
api_handler_class(
|
||||
**_api_handler_kwargs(
|
||||
node_type,
|
||||
config.remote_datasource,
|
||||
config.target_project_ids,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _resolve_strategy_map(config: PipelineRunConfig, node_types: List[str]) -> Dict[str, StrategyRuntimeConfig]:
|
||||
if config.strategies:
|
||||
return {item.node_type: item for item in config.strategies}
|
||||
|
||||
resolved: Dict[str, StrategyRuntimeConfig] = {}
|
||||
for node_type in node_types:
|
||||
strategy_class = DomainRegistry.get_strategy_class(node_type)
|
||||
if strategy_class is None:
|
||||
continue
|
||||
resolved[node_type] = StrategyRuntimeConfig(
|
||||
node_type=node_type,
|
||||
config=strategy_class.default_config.model_copy(deep=True),
|
||||
)
|
||||
return resolved
|
||||
|
||||
|
||||
def _build_strategies(
|
||||
config: PipelineRunConfig,
|
||||
node_types: List[str],
|
||||
local_collection: DataCollection,
|
||||
remote_collection: DataCollection,
|
||||
binding_manager: BindingManager,
|
||||
strategy_map: Dict[str, StrategyRuntimeConfig],
|
||||
) -> List[BaseSyncStrategy[Any]]:
|
||||
strategies: List[BaseSyncStrategy[Any]] = []
|
||||
both_jsonl = isinstance(config.local_datasource, JsonlDataSourceConfig) and isinstance(config.remote_datasource, JsonlDataSourceConfig)
|
||||
|
||||
for node_type in node_types:
|
||||
strategy_class = DomainRegistry.get_strategy_class(node_type)
|
||||
if strategy_class is None:
|
||||
raise RuntimeError(f"Missing strategy for node_type: {node_type}")
|
||||
strategy = strategy_class(node_type, local_collection, remote_collection, binding_manager)
|
||||
|
||||
if both_jsonl:
|
||||
strategy.update_config(
|
||||
local_orphan_action=OrphanAction.CREATE_REMOTE,
|
||||
remote_orphan_action=OrphanAction.NONE,
|
||||
update_direction=UpdateDirection.PUSH,
|
||||
)
|
||||
if node_type == "contract":
|
||||
strategy.update_config(depend_fields={"project_id": "project"})
|
||||
|
||||
runtime_cfg = strategy_map.get(node_type)
|
||||
if runtime_cfg is not None:
|
||||
strategy.set_config(runtime_cfg.config.model_copy(deep=True))
|
||||
|
||||
strategies.append(strategy)
|
||||
return strategies
|
||||
|
||||
|
||||
async def create_pipeline_from_config(config: PipelineRunConfig) -> FullSyncPipeline:
|
||||
if config.logging.initialize and not config.logging.file_path:
|
||||
config.logging.file_path = resolve_log_file_path(config.logging)
|
||||
|
||||
_ensure_pipeline_logger(
|
||||
initialize_logger=config.logging.initialize,
|
||||
logger_file_path=config.logging.file_path,
|
||||
logger_level=config.logging.level,
|
||||
suppress_node_logs=config.logging.suppress_node_logs,
|
||||
)
|
||||
|
||||
all_types = list(config.node_types)
|
||||
strategy_map = _resolve_strategy_map(config, all_types)
|
||||
local_target_project_ids, remote_target_project_ids = _resolve_datasource_target_project_ids(config)
|
||||
|
||||
if config.persist.backend != "sqlite":
|
||||
raise NotImplementedError(f"Persistence backend not supported yet: {config.persist.backend}")
|
||||
|
||||
db_path = Path(config.persist.db_path)
|
||||
if config.persist.wipe_on_start and db_path.exists():
|
||||
db_path.unlink()
|
||||
|
||||
persistence: Optional[PersistenceBackend] = None
|
||||
local_ds = None
|
||||
remote_ds = None
|
||||
try:
|
||||
persistence = PersistenceBackend(str(config.persist.db_path), backend=config.persist.backend)
|
||||
await persistence.initialize()
|
||||
|
||||
local_ds, remote_ds = await _create_datasources(config)
|
||||
local_cfg_with_target = config.local_datasource.model_copy(
|
||||
update={"target_project_ids": list(local_target_project_ids)},
|
||||
deep=True,
|
||||
)
|
||||
remote_cfg_with_target = config.remote_datasource.model_copy(
|
||||
update={"target_project_ids": list(remote_target_project_ids)},
|
||||
deep=True,
|
||||
)
|
||||
handler_target_config = config.model_copy(
|
||||
update={
|
||||
"local_datasource": local_cfg_with_target,
|
||||
"remote_datasource": remote_cfg_with_target,
|
||||
},
|
||||
deep=True,
|
||||
)
|
||||
|
||||
_register_handlers(handler_target_config, local_ds, remote_ds, all_types)
|
||||
local_ds.set_target_project_ids(local_target_project_ids)
|
||||
remote_ds.set_target_project_ids(remote_target_project_ids)
|
||||
|
||||
local_collection = DataCollection("local", persistence, auto_persist=False)
|
||||
remote_collection = DataCollection("remote", persistence, auto_persist=False)
|
||||
binding_manager = BindingManager(persistence, auto_persist=False)
|
||||
|
||||
strategies = _build_strategies(
|
||||
config,
|
||||
all_types,
|
||||
local_collection,
|
||||
remote_collection,
|
||||
binding_manager,
|
||||
strategy_map,
|
||||
)
|
||||
|
||||
pipeline = FullSyncPipeline(
|
||||
local_datasource=local_ds,
|
||||
remote_datasource=remote_ds,
|
||||
strategies=strategies,
|
||||
node_types=all_types,
|
||||
persistence=persistence,
|
||||
local_collection=local_collection,
|
||||
remote_collection=remote_collection,
|
||||
binding_manager=binding_manager,
|
||||
enable_persistence=config.persist.enable,
|
||||
target_project_ids=config.target_project_ids,
|
||||
local_target_project_ids=local_target_project_ids,
|
||||
remote_target_project_ids=remote_target_project_ids,
|
||||
)
|
||||
|
||||
return pipeline
|
||||
except Exception:
|
||||
async def _safe_close(name: str, resource: ApiDataSource | JsonlDataSource | PersistenceBackend | None) -> None:
|
||||
if resource is None:
|
||||
return
|
||||
try:
|
||||
await asyncio.wait_for(resource.close(), timeout=3.0)
|
||||
except Exception as close_exc:
|
||||
logger.warning("Failed to close %s during create cleanup: %s", name, close_exc)
|
||||
|
||||
await _safe_close("remote_datasource", remote_ds)
|
||||
await _safe_close("local_datasource", local_ds)
|
||||
await _safe_close("persistence", persistence)
|
||||
raise
|
||||
|
||||
|
||||
async def run_pipeline_from_config(
|
||||
config: PipelineRunConfig,
|
||||
*,
|
||||
title: str = "sync_state_machine pipeline",
|
||||
print_summary: bool = True,
|
||||
) -> Dict[str, Any]:
|
||||
if config.logging.initialize and not config.logging.file_path:
|
||||
config.logging.file_path = resolve_log_file_path(config.logging)
|
||||
|
||||
_ensure_pipeline_logger(
|
||||
initialize_logger=config.logging.initialize,
|
||||
logger_file_path=config.logging.file_path,
|
||||
logger_level=config.logging.level,
|
||||
suppress_node_logs=config.logging.suppress_node_logs,
|
||||
)
|
||||
|
||||
resolved_cfg = config_snapshot(config)
|
||||
resolved_cfg_text = json.dumps(resolved_cfg, ensure_ascii=False, indent=2)
|
||||
logger.info(
|
||||
"Resolved Config Summary: node_types=%d local=%s remote=%s",
|
||||
len(config.node_types),
|
||||
config.local_datasource.type,
|
||||
config.remote_datasource.type,
|
||||
)
|
||||
if config.logging.file_path:
|
||||
try:
|
||||
with open(config.logging.file_path, "a", encoding="utf-8") as fp:
|
||||
fp.write("Resolved Config:\n")
|
||||
fp.write(resolved_cfg_text)
|
||||
fp.write("\n")
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to write resolved config into log file: %s", exc)
|
||||
|
||||
pipeline: Optional[FullSyncPipeline] = None
|
||||
try:
|
||||
if print_summary:
|
||||
print("\n" + "=" * 80)
|
||||
print(f"🚀 Creating {title}")
|
||||
print(f"📝 Log file: {config.logging.file_path or '-'}")
|
||||
print(
|
||||
f"🔧 Resolved Config Summary: node_types={len(config.node_types)} "
|
||||
f"local={config.local_datasource.type} "
|
||||
f"remote={config.remote_datasource.type}"
|
||||
)
|
||||
print(f"🔧 Resolved Config:\n{resolved_cfg_text}")
|
||||
print("=" * 80)
|
||||
|
||||
pipeline = await create_pipeline_from_config(config)
|
||||
stats = await pipeline.run()
|
||||
|
||||
if print_summary:
|
||||
print(f"\n✅ Pipeline completed: stats={stats}")
|
||||
print(f"🗃️ Persistence DB: {config.persist.db_path}")
|
||||
|
||||
return stats
|
||||
finally:
|
||||
if pipeline is not None:
|
||||
await pipeline.close()
|
||||
|
||||
|
||||
async def create_jsonl_to_api_pipeline(
|
||||
*,
|
||||
# 路径配置
|
||||
local_jsonl_dir: str,
|
||||
persistence_db: str,
|
||||
# API 配置
|
||||
api_base_url: str,
|
||||
api_uid: str,
|
||||
api_secret: str,
|
||||
# 节点类型
|
||||
node_types: List[str],
|
||||
# 项目过滤
|
||||
target_project_ids: Optional[List[str]] = None,
|
||||
strategy_overrides: Optional[Dict[str, Dict[str, Any]]] = None,
|
||||
# Pipeline 行为
|
||||
enable_persistence: bool = True,
|
||||
wipe_persistence_on_start: bool = False,
|
||||
# API 配置
|
||||
api_debug: bool = False,
|
||||
poll_max_retries: int = 10,
|
||||
poll_interval: float = 0.5,
|
||||
api_rate_limit_max_requests: int = 30,
|
||||
api_rate_limit_window_seconds: float = 10.0,
|
||||
# Handler 自定义配置（可选，作用于 API 端）
|
||||
handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
|
||||
# Logger 配置
|
||||
initialize_logger: bool = True,
|
||||
logger_file_path: Optional[str] = None,
|
||||
logger_level: int = logging.INFO,
|
||||
) -> FullSyncPipeline:
|
||||
cfg = PipelineRunConfig(
|
||||
node_types=node_types,
|
||||
target_project_ids=target_project_ids or [],
|
||||
local_datasource=JsonlDataSourceConfig(type="jsonl", jsonl_dir=local_jsonl_dir, read_only=False),
|
||||
remote_datasource=ApiDataSourceConfig(
|
||||
type="api",
|
||||
api_base_url=api_base_url,
|
||||
api_uid=api_uid,
|
||||
api_secret=api_secret,
|
||||
api_debug=api_debug,
|
||||
poll_max_retries=poll_max_retries,
|
||||
poll_interval=poll_interval,
|
||||
api_rate_limit_max_requests=api_rate_limit_max_requests,
|
||||
api_rate_limit_window_seconds=api_rate_limit_window_seconds,
|
||||
target_project_ids=target_project_ids or [],
|
||||
handler_configs=handler_configs or {},
|
||||
),
|
||||
strategies=[],
|
||||
persist=PersistConfig(
|
||||
db_path=persistence_db,
|
||||
enable=enable_persistence,
|
||||
wipe_on_start=wipe_persistence_on_start,
|
||||
),
|
||||
logging=LoggingConfig(
|
||||
initialize=initialize_logger,
|
||||
file_path=logger_file_path,
|
||||
level=logger_level,
|
||||
),
|
||||
)
|
||||
if strategy_overrides:
|
||||
cfg = apply_config_overrides(cfg, {"strategies": strategy_overrides})
|
||||
return await create_pipeline_from_config(cfg)
|
||||
|
||||
|
||||
async def create_api_to_jsonl_pipeline(
|
||||
*,
|
||||
# API 配置（本地）
|
||||
api_base_url: str,
|
||||
api_uid: str,
|
||||
api_secret: str,
|
||||
# 路径配置
|
||||
remote_jsonl_dir: str,
|
||||
persistence_db: str,
|
||||
# 节点类型
|
||||
node_types: List[str],
|
||||
# 项目过滤
|
||||
target_project_ids: Optional[List[str]] = None,
|
||||
strategy_overrides: Optional[Dict[str, Dict[str, Any]]] = None,
|
||||
# Pipeline 行为
|
||||
enable_persistence: bool = True,
|
||||
wipe_persistence_on_start: bool = False,
|
||||
# API 配置
|
||||
api_debug: bool = False,
|
||||
poll_max_retries: int = 10,
|
||||
poll_interval: float = 0.5,
|
||||
api_rate_limit_max_requests: int = 30,
|
||||
api_rate_limit_window_seconds: float = 10.0,
|
||||
# Handler 自定义配置（可选，作用于 API 端）
|
||||
handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
|
||||
# Logger 配置
|
||||
initialize_logger: bool = True,
|
||||
logger_file_path: Optional[str] = None,
|
||||
logger_level: int = logging.INFO,
|
||||
) -> FullSyncPipeline:
|
||||
cfg = PipelineRunConfig(
|
||||
node_types=node_types,
|
||||
target_project_ids=target_project_ids or [],
|
||||
local_datasource=ApiDataSourceConfig(
|
||||
type="api",
|
||||
api_base_url=api_base_url,
|
||||
api_uid=api_uid,
|
||||
api_secret=api_secret,
|
||||
api_debug=api_debug,
|
||||
poll_max_retries=poll_max_retries,
|
||||
poll_interval=poll_interval,
|
||||
api_rate_limit_max_requests=api_rate_limit_max_requests,
|
||||
api_rate_limit_window_seconds=api_rate_limit_window_seconds,
|
||||
target_project_ids=target_project_ids or [],
|
||||
handler_configs=handler_configs or {},
|
||||
),
|
||||
remote_datasource=JsonlDataSourceConfig(type="jsonl", jsonl_dir=remote_jsonl_dir, read_only=False),
|
||||
strategies=[],
|
||||
persist=PersistConfig(
|
||||
db_path=persistence_db,
|
||||
enable=enable_persistence,
|
||||
wipe_on_start=wipe_persistence_on_start,
|
||||
),
|
||||
logging=LoggingConfig(
|
||||
initialize=initialize_logger,
|
||||
file_path=logger_file_path,
|
||||
level=logger_level,
|
||||
),
|
||||
)
|
||||
if strategy_overrides:
|
||||
cfg = apply_config_overrides(cfg, {"strategies": strategy_overrides})
|
||||
return await create_pipeline_from_config(cfg)
|
||||
|
||||
|
||||
async def create_jsonl_to_jsonl_pipeline(
|
||||
*,
|
||||
local_jsonl_dir: str,
|
||||
remote_jsonl_dir: str,
|
||||
persistence_db: str,
|
||||
node_types: List[str],
|
||||
target_project_ids: Optional[List[str]] = None,
|
||||
enable_persistence: bool = True,
|
||||
wipe_persistence_on_start: bool = False,
|
||||
strategy_overrides: Optional[Dict[str, Dict[str, Any]]] = None,
|
||||
initialize_logger: bool = True,
|
||||
logger_file_path: Optional[str] = None,
|
||||
logger_level: int = logging.INFO,
|
||||
) -> FullSyncPipeline:
|
||||
cfg = PipelineRunConfig(
|
||||
node_types=node_types,
|
||||
target_project_ids=target_project_ids or [],
|
||||
local_datasource=JsonlDataSourceConfig(type="jsonl", jsonl_dir=local_jsonl_dir, read_only=False),
|
||||
remote_datasource=JsonlDataSourceConfig(type="jsonl", jsonl_dir=remote_jsonl_dir, read_only=False),
|
||||
strategies=[],
|
||||
persist=PersistConfig(
|
||||
db_path=persistence_db,
|
||||
enable=enable_persistence,
|
||||
wipe_on_start=wipe_persistence_on_start,
|
||||
),
|
||||
logging=LoggingConfig(
|
||||
initialize=initialize_logger,
|
||||
file_path=logger_file_path,
|
||||
level=logger_level,
|
||||
),
|
||||
)
|
||||
if strategy_overrides:
|
||||
cfg = apply_config_overrides(cfg, {"strategies": strategy_overrides})
|
||||
return await create_pipeline_from_config(cfg)
|
||||
@@ -0,0 +1,488 @@
|
||||
"""
|
||||
Full Sync Pipeline - Implements document-specified end-to-end workflow.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional, TYPE_CHECKING
|
||||
from pathlib import Path
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..datasource.datasource import BaseDataSource
|
||||
|
||||
from ..common.collection import DataCollection
|
||||
from ..common.binding import BindingManager
|
||||
from ..common.persistence import PersistenceBackend
|
||||
from ..sync_system.strategy import BaseSyncStrategy
|
||||
from ..sync_system.config import OrphanAction, UpdateDirection
|
||||
from ..engine import StateMachineConfig, StateMachineRuntime
|
||||
from ..engine import e41_post_create_ready
|
||||
from ..sync_system.strategy_ops import finalize_peer_creating_sources
|
||||
from ..sync_system.strategy_ops.post_check_ops import evaluate_consistency_for_node_type
|
||||
from .summary_report import print_pipeline_summary
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FullSyncPipeline:
|
||||
"""
|
||||
Full pipeline phases:
|
||||
1) bootstrap
|
||||
2) bind
|
||||
3) create
|
||||
4) update
|
||||
5) post-check
|
||||
6) persistence
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
local_datasource: "BaseDataSource",
|
||||
remote_datasource: "BaseDataSource",
|
||||
strategies: List[BaseSyncStrategy],
|
||||
persistence: PersistenceBackend,
|
||||
local_collection: DataCollection,
|
||||
remote_collection: DataCollection,
|
||||
binding_manager: BindingManager,
|
||||
node_types: Optional[List[str]] = None,
|
||||
load_order: Optional[List[str]] = None,
|
||||
sync_order: Optional[List[str]] = None,
|
||||
enable_persistence: bool = True,
|
||||
# 项目过滤
|
||||
target_project_ids: Optional[List[str]] = None,
|
||||
local_target_project_ids: Optional[List[str]] = None,
|
||||
remote_target_project_ids: Optional[List[str]] = None,
|
||||
):
|
||||
# 验证必需参数（由工厂函数创建）
|
||||
if strategies is None:
|
||||
raise ValueError("strategies is required (use factory function to create pipeline)")
|
||||
if local_collection is None or remote_collection is None or binding_manager is None:
|
||||
raise ValueError("collections and binding_manager are required (use factory function to create pipeline)")
|
||||
|
||||
self.local_datasource = local_datasource
|
||||
self.remote_datasource = remote_datasource
|
||||
self.strategies = strategies
|
||||
self.node_types = node_types or [s.node_type for s in self.strategies]
|
||||
self.load_order = load_order or self.node_types
|
||||
self.sync_order = sync_order or self.node_types
|
||||
self.persistence = persistence
|
||||
self.enable_persistence = enable_persistence
|
||||
self.target_project_ids = target_project_ids or []
|
||||
self.local_target_project_ids = (
|
||||
list(local_target_project_ids)
|
||||
if local_target_project_ids is not None
|
||||
else list(self.target_project_ids)
|
||||
)
|
||||
self.remote_target_project_ids = (
|
||||
list(remote_target_project_ids)
|
||||
if remote_target_project_ids is not None
|
||||
else list(self.target_project_ids)
|
||||
)
|
||||
|
||||
self.local_collection = local_collection
|
||||
self.remote_collection = remote_collection
|
||||
self.binding_manager = binding_manager
|
||||
|
||||
# 统一状态机 runtime 注入（唯一入口）
|
||||
runtime = self._resolve_pipeline_runtime()
|
||||
self.sm_runtime = runtime
|
||||
self.local_collection.set_state_machine_runtime(runtime)
|
||||
self.remote_collection.set_state_machine_runtime(runtime)
|
||||
self.local_datasource.set_state_machine_runtime(runtime)
|
||||
self.remote_datasource.set_state_machine_runtime(runtime)
|
||||
for strategy in self.strategies:
|
||||
strategy.sm_runtime = runtime
|
||||
|
||||
self.stats: Dict[str, Any] = {
|
||||
"loaded": 0,
|
||||
"synced": 0,
|
||||
"failed": 0,
|
||||
"post_check_passed": True,
|
||||
}
|
||||
self._consistency_by_type: Dict[str, Dict[str, Any]] = {}
|
||||
self._logger = logger
|
||||
self._closed = False
|
||||
|
||||
def _resolve_pipeline_runtime(self) -> StateMachineRuntime:
|
||||
if self.strategies:
|
||||
runtime = self.strategies[0].sm_runtime
|
||||
if runtime is not None:
|
||||
return runtime
|
||||
|
||||
cfg_path = Path(__file__).resolve().parents[1] / "config" / "node_state_machine.yaml"
|
||||
return StateMachineRuntime(StateMachineConfig.load(cfg_path))
|
||||
|
||||
async def run(self) -> Dict[str, Any]:
|
||||
try:
|
||||
await self.phase_bootstrap()
|
||||
self._logger.info("✅ Phase bootstrap completed")
|
||||
|
||||
await self.phase_sync_by_node_type()
|
||||
self._logger.info("✅ Phase sync-by-node-type completed")
|
||||
|
||||
post_ok = await self.phase_post_check()
|
||||
self.stats["post_check_passed"] = post_ok
|
||||
self._logger.info("✅ Phase post-check completed")
|
||||
|
||||
await self.phase_persistence()
|
||||
self._logger.info("✅ Phase persistence completed")
|
||||
return self.stats
|
||||
finally:
|
||||
await self.close()
|
||||
|
||||
async def phase_sync_by_node_type(self) -> None:
|
||||
strategy_map = {s.node_type: s for s in self.strategies}
|
||||
for node_type in self.sync_order:
|
||||
strategy = strategy_map.get(node_type)
|
||||
if strategy is None:
|
||||
continue
|
||||
try:
|
||||
section_width = 96
|
||||
self._logger.info("\n" + "-" * section_width)
|
||||
self._logger.info(f"🧩 Strategy start: {node_type}")
|
||||
self._logger.info("-" * section_width)
|
||||
|
||||
await strategy.bind()
|
||||
self._logger.info(f"✅ Strategy bind: {node_type}")
|
||||
|
||||
if strategy.skip_sync:
|
||||
self._logger.info(f"⏭️ Skipping create/update for {node_type} (skip_sync=True)")
|
||||
continue
|
||||
|
||||
created = await strategy.create()
|
||||
self._logger.info(f"✅ Strategy create: {node_type} (created={len(created)})")
|
||||
create_written = await self.commit_creates(node_type)
|
||||
await self.normalize_create_success_to_update_entry(node_type)
|
||||
if create_written:
|
||||
await self._reload_node_type(node_type, reason="create wrote data")
|
||||
|
||||
updated = await strategy.update()
|
||||
self._logger.info(f"✅ Strategy update: {node_type} (updated={len(updated)})")
|
||||
update_written = await self.commit_updates(node_type)
|
||||
if update_written:
|
||||
await self._reload_node_type(node_type, reason="update wrote data")
|
||||
|
||||
self._logger.info(f"✅ Strategy done: {node_type}")
|
||||
except Exception as exc:
|
||||
self.stats["failed"] += 1
|
||||
self._logger.error(f"❌ Strategy failed but pipeline continues: {node_type} | {type(exc).__name__}: {exc}")
|
||||
continue
|
||||
|
||||
@staticmethod
|
||||
def _is_noop_strategy(strategy: BaseSyncStrategy[Any]) -> bool:
|
||||
cfg = strategy.config
|
||||
return (
|
||||
cfg.local_orphan_action == OrphanAction.NONE
|
||||
and cfg.remote_orphan_action == OrphanAction.NONE
|
||||
and cfg.update_direction == UpdateDirection.NONE
|
||||
)
|
||||
|
||||
async def phase_bootstrap(self) -> None:
|
||||
await self._load_persistence_state()
|
||||
self._logger.info("✅ Bootstrap: persistence loaded")
|
||||
|
||||
await self._load_from_datasource()
|
||||
self._logger.info("✅ Bootstrap: datasource fetched")
|
||||
|
||||
if self.local_target_project_ids or self.remote_target_project_ids:
|
||||
local_deleted = await self.local_collection.filter_by_project_ids(self.local_target_project_ids)
|
||||
remote_deleted = await self.remote_collection.filter_by_project_ids(self.remote_target_project_ids)
|
||||
if local_deleted > 0 or remote_deleted > 0:
|
||||
self._logger.info(f"🔍 Project filtering: deleted {local_deleted} local, {remote_deleted} remote nodes")
|
||||
|
||||
async def phase_bind(self) -> None:
|
||||
strategy_map = {s.node_type: s for s in self.strategies}
|
||||
for node_type in self.sync_order:
|
||||
strategy = strategy_map.get(node_type)
|
||||
if strategy is None:
|
||||
continue
|
||||
try:
|
||||
await strategy.bind()
|
||||
except Exception as exc:
|
||||
self.stats["failed"] += 1
|
||||
self._logger.error(f"❌ Strategy bind failed but pipeline continues: {node_type} | {type(exc).__name__}: {exc}")
|
||||
continue
|
||||
|
||||
async def phase_create(self) -> None:
|
||||
strategy_map = {s.node_type: s for s in self.strategies}
|
||||
for node_type in self.sync_order:
|
||||
strategy = strategy_map.get(node_type)
|
||||
if strategy is None:
|
||||
continue
|
||||
if strategy.skip_sync:
|
||||
self._logger.info(f"⏭️ Skipping create for {node_type} (skip_sync=True)")
|
||||
continue
|
||||
try:
|
||||
created = await strategy.create()
|
||||
self._logger.info(f"✅ Strategy create: {node_type} (created={len(created)})")
|
||||
create_written = await self.commit_creates(node_type)
|
||||
await self.normalize_create_success_to_update_entry(node_type)
|
||||
if create_written:
|
||||
await self._reload_node_type(node_type, reason="create wrote data")
|
||||
except Exception as exc:
|
||||
self.stats["failed"] += 1
|
||||
self._logger.error(f"❌ Strategy create failed but pipeline continues: {node_type} | {type(exc).__name__}: {exc}")
|
||||
continue
|
||||
|
||||
async def phase_update(self) -> None:
|
||||
strategy_map = {s.node_type: s for s in self.strategies}
|
||||
for node_type in self.sync_order:
|
||||
strategy = strategy_map.get(node_type)
|
||||
if strategy is None:
|
||||
continue
|
||||
if strategy.skip_sync:
|
||||
self._logger.info(f"⏭️ Skipping update for {node_type} (skip_sync=True)")
|
||||
continue
|
||||
try:
|
||||
updated = await strategy.update()
|
||||
self._logger.info(f"✅ Strategy update: {node_type} (updated={len(updated)})")
|
||||
update_written = await self.commit_updates(node_type)
|
||||
if update_written:
|
||||
await self._reload_node_type(node_type, reason="update wrote data")
|
||||
await self._evaluate_consistency_for_node_type(node_type)
|
||||
except Exception as exc:
|
||||
self.stats["failed"] += 1
|
||||
self._logger.error(f"❌ Strategy update failed but pipeline continues: {node_type} | {type(exc).__name__}: {exc}")
|
||||
continue
|
||||
|
||||
async def close(self) -> None:
|
||||
"""关闭所有资源（DataSource、Persistence）"""
|
||||
if self._closed:
|
||||
return # 已经关闭过，避免重复关闭
|
||||
self._closed = True
|
||||
|
||||
async def _safe_close(name: str, close_coro, timeout: float = 3.0) -> None:
|
||||
try:
|
||||
await asyncio.wait_for(close_coro, timeout=timeout)
|
||||
except asyncio.TimeoutError:
|
||||
self._logger.warning("⚠️ close timeout: %s (>%ss)", name, timeout)
|
||||
except Exception as exc:
|
||||
self._logger.warning("⚠️ close failed: %s (%s)", name, exc)
|
||||
|
||||
await _safe_close("remote_datasource", self.remote_datasource.close())
|
||||
await _safe_close("local_datasource", self.local_datasource.close())
|
||||
await _safe_close("persistence", self.persistence.close())
|
||||
|
||||
async def _load_persistence_state(self) -> None:
|
||||
"""加载持久化数据并清理僵尸节点"""
|
||||
if not self.enable_persistence:
|
||||
return
|
||||
|
||||
# 加载持久化数据
|
||||
await self.local_collection.load_from_persistence()
|
||||
await self.remote_collection.load_from_persistence()
|
||||
for node_type in self.node_types:
|
||||
await self.binding_manager.load_from_persistence(node_type)
|
||||
self._logger.info("🔄 Persistence loaded (state preserved; pending E01 normalization)")
|
||||
|
||||
# 加载后 reset 清理：清理 CREATE 失败的僵尸绑定
|
||||
if not self.strategies:
|
||||
raise RuntimeError("no strategies configured: cannot run bootstrap event E01")
|
||||
total_cleaned = await BaseSyncStrategy.run_reset(
|
||||
node_types=self.node_types,
|
||||
local_collection=self.local_collection,
|
||||
remote_collection=self.remote_collection,
|
||||
binding_manager=self.binding_manager,
|
||||
runtime=self.sm_runtime,
|
||||
)
|
||||
if total_cleaned > 0:
|
||||
self._logger.info(f"🧹 Reset cleanup: {total_cleaned} CREATE-failed zombies cleaned")
|
||||
self._logger.info("🔄 Post-load reset cleanup completed")
|
||||
|
||||
async def _load_from_datasource(self) -> None:
|
||||
"""从数据源加载数据"""
|
||||
self.local_datasource.set_collection(self.local_collection)
|
||||
await self.local_datasource.load_all(order=self.load_order)
|
||||
|
||||
self.remote_datasource.set_collection(self.remote_collection)
|
||||
await self.remote_datasource.load_all(order=self.load_order)
|
||||
|
||||
for node_type in self.node_types:
|
||||
self.stats["loaded"] += len(self.local_collection.filter(node_type=node_type))
|
||||
self.stats["loaded"] += len(self.remote_collection.filter(node_type=node_type))
|
||||
def _get_action_success_count(self, datasource: "BaseDataSource", node_type: str, action_key: str) -> int:
|
||||
summary = datasource.get_action_summary().get(node_type, {})
|
||||
action_summary = summary.get(action_key, {}) if isinstance(summary, dict) else {}
|
||||
return int(action_summary.get("success", 0)) if isinstance(action_summary, dict) else 0
|
||||
|
||||
async def _reload_node_type(self, node_type: str, *, reason: str) -> None:
|
||||
self._logger.info(f"🔄 Reload after write: node_type={node_type}, reason={reason}")
|
||||
await self.local_datasource.load_all(order=[node_type])
|
||||
await self.remote_datasource.load_all(order=[node_type])
|
||||
|
||||
async def _evaluate_consistency_for_node_type(self, node_type: str) -> Dict[str, Any]:
|
||||
strategy = next((item for item in self.strategies if item.node_type == node_type), None)
|
||||
depend_fields = dict(strategy.config.depend_fields) if strategy and strategy.config.depend_fields else {}
|
||||
compare_to_remote = True
|
||||
if strategy and strategy.config.update_direction == UpdateDirection.PULL:
|
||||
compare_to_remote = False
|
||||
ignore_list_item_fields = dict(strategy.config.post_check_ignore_list_item_fields) if strategy else {}
|
||||
|
||||
# 从远端 handler 的 update_schemas 提取只读字段过滤白名单
|
||||
post_check_fields: List[str] | None = None
|
||||
try:
|
||||
remote_handler = self.remote_datasource.get_handler(node_type)
|
||||
fields = remote_handler.get_update_fields()
|
||||
if fields:
|
||||
post_check_fields = fields
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
result = await evaluate_consistency_for_node_type(
|
||||
node_type=node_type,
|
||||
local_collection=self.local_collection,
|
||||
remote_collection=self.remote_collection,
|
||||
binding_manager=self.binding_manager,
|
||||
logger=self._logger,
|
||||
depend_fields=depend_fields,
|
||||
compare_to_remote=compare_to_remote,
|
||||
ignore_list_item_fields=ignore_list_item_fields,
|
||||
post_check_fields=post_check_fields,
|
||||
)
|
||||
self._consistency_by_type[node_type] = result
|
||||
return result
|
||||
|
||||
async def commit_creates(self, node_type: str) -> bool:
|
||||
"""提交 CREATE 操作（包含异步轮询）"""
|
||||
# 执行同步（sync_all 内部会根据 action 筛选节点）
|
||||
await self.local_datasource.sync_all(
|
||||
order=[node_type],
|
||||
poll_async_tasks=True
|
||||
)
|
||||
|
||||
await self.remote_datasource.sync_all(
|
||||
order=[node_type],
|
||||
poll_async_tasks=True
|
||||
)
|
||||
|
||||
await self._finalize_peer_creating_sources(node_type)
|
||||
|
||||
# 更新统计
|
||||
local_nodes = self.local_collection.filter(node_type=node_type)
|
||||
remote_nodes = self.remote_collection.filter(node_type=node_type)
|
||||
self.stats["synced"] += len([n for n in local_nodes if n.action.value == "CREATE"])
|
||||
self.stats["synced"] += len([n for n in remote_nodes if n.action.value == "CREATE"])
|
||||
self.stats["failed"] += len([n for n in local_nodes if n.status.value == "FAILED"])
|
||||
self.stats["failed"] += len([n for n in remote_nodes if n.status.value == "FAILED"])
|
||||
|
||||
local_success = self._get_action_success_count(self.local_datasource, node_type, "create")
|
||||
remote_success = self._get_action_success_count(self.remote_datasource, node_type, "create")
|
||||
wrote = (local_success + remote_success) > 0
|
||||
return wrote
|
||||
|
||||
async def _finalize_peer_creating_sources(self, node_type: str) -> None:
|
||||
finalized_success, finalized_failed = await finalize_peer_creating_sources(
|
||||
node_type=node_type,
|
||||
local_collection=self.local_collection,
|
||||
remote_collection=self.remote_collection,
|
||||
binding_manager=self.binding_manager,
|
||||
runtime=self.sm_runtime,
|
||||
)
|
||||
|
||||
if finalized_success or finalized_failed:
|
||||
self._logger.info(
|
||||
f"🔁 Peer-creating finalize: node_type={node_type}, success={finalized_success}, failed={finalized_failed}"
|
||||
)
|
||||
|
||||
async def normalize_create_success_to_update_entry(self, node_type: str) -> None:
|
||||
"""将 CREATE 成功节点从 S11C 归并到 S01，作为 update 起点。"""
|
||||
normalized = 0
|
||||
for collection in (self.local_collection, self.remote_collection):
|
||||
candidates = collection.filter_by_state_ids(
|
||||
node_type=node_type,
|
||||
state_ids=["S11C"],
|
||||
)
|
||||
for node in candidates:
|
||||
if node.action.value != "CREATE":
|
||||
continue
|
||||
decision = e41_post_create_ready(self.sm_runtime, node=node, execute_action="CREATE")
|
||||
if decision.to_state != "S01":
|
||||
raise RuntimeError(
|
||||
f"[{node_type}] E41 expected S01, got {decision.to_state} for node={node.node_id}"
|
||||
)
|
||||
normalized += 1
|
||||
|
||||
if normalized > 0:
|
||||
self._logger.info(f"🔁 Post-create normalization: {normalized} nodes moved S11C->S01 for {node_type}")
|
||||
|
||||
async def commit_updates(self, node_type: str) -> bool:
|
||||
"""提交 UPDATE 操作（包含异步轮询）"""
|
||||
# 执行同步（sync_all 内部会根据 action 筛选节点）
|
||||
await self.local_datasource.sync_all(
|
||||
order=[node_type],
|
||||
poll_async_tasks=True
|
||||
)
|
||||
|
||||
await self.remote_datasource.sync_all(
|
||||
order=[node_type],
|
||||
poll_async_tasks=True
|
||||
)
|
||||
|
||||
# 更新统计
|
||||
local_nodes = self.local_collection.filter(node_type=node_type)
|
||||
remote_nodes = self.remote_collection.filter(node_type=node_type)
|
||||
self.stats["synced"] += len([n for n in local_nodes if n.action.value == "UPDATE"])
|
||||
self.stats["synced"] += len([n for n in remote_nodes if n.action.value == "UPDATE"])
|
||||
self.stats["failed"] += len([n for n in local_nodes if n.status.value == "FAILED"])
|
||||
self.stats["failed"] += len([n for n in remote_nodes if n.status.value == "FAILED"])
|
||||
|
||||
local_success = self._get_action_success_count(self.local_datasource, node_type, "update")
|
||||
remote_success = self._get_action_success_count(self.remote_datasource, node_type, "update")
|
||||
wrote = (local_success + remote_success) > 0
|
||||
return wrote
|
||||
|
||||
async def phase_post_check(self) -> bool:
|
||||
"""同步结果报告。
|
||||
|
||||
所有类型均已同步完毕后，统一做一次全量 reload（触发后端派生字段计算），
|
||||
再对所有类型做一致性评估，确保 post-check 结果反映最终状态。
|
||||
"""
|
||||
self._logger.info("🔄 Post-check: reloading all node types for final consistency evaluation...")
|
||||
for node_type in self.node_types:
|
||||
await self._reload_node_type(node_type, reason="post-check final reload")
|
||||
for node_type in self.node_types:
|
||||
await self._evaluate_consistency_for_node_type(node_type)
|
||||
|
||||
self.stats["consistency"] = {
|
||||
node_type: {
|
||||
"ok": item.get("ok", True),
|
||||
"checked_pairs": item.get("checked_pairs", 0),
|
||||
"mismatch_count": item.get("mismatch_count", 0),
|
||||
}
|
||||
for node_type, item in self._consistency_by_type.items()
|
||||
}
|
||||
await self.print_summary()
|
||||
return True # 总是返回 True，不阻断流程
|
||||
|
||||
async def phase_persistence(self) -> None:
|
||||
"""持久化所有数据。"""
|
||||
if not self.enable_persistence:
|
||||
self._logger.info("⏭️ Persistence disabled, skipping...")
|
||||
return
|
||||
|
||||
self._logger.info("🔍 Persisting local collection...")
|
||||
await self.local_collection.persist()
|
||||
self._logger.info("🔍 Persisting remote collection...")
|
||||
await self.remote_collection.persist()
|
||||
self._logger.info("🔍 Persisting binding manager...")
|
||||
await self.binding_manager.persist()
|
||||
self._logger.info("✅ All data persisted successfully")
|
||||
|
||||
# ========== Summary & Reporting ==========
|
||||
|
||||
async def print_summary(self) -> None:
|
||||
"""打印同步结果摘要"""
|
||||
await print_pipeline_summary(
|
||||
logger=self._logger,
|
||||
node_types=self.node_types,
|
||||
binding_manager=self.binding_manager,
|
||||
local_datasource=self.local_datasource,
|
||||
remote_datasource=self.remote_datasource,
|
||||
local_collection=self.local_collection,
|
||||
remote_collection=self.remote_collection,
|
||||
consistency_by_type=self._consistency_by_type,
|
||||
stats=self.stats,
|
||||
)
|
||||
@@ -0,0 +1,65 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class _NodeLogFilter(logging.Filter):
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
message = record.getMessage()
|
||||
if "node=" in message:
|
||||
return False
|
||||
if "节点 " in message and "状态" in message:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def init_pipeline_logger(
|
||||
*,
|
||||
log_file_path: Optional[str] = None,
|
||||
level: int = logging.INFO,
|
||||
mirror_console: bool = True,
|
||||
replace_handlers: bool = True,
|
||||
suppress_node_logs: bool = True,
|
||||
) -> logging.Logger:
|
||||
logger = logging.getLogger("sync_state_machine")
|
||||
logger.setLevel(level)
|
||||
logger.propagate = False
|
||||
|
||||
if replace_handlers:
|
||||
clear_pipeline_logger_handlers(logger)
|
||||
|
||||
formatter = logging.Formatter("%(message)s")
|
||||
node_filter = _NodeLogFilter() if suppress_node_logs else None
|
||||
|
||||
if mirror_console:
|
||||
console_handler = logging.StreamHandler()
|
||||
console_handler.setLevel(level)
|
||||
console_handler.setFormatter(formatter)
|
||||
if node_filter is not None:
|
||||
console_handler.addFilter(node_filter)
|
||||
logger.addHandler(console_handler)
|
||||
|
||||
if log_file_path:
|
||||
path = Path(log_file_path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
file_handler = logging.FileHandler(path, mode="w", encoding="utf-8")
|
||||
file_handler.setLevel(level)
|
||||
file_handler.setFormatter(formatter)
|
||||
if node_filter is not None:
|
||||
file_handler.addFilter(node_filter)
|
||||
logger.addHandler(file_handler)
|
||||
|
||||
return logger
|
||||
|
||||
|
||||
def clear_pipeline_logger_handlers(logger: Optional[logging.Logger] = None) -> None:
|
||||
target = logger or logging.getLogger("sync_state_machine")
|
||||
handlers = list(target.handlers)
|
||||
for handler in handlers:
|
||||
target.removeHandler(handler)
|
||||
try:
|
||||
handler.close()
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,150 @@
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
async def print_pipeline_summary(*, logger, node_types, binding_manager, local_datasource, remote_datasource, local_collection, remote_collection, consistency_by_type, stats) -> None:
|
||||
binding_counts = {}
|
||||
for node_type in node_types:
|
||||
records = await binding_manager.get_all_records(node_type)
|
||||
binding_counts[node_type] = len(records)
|
||||
|
||||
def format_action_sft(summary: dict, key: str) -> str:
|
||||
action = summary.get(key, {})
|
||||
success = int(action.get("success", 0))
|
||||
failed = int(action.get("failed", 0)) + int(action.get("skipped", 0))
|
||||
total = int(action.get("total", 0))
|
||||
return f"{success}/{failed}/{total}"
|
||||
|
||||
def format_consistency_sft(node_type: str) -> str:
|
||||
consistency = consistency_by_type.get(node_type, {})
|
||||
checked = int(consistency.get("checked_pairs", 0))
|
||||
mismatch = int(consistency.get("mismatch_count", 0))
|
||||
matched = max(0, checked - mismatch)
|
||||
return f"{matched}/{mismatch}/{checked}"
|
||||
|
||||
def _clip(text: str, width: int) -> str:
|
||||
if len(text) <= width:
|
||||
return text
|
||||
if width <= 1:
|
||||
return text[:width]
|
||||
return text[: width - 1] + "…"
|
||||
|
||||
def print_collection_summary(title: str, datasource, collection) -> None:
|
||||
action_summary = datasource.get_action_summary()
|
||||
line_width = 133
|
||||
|
||||
node_type_w = 28
|
||||
bound_w = 16
|
||||
action_w = 16
|
||||
consistency_w = 18
|
||||
|
||||
logger.info(f"\n{'='*line_width}")
|
||||
logger.info(f"📊 {title} Summary")
|
||||
logger.info(f"{'='*line_width}")
|
||||
logger.info("说明: Create/Update/Delete = 成功/失败/总数 (失败=FAILED+SKIPPED)")
|
||||
logger.info("说明: Consistent = 一致对数/不一致对数/已检查绑定对数")
|
||||
logger.info(
|
||||
f"{'Node Type':<{node_type_w}} {'Bound/Rec/Load':<{bound_w}} {'Create(S/F/T)':<{action_w}} {'Update(S/F/T)':<{action_w}} {'Delete(S/F/T)':<{action_w}} {'Consistent(S/F/T)':<{consistency_w}}"
|
||||
)
|
||||
logger.info(f"{'-'*line_width}")
|
||||
|
||||
total_bound_normal = 0
|
||||
total_bindings = 0
|
||||
total_loaded = 0
|
||||
total_create = {"total": 0, "success": 0, "failed_effective": 0}
|
||||
total_update = {"total": 0, "success": 0, "failed_effective": 0}
|
||||
total_delete = {"total": 0, "success": 0, "failed_effective": 0}
|
||||
total_consistency = {"checked": 0, "mismatch": 0}
|
||||
|
||||
for node_type in node_types:
|
||||
summary = action_summary.get(node_type, {})
|
||||
bindings = binding_counts.get(node_type, 0)
|
||||
nodes = collection.filter(node_type=node_type)
|
||||
loaded_count = len(nodes)
|
||||
bound_normal_count = len([n for n in nodes if n.binding_status.value == "normal"])
|
||||
|
||||
create_str = format_action_sft(summary, "create")
|
||||
update_str = format_action_sft(summary, "update")
|
||||
delete_str = format_action_sft(summary, "delete")
|
||||
consistency_str = format_consistency_sft(node_type)
|
||||
|
||||
bound_str = f"{bound_normal_count}/{bindings}/{loaded_count}"
|
||||
logger.info(
|
||||
f"{_clip(node_type, node_type_w):<{node_type_w}} "
|
||||
f"{_clip(bound_str, bound_w):<{bound_w}} "
|
||||
f"{_clip(create_str, action_w):<{action_w}} "
|
||||
f"{_clip(update_str, action_w):<{action_w}} "
|
||||
f"{_clip(delete_str, action_w):<{action_w}} "
|
||||
f"{_clip(consistency_str, consistency_w):<{consistency_w}}"
|
||||
)
|
||||
|
||||
total_bound_normal += bound_normal_count
|
||||
total_bindings += bindings
|
||||
total_loaded += loaded_count
|
||||
for bucket, key in (
|
||||
(total_create, "create"),
|
||||
(total_update, "update"),
|
||||
(total_delete, "delete"),
|
||||
):
|
||||
action = summary.get(key, {})
|
||||
bucket["total"] += action.get("total", 0)
|
||||
bucket["success"] += action.get("success", 0)
|
||||
bucket["failed_effective"] += action.get("failed", 0) + action.get("skipped", 0)
|
||||
|
||||
consistency = consistency_by_type.get(node_type, {})
|
||||
total_consistency["checked"] += int(consistency.get("checked_pairs", 0))
|
||||
total_consistency["mismatch"] += int(consistency.get("mismatch_count", 0))
|
||||
|
||||
logger.info(f"{'-'*line_width}")
|
||||
total_consistent = max(0, total_consistency["checked"] - total_consistency["mismatch"])
|
||||
total_create_sft = f"{total_create['success']}/{total_create['failed_effective']}/{total_create['total']}"
|
||||
total_update_sft = f"{total_update['success']}/{total_update['failed_effective']}/{total_update['total']}"
|
||||
total_delete_sft = f"{total_delete['success']}/{total_delete['failed_effective']}/{total_delete['total']}"
|
||||
total_consistency_sft = f"{total_consistent}/{total_consistency['mismatch']}/{total_consistency['checked']}"
|
||||
total_bound_str = f"{total_bound_normal}/{total_bindings}/{total_loaded}"
|
||||
logger.info(
|
||||
f"{'TOTAL':<{node_type_w}} "
|
||||
f"{_clip(total_bound_str, bound_w):<{bound_w}} "
|
||||
f"{_clip(total_create_sft, action_w):<{action_w}} "
|
||||
f"{_clip(total_update_sft, action_w):<{action_w}} "
|
||||
f"{_clip(total_delete_sft, action_w):<{action_w}} "
|
||||
f"{_clip(total_consistency_sft, consistency_w):<{consistency_w}}"
|
||||
)
|
||||
|
||||
print_collection_summary("Local", local_datasource, local_collection)
|
||||
print_collection_summary("Remote", remote_datasource, remote_collection)
|
||||
|
||||
loaded = int(stats.get("loaded", 0))
|
||||
synced = int(stats.get("synced", 0))
|
||||
failed = int(stats.get("failed", 0))
|
||||
post_check_passed = bool(stats.get("post_check_passed", True))
|
||||
|
||||
mismatch_node_types = [
|
||||
node_type
|
||||
for node_type, item in consistency_by_type.items()
|
||||
if int(item.get("mismatch_count", 0)) > 0
|
||||
]
|
||||
|
||||
logger.info("\n" + "=" * 96)
|
||||
logger.info(
|
||||
f"📈 Pipeline Stats: loaded={loaded}, synced={synced}, failed={failed}, post_check_passed={post_check_passed}, mismatch_types={len(mismatch_node_types)}"
|
||||
)
|
||||
if mismatch_node_types:
|
||||
logger.info(f"🔎 Mismatch Node Types: {', '.join(mismatch_node_types)}")
|
||||
logger.info("=" * 96)
|
||||
|
||||
sample_per_type = 3
|
||||
has_mismatch_samples = any(int(item.get("mismatch_count", 0)) > 0 for item in consistency_by_type.values())
|
||||
if has_mismatch_samples:
|
||||
logger.info("\n" + "=" * 96)
|
||||
logger.info("🧪 Inconsistency Samples (per type)")
|
||||
logger.info("=" * 96)
|
||||
for node_type in node_types:
|
||||
item = consistency_by_type.get(node_type, {})
|
||||
mismatch_count = int(item.get("mismatch_count", 0))
|
||||
if mismatch_count <= 0:
|
||||
continue
|
||||
checked_pairs = int(item.get("checked_pairs", 0))
|
||||
samples = item.get("samples", []) or []
|
||||
logger.info(f"- {node_type}: mismatches={mismatch_count}, checked_pairs={checked_pairs}")
|
||||
for sample in samples[:sample_per_type]:
|
||||
logger.info(f" • {sample}")
|
||||
Reference in New Issue
Block a user