初步实现多project pipeline

sync_state_machine/pipeline/full_sync_pipeline.py

@@ -15,7 +15,7 @@ if TYPE_CHECKING:
 from ..datasource.jsonl import JsonlDataSource
 from ..common.collection import DataCollection
 from ..common.binding import BindingManager
-from ..logging import get_logger
+from ..logging import get_scope_logger
 from ..common.persistence import PersistenceBackend
 from ..sync_system.config import UpdateDirection
 from ..sync_system.strategy import BaseSyncStrategy
@@ -25,9 +25,7 @@ from ..sync_system.strategy_ops import finalize_peer_creating_sources, run_phase
 from ..sync_system.strategy_ops.bind_ops import refresh_bind_data_id
 from ..sync_system.strategy_ops.post_check_ops import evaluate_consistency_for_node_type
 from .summary_report import print_pipeline_summary
-
-
-logger = get_logger(__name__)
+from .project_scope_runtime import ProjectScopeRuntime
 
 
 class NodeTypeLoadError(RuntimeError):
@@ -72,6 +70,8 @@ class FullSyncPipeline:
         load_order: Optional[List[str]] = None,
         sync_order: Optional[List[str]] = None,
         enable_persistence: bool = True,
+        scope_runtime: Optional[ProjectScopeRuntime] = None,
+        close_resources: bool = True,
     ):
         # 验证必需参数（由工厂函数创建）
         if strategies is None:
@@ -91,6 +91,8 @@ class FullSyncPipeline:
         self.sync_order = sync_order or self.node_types
         self.persistence = persistence
         self.enable_persistence = enable_persistence
+        self.scope_runtime = scope_runtime
+        self.close_resources = close_resources
         
         self.local_collection = local_collection
         self.remote_collection = remote_collection
@@ -113,12 +115,9 @@ class FullSyncPipeline:
             "post_check_passed": True,
         }
         self._consistency_by_type: Dict[str, Dict[str, Any]] = {}
-        self._logger = logger
+        self._logger = get_scope_logger(scope, __name__)
         self._closed = False
         self._loaded_node_types: set[str] = set()
-        self._preloaded_local_nodes: List[Any] = []
-        self._preloaded_remote_nodes: List[Any] = []
-        self._preloaded_bindings: Dict[str, Dict[str, Optional[str]]] = {}
 
     def _resolve_pipeline_runtime(self) -> StateMachineRuntime:
         if self.strategies:
@@ -290,12 +289,17 @@ class FullSyncPipeline:
                 continue
 
     async def phase_bootstrap(self) -> None:
-        await self._load_persistence_state()
+        if self.scope_runtime is not None:
+            await self.scope_runtime.load(
+                node_types=self.node_types,
+                bootstrap_binding_node_types=self.bootstrap_binding_node_types,
+                enable_persistence=self.enable_persistence,
+                ephemeral_node_types=self.ephemeral_node_types,
+            )
+        else:
+            await self._load_persistence_state()
         self._logger.info("✅ Bootstrap: persistence loaded")
 
-        await self._apply_preloaded_state()
-        self._logger.info("✅ Bootstrap: shared preload applied")
-
         self._prepare_datasources()
         self._logger.info("✅ Bootstrap: datasource prepared")
 
@@ -383,6 +387,14 @@ class FullSyncPipeline:
             return  # 已经关闭过，避免重复关闭
         self._closed = True
 
+        if not self.close_resources:
+            return
+
+        if self.scope_runtime is not None:
+            await self.scope_runtime.unload()
+            await self.scope_runtime.close()
+            return
+
         await self._safe_close("remote_datasource", self.remote_datasource.close())
         await self._safe_close("local_datasource", self.local_datasource.close())
         await self._safe_close("persistence", self.persistence.close())
@@ -459,28 +471,6 @@ class FullSyncPipeline:
         if total_cleaned > 0:
             self._logger.info(f"🧹 Reset cleanup: {total_cleaned} bootstrap zombies cleaned")
 
-    async def _apply_preloaded_state(self) -> None:
-        if self._preloaded_local_nodes:
-            await self.local_collection.import_nodes(self._preloaded_local_nodes)
-        if self._preloaded_remote_nodes:
-            await self.remote_collection.import_nodes(self._preloaded_remote_nodes)
-        for node_type, bindings in self._preloaded_bindings.items():
-            self.binding_manager.import_bindings(node_type, bindings)
-
-    def preload_shared_state(
-        self,
-        *,
-        local_nodes: List[Any],
-        remote_nodes: List[Any],
-        bindings_by_type: Dict[str, Dict[str, Optional[str]]],
-    ) -> None:
-        self._preloaded_local_nodes = list(local_nodes)
-        self._preloaded_remote_nodes = list(remote_nodes)
-        self._preloaded_bindings = {
-            node_type: dict(bindings)
-            for node_type, bindings in bindings_by_type.items()
-        }
-
     def _prepare_datasources(self) -> None:
         """为后续按 node_type 加载准备 collection 绑定。"""
         self.local_datasource.set_collection(self.local_collection)
@@ -772,6 +762,18 @@ class FullSyncPipeline:
 
     async def phase_persistence(self) -> None:
         """持久化所有数据。"""
+        if self.scope_runtime is not None:
+            if not self.enable_persistence:
+                self._logger.info("⏭️  Persistence disabled, skipping...")
+                return
+            self._logger.info("🔍 Persisting scope runtime...")
+            await self.scope_runtime.persist(
+                enable_persistence=self.enable_persistence,
+                ephemeral_node_types=self.ephemeral_node_types,
+            )
+            self._logger.info("✅ All data persisted successfully")
+            return
+
         if not self.enable_persistence:
             self._logger.info("⏭️  Persistence disabled, skipping...")
             return