多pipeline同步支持

sync_state_machine/common/collection.py

@@ -1,4 +1,5 @@
 import uuid
+import copy
 from enum import Enum
 from typing import Dict, List, Optional, Any, Set, Callable, Literal
 from .sync_node import SyncNode
@@ -22,11 +23,13 @@ class DataCollection:
     def __init__(
         self,
         collection_id: str,
+        scope: str = "global",
         persistence: Optional[PersistenceBackend] = None,
         auto_persist: bool = False,
         sm_runtime: Optional[Any] = None,
     ):
         self.collection_id = collection_id
+        self.scope = scope
         self.persistence = persistence
         self.auto_persist = auto_persist
         self._sm_runtime = sm_runtime
@@ -67,7 +70,7 @@ class DataCollection:
         self._nodes[node.node_id] = node
         self._deleted_node_ids.discard(node.node_id)
         if self.persistence and self.auto_persist:
-            await self.persistence.save_node(self.collection_id, node)
+            await self.persistence.save_node(self.collection_id, self.scope, node)
 
     async def update(self, node: SyncNode):
         old_node = self._nodes.get(node.node_id)
@@ -85,7 +88,7 @@ class DataCollection:
         self._nodes[node.node_id] = node
         self._deleted_node_ids.discard(node.node_id)
         if self.persistence and self.auto_persist:
-            await self.persistence.save_node(self.collection_id, node)
+            await self.persistence.save_node(self.collection_id, self.scope, node)
 
     async def delete(self, node_id: str):
         old_node = self._nodes.get(node_id)
@@ -96,7 +99,7 @@ class DataCollection:
             del self._nodes[node_id]
         if self.persistence:
             if self.auto_persist:
-                await self.persistence.delete_node(self.collection_id, node_id)
+                await self.persistence.delete_node(self.collection_id, self.scope, node_id)
             else:
                 self._deleted_node_ids.add(node_id)
 
@@ -370,7 +373,7 @@ class DataCollection:
         self._data_id_to_node_id = {}
         self._deleted_node_ids.clear()
 
-        node_dicts = await self.persistence.load_nodes(self.collection_id)
+        node_dicts = await self.persistence.load_nodes(self.collection_id, self.scope)
         
         from .types import SyncAction, SyncStatus, BindingStatus
         from .registry import DomainRegistry
@@ -491,7 +494,7 @@ class DataCollection:
 
         node_types = {node.node_type for node in self._nodes.values()}
         for node_type in node_types:
-            override_map = await self.persistence.load_bind_data_id_overrides(node_type)
+            override_map = await self.persistence.load_bind_data_id_overrides(node_type, self.scope)
             for node in self.filter(node_type=node_type):
                 bind_data_id = override_map.get(node.node_id)
                 if bind_data_id:
@@ -499,20 +502,169 @@ class DataCollection:
                 else:
                     node.context.pop("bind_data_id", None)
 
-    async def persist(self) -> None:
+    async def load_from_persistence_excluding(self, node_types: Optional[List[str]] = None):
+        """从持久化恢复数据，但跳过指定 node_type。"""
+        if not self.persistence:
+            return
+
+        self._nodes = {}
+        self._data_id_to_node_id = {}
+        self._deleted_node_ids.clear()
+
+        node_dicts = await self.persistence.load_nodes(
+            self.collection_id,
+            self.scope,
+            exclude_node_types=node_types,
+        )
+
+        from .types import SyncAction, SyncStatus, BindingStatus
+        from .registry import DomainRegistry
+
+        for node_dict in node_dicts:
+            node_type = node_dict["node_type"]
+            node_class = DomainRegistry.get_node_class(node_type)
+            assert issubclass(node_class, SyncNode)
+
+            parse_errors: list[str] = []
+
+            raw_action = node_dict.get("action", SyncAction.NONE)
+            if isinstance(raw_action, str):
+                try:
+                    action_value = SyncAction(raw_action)
+                except ValueError:
+                    parse_errors.append(f"invalid action={raw_action}")
+                    action_value = SyncAction.NONE
+            elif isinstance(raw_action, SyncAction):
+                action_value = raw_action
+            else:
+                if raw_action is None:
+                    action_value = SyncAction.NONE
+                else:
+                    parse_errors.append(f"invalid action type={type(raw_action).__name__}")
+                    action_value = SyncAction.NONE
+
+            raw_status = node_dict.get("status", SyncStatus.PENDING)
+            if isinstance(raw_status, str):
+                try:
+                    status_value = SyncStatus(raw_status)
+                except ValueError:
+                    parse_errors.append(f"invalid status={raw_status}")
+                    status_value = SyncStatus.FAILED
+            elif isinstance(raw_status, SyncStatus):
+                status_value = raw_status
+            else:
+                if raw_status is None:
+                    status_value = SyncStatus.PENDING
+                else:
+                    parse_errors.append(f"invalid status type={type(raw_status).__name__}")
+                    status_value = SyncStatus.FAILED
+
+            raw_binding_status = node_dict.get("binding_status", BindingStatus.UNCHECKED)
+            if isinstance(raw_binding_status, str):
+                try:
+                    binding_status_value = BindingStatus(raw_binding_status)
+                except ValueError:
+                    parse_errors.append(f"invalid binding_status={raw_binding_status}")
+                    binding_status_value = BindingStatus.ABNORMAL
+            elif isinstance(raw_binding_status, BindingStatus):
+                binding_status_value = raw_binding_status
+            else:
+                if raw_binding_status is None:
+                    binding_status_value = BindingStatus.UNCHECKED
+                else:
+                    parse_errors.append(
+                        f"invalid binding_status type={type(raw_binding_status).__name__}"
+                    )
+                    binding_status_value = BindingStatus.ABNORMAL
+
+            raw_context = node_dict.get("context", {})
+            context_value = raw_context if isinstance(raw_context, dict) else {}
+            error_value = node_dict.get("error")
+
+            if parse_errors:
+                context_value = dict(context_value)
+                context_value["bootstrap_blocked"] = True
+                context_value["bootstrap_block_reason"] = "invalid_persisted_enum"
+                context_value["bootstrap_block_errors"] = parse_errors
+                binding_status_value = BindingStatus.ABNORMAL
+                action_value = SyncAction.NONE
+                status_value = SyncStatus.FAILED
+                details = "; ".join(parse_errors)
+                error_value = f"LOAD_INVALID_ENUM: {details}"
+
+            node = node_class(
+                node_id=node_dict["node_id"],
+                data_id=node_dict.get("data_id", ""),
+                depend_ids=node_dict.get("depend_ids", []),
+                data=node_dict.get("data"),
+                origin_data=node_dict.get("origin_data"),
+                action=action_value,
+                status=status_value,
+                binding_status=binding_status_value,
+                error=error_value,
+                sync_log=None,
+                context=context_value,
+            )
+            node.context["_loaded_from_persistence"] = True
+            self._nodes[node.node_id] = node
+
+            if node.data_id and node.data_id != "":
+                existing_node_id = self._data_id_to_node_id.get(node.data_id)
+                if existing_node_id is not None and existing_node_id != node.node_id:
+                    raise ValueError(
+                        f"Duplicate data_id detected while loading: {node.data_id} already bound to node_id={existing_node_id}"
+                    )
+                self._data_id_to_node_id[node.data_id] = node.node_id
+
+        loaded_node_types = {node.node_type for node in self._nodes.values()}
+        for node_type in loaded_node_types:
+            override_map = await self.persistence.load_bind_data_id_overrides(node_type, self.scope)
+            for node in self.filter(node_type=node_type):
+                bind_data_id = override_map.get(node.node_id)
+                if bind_data_id:
+                    node.context["bind_data_id"] = bind_data_id
+                else:
+                    node.context.pop("bind_data_id", None)
+
+    async def import_nodes(self, nodes: List[SyncNode]) -> None:
+        """导入现有节点到当前 collection，仅写内存，不触发持久化。"""
+        for source_node in nodes:
+            node = copy.deepcopy(source_node)
+            if node.node_id in self._nodes:
+                continue
+
+            if node.data_id and node.data_id != "":
+                existing_node_id = self._data_id_to_node_id.get(node.data_id)
+                if existing_node_id is not None and existing_node_id != node.node_id:
+                    raise ValueError(
+                        f"Duplicate data_id detected: {node.data_id} already bound to node_id={existing_node_id}"
+                    )
+                self._data_id_to_node_id[node.data_id] = node.node_id
+
+            self._nodes[node.node_id] = node
+            self._deleted_node_ids.discard(node.node_id)
+
+    async def persist(self, *, exclude_node_types: Optional[List[str]] = None) -> None:
         """将当前 collection 全量持久化到后端"""
         if not self.persistence:
             return
 
         # 先落库删除（用于 auto_persist=False 的场景）
         if self._deleted_node_ids:
-            await self.persistence.delete_nodes_bulk(self.collection_id, list(self._deleted_node_ids))
+            await self.persistence.delete_nodes_bulk(self.collection_id, self.scope, list(self._deleted_node_ids))
             self._deleted_node_ids.clear()
 
         if not self._nodes:
             return
 
-        await self.persistence.save_nodes_bulk(self.collection_id, list(self._nodes.values()))
+        excluded = set(node_type for node_type in (exclude_node_types or []) if node_type)
+        nodes_to_persist = [
+            node for node in self._nodes.values() if node.node_type not in excluded
+        ]
+        if not nodes_to_persist:
+            return
+
+        await self.persistence.save_nodes_bulk(self.collection_id, self.scope, nodes_to_persist)
 
     async def filter_by_project_ids(self, data_id_filter: List[str]) -> int:
         """按目标项目 data_id 集合过滤当前 collection，返回删除数量。"""