初步实现多project pipeline

2026-04-07 14:45:19 +08:00
parent e3eab5ade1
commit 966f2e1270
11 changed files with 1083 additions and 169 deletions
@@ -4,6 +4,7 @@ from typing import Optional, Dict, Any, List

 from ..logging import get_logger
 from .persistence import PersistenceBackend
+from .persistence_service import ScopedPersistenceView
 from .types import BindingStatus, BindingRecord

 logger = get_logger(__name__)
@@ -14,14 +15,53 @@ class BindingManager:
    BindingManager 负责维护不同 DataCollection 间节点的绑定关系。
    所有查询均在内存中进行，持久化仅作为备份。
    """
-    def __init__(self, persistence: PersistenceBackend, auto_persist: bool = False, scope: str = "global"):
+    def __init__(
+        self,
+        persistence: PersistenceBackend | ScopedPersistenceView,
+        auto_persist: bool = False,
+        scope: str = "global",
+        global_fallback_manager: Optional["BindingManager"] = None,
+    ):
        self.persistence = persistence
        self.auto_persist = auto_persist
        self.scope = scope
+        self._global_fallback_manager = global_fallback_manager
        # { node_type: { local_id: remote_id } }
        self._bindings: Dict[str, Dict[str, Optional[str]]] = {}
        self._deleted: Dict[str, set[str]] = {}

+    def set_global_fallback_manager(self, manager: Optional["BindingManager"]) -> None:
+        self._global_fallback_manager = manager
+
+    async def _persistence_save_binding(self, node_type: str, local_id: str, payload_dict: Dict[str, Any]) -> None:
+        if isinstance(self.persistence, ScopedPersistenceView):
+            await self.persistence.save_binding(node_type, local_id, payload_dict)
+            return
+        await self.persistence.save_binding(self.scope, node_type, local_id, payload_dict)
+
+    async def _persistence_delete_binding(self, node_type: str, local_id: str) -> None:
+        if isinstance(self.persistence, ScopedPersistenceView):
+            await self.persistence.delete_binding(node_type, local_id)
+            return
+        await self.persistence.delete_binding(self.scope, node_type, local_id)
+
+    async def _persistence_delete_bindings_bulk(self, node_type: str, local_ids: List[str]) -> None:
+        if isinstance(self.persistence, ScopedPersistenceView):
+            await self.persistence.delete_bindings_bulk(node_type, local_ids)
+            return
+        await self.persistence.delete_bindings_bulk(self.scope, node_type, local_ids)
+
+    async def _persistence_save_bindings_bulk(self, node_type: str, bindings: Dict[str, Optional[str]]) -> None:
+        if isinstance(self.persistence, ScopedPersistenceView):
+            await self.persistence.save_bindings_bulk(node_type, bindings)
+            return
+        await self.persistence.save_bindings_bulk(self.scope, node_type, bindings)
+
+    async def _persistence_load_bindings(self, node_type: str) -> List[Dict[str, Any]]:
+        if isinstance(self.persistence, ScopedPersistenceView):
+            return await self.persistence.load_bindings(node_type)
+        return await self.persistence.load_bindings(self.scope, node_type)
+
    def _get_type_bindings(self, node_type: str) -> Dict[str, Optional[str]]:
        if node_type not in self._bindings:
            self._bindings[node_type] = {}
@@ -58,7 +98,7 @@ class BindingManager:
            logger.debug(log_line)
        
        if self.auto_persist:
-            await self.persistence.save_binding(self.scope, node_type, local_id, {"remote_id": remote_id})
+            await self._persistence_save_binding(node_type, local_id, {"remote_id": remote_id})

    async def unbind(self, node_type: str, local_id: str, manual: bool = False):
        """
@@ -83,14 +123,19 @@ class BindingManager:
        )
        
        if self.auto_persist:
-            await self.persistence.delete_binding(self.scope, node_type, local_id)
+            await self._persistence_delete_binding(node_type, local_id)
        else:
            self._get_deleted(node_type).add(local_id)

    async def get_remote_id(self, node_type: str, local_id: str) -> Optional[str]:
        """查询本地节点对应的远程 ID"""
        bindings = self._get_type_bindings(node_type)
-        return bindings.get(local_id)
+        remote_id = bindings.get(local_id)
+        if remote_id is not None or local_id in bindings:
+            return remote_id
+        if self._global_fallback_manager is None:
+            return None
+        return await self._global_fallback_manager.get_remote_id(node_type, local_id)

    async def get_local_id(self, node_type: str, remote_id: str) -> Optional[str]:
        """查询远程节点对应的本地 ID (内存遍历)"""
@@ -98,11 +143,13 @@ class BindingManager:
        for lid, rid in bindings.items():
            if rid == remote_id:
                return lid
-        return None
+        if self._global_fallback_manager is None:
+            return None
+        return await self._global_fallback_manager.get_local_id(node_type, remote_id)

    async def get_record(self, node_type: str, local_id: str) -> Optional[BindingRecord]:
        """获取结构化绑定记录"""
-        remote_id = self._get_type_bindings(node_type).get(local_id)
+        remote_id = await self.get_remote_id(node_type, local_id)
        if remote_id is None:
            return None
        return BindingRecord(
@@ -122,7 +169,7 @@ class BindingManager:

    async def load_from_persistence(self, node_type: str):
        """从持久化层加载特定类型的绑定关系到内存"""
-        raw_bindings = await self.persistence.load_bindings(self.scope, node_type)
+        raw_bindings = await self._persistence_load_bindings(node_type)
        bindings = self._get_type_bindings(node_type)
        bindings.clear()
        self._get_deleted(node_type).clear()
@@ -165,7 +212,7 @@ class BindingManager:
                del bindings[local_id]
                deleted_count += 1
                if self.auto_persist:
-                    await self.persistence.delete_binding(self.scope, node_type, local_id)
+                    await self._persistence_delete_binding(node_type, local_id)
                else:
                    self._get_deleted(node_type).add(local_id)

@@ -198,13 +245,19 @@ class BindingManager:
                touched_types.append(node_type)

            if deleted:
-                await self.persistence.delete_bindings_bulk(self.scope, node_type, deleted)
+                await self._persistence_delete_bindings_bulk(node_type, deleted)
                self._get_deleted(node_type).clear()

            if bindings:
-                await self.persistence.save_bindings_bulk(self.scope, node_type, bindings)
+                await self._persistence_save_bindings_bulk(node_type, bindings)

        touched_text = ",".join(touched_types) if touched_types else "none"
        logger.info(
            f"🔍 [BindingManager.persist] 完成: scope={self.scope}, types={type_count}, saved={saved_total}, deleted={deleted_total}, touched=[{touched_text}]"
        )
+
+    async def unload(self, *, node_types: Optional[List[str]] = None) -> None:
+        target_types = list(node_types) if node_types is not None else list(self._bindings.keys())
+        for node_type in target_types:
+            self._bindings.pop(node_type, None)
+            self._deleted.pop(node_type, None)
@@ -5,6 +5,7 @@ from typing import Dict, List, Optional, Any, Set, Callable, Literal
 from .sync_node import SyncNode
 from .types import SyncStatus
 from .persistence import PersistenceBackend
+from .persistence_service import ScopedPersistenceView


 class DataCollection:
@@ -24,15 +25,17 @@ class DataCollection:
        self,
        collection_id: str,
        scope: str = "global",
-        persistence: Optional[PersistenceBackend] = None,
+        persistence: Optional[PersistenceBackend | ScopedPersistenceView] = None,
        auto_persist: bool = False,
        sm_runtime: Optional[Any] = None,
+        global_fallback_collection: Optional["DataCollection"] = None,
    ):
        self.collection_id = collection_id
        self.scope = scope
        self.persistence = persistence
        self.auto_persist = auto_persist
        self._sm_runtime = sm_runtime
+        self._global_fallback_collection = global_fallback_collection
        self._nodes: Dict[str, SyncNode] = {}  # 内存缓存
        # data_id 全局唯一（按 collection 维度），用于 O(1) 反查 node_id
        self._data_id_to_node_id: Dict[str, str] = {}
@@ -53,6 +56,69 @@ class DataCollection:
        """设置 Collection 默认状态机 runtime。"""
        self._sm_runtime = runtime

+    def set_global_fallback_collection(self, collection: Optional["DataCollection"]) -> None:
+        self._global_fallback_collection = collection
+
+    def _get_global_fallback(self) -> Optional["DataCollection"]:
+        return self._global_fallback_collection
+
+    async def _persistence_save_node(self, node: SyncNode) -> None:
+        if not self.persistence:
+            return
+        if isinstance(self.persistence, ScopedPersistenceView):
+            await self.persistence.save_node(self.collection_id, node)
+            return
+        await self.persistence.save_node(self.collection_id, self.scope, node)
+
+    async def _persistence_delete_node(self, node_id: str) -> None:
+        if not self.persistence:
+            return
+        if isinstance(self.persistence, ScopedPersistenceView):
+            await self.persistence.delete_node(self.collection_id, node_id)
+            return
+        await self.persistence.delete_node(self.collection_id, self.scope, node_id)
+
+    async def _persistence_delete_nodes_bulk(self, node_ids: List[str]) -> None:
+        if not self.persistence:
+            return
+        if isinstance(self.persistence, ScopedPersistenceView):
+            await self.persistence.delete_nodes_bulk(self.collection_id, node_ids)
+            return
+        await self.persistence.delete_nodes_bulk(self.collection_id, self.scope, node_ids)
+
+    async def _persistence_save_nodes_bulk(self, nodes: List[SyncNode]) -> None:
+        if not self.persistence:
+            return
+        if isinstance(self.persistence, ScopedPersistenceView):
+            await self.persistence.save_nodes_bulk(self.collection_id, nodes)
+            return
+        await self.persistence.save_nodes_bulk(self.collection_id, self.scope, nodes)
+
+    async def _persistence_load_nodes(
+        self,
+        *,
+        exclude_node_types: Optional[List[str]] = None,
+    ) -> List[Dict[str, Any]]:
+        if not self.persistence:
+            return []
+        if isinstance(self.persistence, ScopedPersistenceView):
+            return await self.persistence.load_nodes(
+                self.collection_id,
+                exclude_node_types=exclude_node_types,
+            )
+        return await self.persistence.load_nodes(
+            self.collection_id,
+            self.scope,
+            exclude_node_types=exclude_node_types,
+        )
+
+    async def _persistence_load_bind_data_id_overrides(self, node_type: str) -> Dict[str, Optional[str]]:
+        if not self.persistence:
+            return {}
+        if isinstance(self.persistence, ScopedPersistenceView):
+            return await self.persistence.load_bind_data_id_overrides(node_type)
+        return await self.persistence.load_bind_data_id_overrides(node_type, self.scope)
+
    def _remove_data_id_index(self, node_id: str, data_id: Optional[str] = None) -> None:
        target_data_id = data_id
        if target_data_id is None:
@@ -86,7 +152,7 @@ class DataCollection:
        self._nodes[node.node_id] = node
        self._deleted_node_ids.discard(node.node_id)
        if self.persistence and self.auto_persist:
-            await self.persistence.save_node(self.collection_id, self.scope, node)
+            await self._persistence_save_node(node)

    async def update(self, node: SyncNode):
        old_data_id = self._node_id_to_data_id.get(node.node_id)
@@ -105,7 +171,7 @@ class DataCollection:
        self._nodes[node.node_id] = node
        self._deleted_node_ids.discard(node.node_id)
        if self.persistence and self.auto_persist:
-            await self.persistence.save_node(self.collection_id, self.scope, node)
+            await self._persistence_save_node(node)

    async def delete(self, node_id: str):
        self._remove_data_id_index(node_id)
@@ -114,12 +180,18 @@ class DataCollection:
            del self._nodes[node_id]
        if self.persistence:
            if self.auto_persist:
-                await self.persistence.delete_node(self.collection_id, self.scope, node_id)
+                await self._persistence_delete_node(node_id)
            else:
                self._deleted_node_ids.add(node_id)

    def get(self, node_id: str) -> Optional[SyncNode]:
-        return self._nodes.get(node_id)
+        node = self._nodes.get(node_id)
+        if node is not None:
+            return node
+        fallback = self._get_global_fallback()
+        if fallback is None:
+            return None
+        return fallback.get(node_id)
    
    def get_by_node_id(self, node_id: str) -> Optional[SyncNode]:
        """别名方法，与 get() 功能相同"""
@@ -131,14 +203,23 @@ class DataCollection:

    def get_node_id_by_data_id(self, data_id: str) -> Optional[str]:
        """按 data_id 反查 node_id（O(1)）。"""
-        return self._data_id_to_node_id.get(data_id)
+        node_id = self._data_id_to_node_id.get(data_id)
+        if node_id is not None:
+            return node_id
+        fallback = self._get_global_fallback()
+        if fallback is None:
+            return None
+        return fallback.get_node_id_by_data_id(data_id)

    def get_by_data_id_any(self, data_id: str) -> Optional[SyncNode]:
        """按 data_id 查找节点（不要求 node_type）（O(1)）。"""
        node_id = self._data_id_to_node_id.get(data_id)
-        if not node_id:
+        if node_id:
+            return self._nodes.get(node_id)
+        fallback = self._get_global_fallback()
+        if fallback is None:
            return None
-        return self._nodes.get(node_id)
+        return fallback.get_by_data_id_any(data_id)

    def get_by_data_id(self, node_type: str, data_id: str) -> Optional[SyncNode]:
        """按 node_type 和 data_id 查找节点"""
@@ -389,7 +470,7 @@ class DataCollection:
        self._node_id_to_data_id = {}
        self._deleted_node_ids.clear()

-        node_dicts = await self.persistence.load_nodes(self.collection_id, self.scope)
+        node_dicts = await self._persistence_load_nodes()
        
        from .types import SyncAction, SyncStatus, BindingStatus
        from .registry import DomainRegistry
@@ -511,7 +592,7 @@ class DataCollection:

        node_types = {node.node_type for node in self._nodes.values()}
        for node_type in node_types:
-            override_map = await self.persistence.load_bind_data_id_overrides(node_type, self.scope)
+            override_map = await self._persistence_load_bind_data_id_overrides(node_type)
            for node in self.filter(node_type=node_type):
                bind_data_id = override_map.get(node.node_id)
                if bind_data_id:
@@ -529,11 +610,7 @@ class DataCollection:
        self._node_id_to_data_id = {}
        self._deleted_node_ids.clear()

-        node_dicts = await self.persistence.load_nodes(
-            self.collection_id,
-            self.scope,
-            exclude_node_types=node_types,
-        )
+        node_dicts = await self._persistence_load_nodes(exclude_node_types=node_types)

        from .types import SyncAction, SyncStatus, BindingStatus
        from .registry import DomainRegistry
@@ -637,7 +714,7 @@ class DataCollection:

        loaded_node_types = {node.node_type for node in self._nodes.values()}
        for node_type in loaded_node_types:
-            override_map = await self.persistence.load_bind_data_id_overrides(node_type, self.scope)
+            override_map = await self._persistence_load_bind_data_id_overrides(node_type)
            for node in self.filter(node_type=node_type):
                bind_data_id = override_map.get(node.node_id)
                if bind_data_id:
@@ -670,7 +747,7 @@ class DataCollection:

        # 先落库删除（用于 auto_persist=False 的场景）
        if self._deleted_node_ids:
-            await self.persistence.delete_nodes_bulk(self.collection_id, self.scope, list(self._deleted_node_ids))
+            await self._persistence_delete_nodes_bulk(list(self._deleted_node_ids))
            self._deleted_node_ids.clear()

        if not self._nodes:
@@ -683,7 +760,13 @@ class DataCollection:
        if not nodes_to_persist:
            return

-        await self.persistence.save_nodes_bulk(self.collection_id, self.scope, nodes_to_persist)
+        await self._persistence_save_nodes_bulk(nodes_to_persist)
+
+    async def unload(self) -> None:
+        self._nodes.clear()
+        self._data_id_to_node_id.clear()
+        self._node_id_to_data_id.clear()
+        self._deleted_node_ids.clear()

    async def filter_by_project_ids(self, data_id_filter: List[str]) -> int:
        """按目标项目 data_id 集合过滤当前 collection，返回删除数量。"""
@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from .persistence import PersistenceBackend
+
+
+class PersistenceService:
+    """Shared persistence service that owns the backend lifecycle."""
+
+    def __init__(self, backend: PersistenceBackend):
+        self._backend = backend
+
+    @property
+    def backend(self) -> PersistenceBackend:
+        return self._backend
+
+    async def initialize(self) -> None:
+        await self._backend.initialize()
+
+    def view(self, scope: str) -> "ScopedPersistenceView":
+        return ScopedPersistenceView(self._backend, scope=scope)
+
+    async def close(self) -> None:
+        await self._backend.close()
+
+
+class ScopedPersistenceView:
+    """Thin scope-bound persistence view used by project runtimes."""
+
+    def __init__(self, backend: PersistenceBackend, *, scope: str):
+        self._backend = backend
+        self.scope = scope
+
+    @property
+    def backend(self) -> PersistenceBackend:
+        return self._backend
+
+    async def load_nodes(
+        self,
+        collection_id: str,
+        *,
+        exclude_node_types: Optional[List[str]] = None,
+    ) -> List[Dict[str, Any]]:
+        return await self._backend.load_nodes(
+            collection_id,
+            self.scope,
+            exclude_node_types=exclude_node_types,
+        )
+
+    async def save_node(self, collection_id: str, node) -> None:
+        await self._backend.save_node(collection_id, self.scope, node)
+
+    async def save_nodes_bulk(self, collection_id: str, nodes) -> None:
+        await self._backend.save_nodes_bulk(collection_id, self.scope, nodes)
+
+    async def delete_node(self, collection_id: str, node_id: str) -> None:
+        await self._backend.delete_node(collection_id, self.scope, node_id)
+
+    async def delete_nodes_bulk(self, collection_id: str, node_ids: List[str]) -> None:
+        await self._backend.delete_nodes_bulk(collection_id, self.scope, node_ids)
+
+    async def load_bindings(self, node_type: str) -> List[Dict[str, Any]]:
+        return await self._backend.load_bindings(self.scope, node_type)
+
+    async def save_binding(self, node_type: str, local_id: str, payload_dict: Dict[str, Any]) -> None:
+        await self._backend.save_binding(self.scope, node_type, local_id, payload_dict)
+
+    async def save_bindings_bulk(self, node_type: str, bindings: Dict[str, Optional[str]]) -> None:
+        await self._backend.save_bindings_bulk(self.scope, node_type, bindings)
+
+    async def delete_binding(self, node_type: str, local_id: str) -> None:
+        await self._backend.delete_binding(self.scope, node_type, local_id)
+
+    async def delete_bindings_bulk(self, node_type: str, local_ids: List[str]) -> None:
+        await self._backend.delete_bindings_bulk(self.scope, node_type, local_ids)
+
+    async def load_bind_data_id_overrides(self, node_type: str) -> Dict[str, Optional[str]]:
+        return await self._backend.load_bind_data_id_overrides(node_type, self.scope)
+
+    async def delete_scope_node_types(self, *, node_types: List[str]) -> None:
+        await self._backend.delete_scope_node_types(scope=self.scope, node_types=node_types)
+
+    async def close(self) -> None:
+        return None