import uuid import copy from enum import Enum from typing import Dict, List, Optional, Any, Set, Callable, Literal from .sync_node import SyncNode from .types import SyncStatus from .persistence import PersistenceBackend class DataCollection: """ DataCollection 管理一组同步节点的内存集合。 支持快速遍历、CRUD 以及依赖检查。 唯一性保证： - node_id: 全局唯一，由 Collection.generate_node_id() 生成 - data_id: 业务主键，允许空字符串但不允许重复（非空时） node_id 生成策略： - 由 Collection.generate_node_id() 统一生成 - Collection.add() 会检查 node_id 唯一性，重复时抛出异常 """ def __init__( self, collection_id: str, scope: str = "global", persistence: Optional[PersistenceBackend] = None, auto_persist: bool = False, sm_runtime: Optional[Any] = None, ): self.collection_id = collection_id self.scope = scope self.persistence = persistence self.auto_persist = auto_persist self._sm_runtime = sm_runtime self._nodes: Dict[str, SyncNode] = {} # 内存缓存 # data_id 全局唯一（按 collection 维度），用于 O(1) 反查 node_id self._data_id_to_node_id: Dict[str, str] = {} # node_id -> data_id 反向索引，用于 O(1) 维护 data_id 更新 self._node_id_to_data_id: Dict[str, str] = {} # auto_persist=False 时，delete() 不会立即落库；这里记录删除集合，persist() 时统一写回 self._deleted_node_ids: set[str] = set() @staticmethod def is_bootstrap_blocked(node: SyncNode) -> bool: """是否为加载异常隔离节点（不参与后续自动流程）。""" context = node.context if not isinstance(context, dict): return False return bool(context.get("bootstrap_blocked")) def set_state_machine_runtime(self, runtime: Any) -> None: """设置 Collection 默认状态机 runtime。""" self._sm_runtime = runtime def _remove_data_id_index(self, node_id: str, data_id: Optional[str] = None) -> None: target_data_id = data_id if target_data_id is None: target_data_id = self._node_id_to_data_id.pop(node_id, None) else: self._node_id_to_data_id.pop(node_id, None) if target_data_id and self._data_id_to_node_id.get(target_data_id) == node_id: self._data_id_to_node_id.pop(target_data_id, None) def _set_data_id_index(self, node_id: str, data_id: str) -> None: self._data_id_to_node_id[data_id] = node_id self._node_id_to_data_id[node_id] = data_id async def add(self, node: SyncNode): # 检查 node_id 唯一性 if node.node_id in self._nodes: raise ValueError( f"Duplicate node_id detected: {node.node_id} already exists in collection" ) # 检查 data_id 唯一性（允许空字符串） if node.data_id and node.data_id != "": existing_node_id = self._data_id_to_node_id.get(node.data_id) if existing_node_id is not None: raise ValueError( f"Duplicate data_id detected: {node.data_id} already bound to node_id={existing_node_id}" ) self._set_data_id_index(node.node_id, node.data_id) self._nodes[node.node_id] = node self._deleted_node_ids.discard(node.node_id) if self.persistence and self.auto_persist: await self.persistence.save_node(self.collection_id, self.scope, node) async def update(self, node: SyncNode): old_data_id = self._node_id_to_data_id.get(node.node_id) if old_data_id and old_data_id != node.data_id: self._remove_data_id_index(node.node_id, old_data_id) if node.data_id and node.data_id != "": existing_node_id = self._data_id_to_node_id.get(node.data_id) if existing_node_id is not None and existing_node_id != node.node_id: raise ValueError( f"Duplicate data_id detected: {node.data_id} already bound to node_id={existing_node_id}" ) self._set_data_id_index(node.node_id, node.data_id) else: self._node_id_to_data_id.pop(node.node_id, None) self._nodes[node.node_id] = node self._deleted_node_ids.discard(node.node_id) if self.persistence and self.auto_persist: await self.persistence.save_node(self.collection_id, self.scope, node) async def delete(self, node_id: str): self._remove_data_id_index(node_id) if node_id in self._nodes: del self._nodes[node_id] if self.persistence: if self.auto_persist: await self.persistence.delete_node(self.collection_id, self.scope, node_id) else: self._deleted_node_ids.add(node_id) def get(self, node_id: str) -> Optional[SyncNode]: return self._nodes.get(node_id) def get_by_node_id(self, node_id: str) -> Optional[SyncNode]: """别名方法，与 get() 功能相同""" return self.get(node_id) def generate_node_id(self) -> str: """生成唯一的 node_id""" return str(uuid.uuid4()) def get_node_id_by_data_id(self, data_id: str) -> Optional[str]: """按 data_id 反查 node_id（O(1)）。""" return self._data_id_to_node_id.get(data_id) def get_by_data_id_any(self, data_id: str) -> Optional[SyncNode]: """按 data_id 查找节点（不要求 node_type）（O(1)）。""" node_id = self._data_id_to_node_id.get(data_id) if not node_id: return None return self._nodes.get(node_id) def get_by_data_id(self, node_type: str, data_id: str) -> Optional[SyncNode]: """按 node_type 和 data_id 查找节点""" node = self.get_by_data_id_any(data_id) if not node: return None return node if node.node_type == node_type else None def filter( self, node_type: Optional[str] = None, filter: Optional[Dict[str, Any]] = None, node_filter: Optional[ Dict[ Literal[ "node_id", "data_id", "depend_ids", "origin_data", "action", "status", "binding_status", "error", "context", "sync_log", "node_type", ], Any, ] | Callable[[SyncNode], bool] ] = None ) -> List[SyncNode]: """ 获取节点列表。 - node_type: 按 node_type 过滤。 - filter: 按 node.data 中的属性过滤 (业务数据)。 - node_filter: 按 SyncNode 顶级属性过滤 - 如果是字典：按属性匹配 (如 {"status": SyncStatus.PENDING}) - 如果是函数：自定义过滤函数 (如 lambda n: n.status == SyncStatus.PENDING) """ results = self._nodes.values() # 1. 按 node_type 过滤 if node_type: results = [n for n in results if n.node_type == node_type] else: results = list(results) # 2. 按 node_filter 过滤 (顶级属性) if node_filter: if callable(node_filter): # 函数过滤器 results = [n for n in results if node_filter(n)] else: filtered = [] for node in results: match = True for k, v in node_filter.items(): if k == "node_id": actual = node.node_id elif k == "data_id": actual = node.data_id elif k == "depend_ids": actual = node.depend_ids elif k == "origin_data": actual = node.origin_data elif k == "action": actual = node.action elif k == "status": actual = node.status elif k == "binding_status": actual = node.binding_status elif k == "error": actual = node.error elif k == "context": actual = node.context elif k == "sync_log": actual = node.sync_log elif k == "node_type": actual = node.node_type else: match = False break if actual != v: match = False break if match: filtered.append(node) results = filtered # 3. 按 filter 过滤 (data 内部) if filter: filtered = [] for node in results: node_data = node.get_data() if not node_data: match = False else: match = True for k, v in filter.items(): actual_val = node_data.get(k) if actual_val != v: match = False break if match: filtered.append(node) results = filtered return results @staticmethod def _expected_matches_actual(expected: Any, actual: Any) -> bool: if isinstance(expected, list): return any(DataCollection._expected_matches_actual(item, actual) for item in expected) if isinstance(expected, str): actual_candidates = set() if isinstance(actual, Enum): actual_candidates.update({str(actual.name), str(actual.value)}) else: actual_candidates.add(str(actual)) expected_lower = expected.lower() return any(str(candidate).lower() == expected_lower for candidate in actual_candidates) return expected == actual @classmethod def _node_matches_state_id( cls, runtime, *, node: SyncNode, state_id: str, action_hint: Optional[str] = None, match_fields: Optional[Set[str]] = None, ) -> bool: states = runtime._config.raw.get("states") pseudo_states = runtime._config.raw.get("pseudo_states") if not isinstance(states, dict): raise RuntimeError("invalid state-machine config: missing states") if pseudo_states is not None and not isinstance(pseudo_states, dict): raise RuntimeError("invalid state-machine config: pseudo_states must be mapping") state_def = states.get(state_id) if state_def is None and isinstance(pseudo_states, dict): state_def = pseudo_states.get(state_id) if not isinstance(state_def, dict): return False snapshot = { "binding_status": node.binding_status, "action": action_hint if action_hint is not None else node.action, "status": node.status, "data_id_exist": bool(node.data_id), } selected_fields = match_fields or set(snapshot.keys()) for field, actual in snapshot.items(): if field not in selected_fields: continue if field not in state_def: continue expected = state_def[field] if not cls._expected_matches_actual(expected, actual): return False return True def filter_by_state_ids( self, runtime: Optional[Any] = None, *, state_ids: List[str], node_type: Optional[str] = None, include_bootstrap_blocked: bool = False, action_hint: Optional[str] = None, match_fields: Optional[Set[str]] = None, ) -> List[SyncNode]: """ 按状态机状态名（Sxx）筛选节点。 说明： - 状态定义来源于 `runtime` 关联的 YAML 配置； - 默认排除 `bootstrap_blocked` 隔离节点； - `action_hint` 可用于匹配类似 E40 场景的动作分支。 """ runtime_obj = runtime or self._sm_runtime if runtime_obj is None: raise RuntimeError("state-machine runtime is not set for collection") candidates = self.filter(node_type=node_type) matched: List[SyncNode] = [] for node in candidates: if not include_bootstrap_blocked and self.is_bootstrap_blocked(node): continue if any( self._node_matches_state_id( runtime_obj, node=node, state_id=state_id, action_hint=action_hint, match_fields=match_fields, ) for state_id in state_ids ): matched.append(node) return matched def check_depends(self, node_ids: List[str]) -> bool: """ 检查指定的 node_id 列表是否存在，且状态是否不是 FAILED。 Args: node_ids: 依赖的 node_id 列表 Returns: bool: 所有依赖均满足则返回 True """ for nid in node_ids: node = self.get(nid) if not node: return False if node.status == SyncStatus.FAILED: return False return True async def load_from_persistence(self): """ 从后端加载数据到内存 使用 DomainRegistry 恢复正确的 SyncNode 类型。 如果 node_type 未注册，会抛出 ValueError。 **加载阶段**： - 原样恢复持久化状态（binding_status/action/status/error） - 清空 sync_log（避免跨轮次累积日志） - data/origin_data 保留（由 DataSource 覆盖） **初始化归并（E01）**： - 由 Pipeline 在加载后调用 `run_phase2_cleanup()` 触发 E01 - CREATE 僵尸节点进入 S15 并删除 - 其余节点进入 S00（UNCHECKED/NONE/PENDING） """ if not self.persistence: return self._nodes = {} self._data_id_to_node_id = {} self._node_id_to_data_id = {} self._deleted_node_ids.clear() node_dicts = await self.persistence.load_nodes(self.collection_id, self.scope) from .types import SyncAction, SyncStatus, BindingStatus from .registry import DomainRegistry for node_dict in node_dicts: node_type = node_dict["node_type"] # 从注册中心获取正确的 SyncNode 类（未注册会抛出 ValueError） node_class = DomainRegistry.get_node_class(node_type) assert issubclass(node_class, SyncNode) # 严格恢复 action/status/binding_status；非法持久化值进入隔离态（S17 语义） parse_errors: list[str] = [] # 原样恢复 action 字段（可能是字符串或枚举） raw_action = node_dict.get("action", SyncAction.NONE) if isinstance(raw_action, str): try: action_value = SyncAction(raw_action) except ValueError: parse_errors.append(f"invalid action={raw_action}") action_value = SyncAction.NONE elif isinstance(raw_action, SyncAction): action_value = raw_action else: if raw_action is None: action_value = SyncAction.NONE else: parse_errors.append(f"invalid action type={type(raw_action).__name__}") action_value = SyncAction.NONE # 原样恢复 status 字段（可能是字符串或枚举） raw_status = node_dict.get("status", SyncStatus.PENDING) if isinstance(raw_status, str): try: status_value = SyncStatus(raw_status) except ValueError: parse_errors.append(f"invalid status={raw_status}") status_value = SyncStatus.FAILED elif isinstance(raw_status, SyncStatus): status_value = raw_status else: if raw_status is None: status_value = SyncStatus.PENDING else: parse_errors.append(f"invalid status type={type(raw_status).__name__}") status_value = SyncStatus.FAILED # 原样恢复 binding_status 字段（可能是字符串或枚举） raw_binding_status = node_dict.get("binding_status", BindingStatus.UNCHECKED) if isinstance(raw_binding_status, str): try: binding_status_value = BindingStatus(raw_binding_status) except ValueError: parse_errors.append(f"invalid binding_status={raw_binding_status}") binding_status_value = BindingStatus.ABNORMAL elif isinstance(raw_binding_status, BindingStatus): binding_status_value = raw_binding_status else: if raw_binding_status is None: binding_status_value = BindingStatus.UNCHECKED else: parse_errors.append( f"invalid binding_status type={type(raw_binding_status).__name__}" ) binding_status_value = BindingStatus.ABNORMAL raw_context = node_dict.get("context", {}) context_value = raw_context if isinstance(raw_context, dict) else {} error_value = node_dict.get("error") if parse_errors: context_value = dict(context_value) context_value["bootstrap_blocked"] = True context_value["bootstrap_block_reason"] = "invalid_persisted_enum" context_value["bootstrap_block_errors"] = parse_errors binding_status_value = BindingStatus.ABNORMAL action_value = SyncAction.NONE status_value = SyncStatus.FAILED details = "; ".join(parse_errors) error_value = f"LOAD_INVALID_ENUM: {details}" # 使用正确的类型创建节点（保留持久化状态，后续由 E01 统一归并） try: node = node_class( node_id=node_dict["node_id"], data_id=node_dict.get("data_id", ""), depend_ids=node_dict.get("depend_ids", []), data=node_dict.get("data"), # 保留，由 DataSource 覆盖 origin_data=node_dict.get("origin_data"), # 保留，由 DataSource 覆盖 action=action_value, status=status_value, binding_status=binding_status_value, error=error_value, sync_log=None, # 初始化加载时清空上一轮流转日志 context=context_value, ) node.context["_loaded_from_persistence"] = True except Exception: import logging logger = logging.getLogger(__name__) logger.error( f"Load failed for node_type={node_type}, node_id={node_dict.get('node_id')}", exc_info=True ) logger.error(f"Offending data: {node_dict.get('data')}") raise self._nodes[node.node_id] = node if node.data_id and node.data_id != "": existing_node_id = self._data_id_to_node_id.get(node.data_id) if existing_node_id is not None and existing_node_id != node.node_id: raise ValueError( f"Duplicate data_id detected while loading: {node.data_id} already bound to node_id={existing_node_id}" ) self._set_data_id_index(node.node_id, node.data_id) node_types = {node.node_type for node in self._nodes.values()} for node_type in node_types: override_map = await self.persistence.load_bind_data_id_overrides(node_type, self.scope) for node in self.filter(node_type=node_type): bind_data_id = override_map.get(node.node_id) if bind_data_id: node.context["bind_data_id"] = bind_data_id else: node.context.pop("bind_data_id", None) async def load_from_persistence_excluding(self, node_types: Optional[List[str]] = None): """从持久化恢复数据，但跳过指定 node_type。""" if not self.persistence: return self._nodes = {} self._data_id_to_node_id = {} self._node_id_to_data_id = {} self._deleted_node_ids.clear() node_dicts = await self.persistence.load_nodes( self.collection_id, self.scope, exclude_node_types=node_types, ) from .types import SyncAction, SyncStatus, BindingStatus from .registry import DomainRegistry for node_dict in node_dicts: node_type = node_dict["node_type"] node_class = DomainRegistry.get_node_class(node_type) assert issubclass(node_class, SyncNode) parse_errors: list[str] = [] raw_action = node_dict.get("action", SyncAction.NONE) if isinstance(raw_action, str): try: action_value = SyncAction(raw_action) except ValueError: parse_errors.append(f"invalid action={raw_action}") action_value = SyncAction.NONE elif isinstance(raw_action, SyncAction): action_value = raw_action else: if raw_action is None: action_value = SyncAction.NONE else: parse_errors.append(f"invalid action type={type(raw_action).__name__}") action_value = SyncAction.NONE raw_status = node_dict.get("status", SyncStatus.PENDING) if isinstance(raw_status, str): try: status_value = SyncStatus(raw_status) except ValueError: parse_errors.append(f"invalid status={raw_status}") status_value = SyncStatus.FAILED elif isinstance(raw_status, SyncStatus): status_value = raw_status else: if raw_status is None: status_value = SyncStatus.PENDING else: parse_errors.append(f"invalid status type={type(raw_status).__name__}") status_value = SyncStatus.FAILED raw_binding_status = node_dict.get("binding_status", BindingStatus.UNCHECKED) if isinstance(raw_binding_status, str): try: binding_status_value = BindingStatus(raw_binding_status) except ValueError: parse_errors.append(f"invalid binding_status={raw_binding_status}") binding_status_value = BindingStatus.ABNORMAL elif isinstance(raw_binding_status, BindingStatus): binding_status_value = raw_binding_status else: if raw_binding_status is None: binding_status_value = BindingStatus.UNCHECKED else: parse_errors.append( f"invalid binding_status type={type(raw_binding_status).__name__}" ) binding_status_value = BindingStatus.ABNORMAL raw_context = node_dict.get("context", {}) context_value = raw_context if isinstance(raw_context, dict) else {} error_value = node_dict.get("error") if parse_errors: context_value = dict(context_value) context_value["bootstrap_blocked"] = True context_value["bootstrap_block_reason"] = "invalid_persisted_enum" context_value["bootstrap_block_errors"] = parse_errors binding_status_value = BindingStatus.ABNORMAL action_value = SyncAction.NONE status_value = SyncStatus.FAILED details = "; ".join(parse_errors) error_value = f"LOAD_INVALID_ENUM: {details}" node = node_class( node_id=node_dict["node_id"], data_id=node_dict.get("data_id", ""), depend_ids=node_dict.get("depend_ids", []), data=node_dict.get("data"), origin_data=node_dict.get("origin_data"), action=action_value, status=status_value, binding_status=binding_status_value, error=error_value, sync_log=None, context=context_value, ) node.context["_loaded_from_persistence"] = True self._nodes[node.node_id] = node if node.data_id and node.data_id != "": existing_node_id = self._data_id_to_node_id.get(node.data_id) if existing_node_id is not None and existing_node_id != node.node_id: raise ValueError( f"Duplicate data_id detected while loading: {node.data_id} already bound to node_id={existing_node_id}" ) self._set_data_id_index(node.node_id, node.data_id) loaded_node_types = {node.node_type for node in self._nodes.values()} for node_type in loaded_node_types: override_map = await self.persistence.load_bind_data_id_overrides(node_type, self.scope) for node in self.filter(node_type=node_type): bind_data_id = override_map.get(node.node_id) if bind_data_id: node.context["bind_data_id"] = bind_data_id else: node.context.pop("bind_data_id", None) async def import_nodes(self, nodes: List[SyncNode]) -> None: """导入现有节点到当前 collection，仅写内存，不触发持久化。""" for source_node in nodes: node = copy.deepcopy(source_node) if node.node_id in self._nodes: continue if node.data_id and node.data_id != "": existing_node_id = self._data_id_to_node_id.get(node.data_id) if existing_node_id is not None and existing_node_id != node.node_id: raise ValueError( f"Duplicate data_id detected: {node.data_id} already bound to node_id={existing_node_id}" ) self._set_data_id_index(node.node_id, node.data_id) self._nodes[node.node_id] = node self._deleted_node_ids.discard(node.node_id) async def persist(self, *, exclude_node_types: Optional[List[str]] = None) -> None: """将当前 collection 全量持久化到后端""" if not self.persistence: return # 先落库删除（用于 auto_persist=False 的场景） if self._deleted_node_ids: await self.persistence.delete_nodes_bulk(self.collection_id, self.scope, list(self._deleted_node_ids)) self._deleted_node_ids.clear() if not self._nodes: return excluded = set(node_type for node_type in (exclude_node_types or []) if node_type) nodes_to_persist = [ node for node in self._nodes.values() if node.node_type not in excluded ] if not nodes_to_persist: return await self.persistence.save_nodes_bulk(self.collection_id, self.scope, nodes_to_persist) async def filter_by_project_ids(self, data_id_filter: List[str]) -> int: """按目标项目 data_id 集合过滤当前 collection，返回删除数量。""" target_set = set(data_id_filter or []) if not target_set: return 0 nodes = self.filter() deleted_count = 0 for node in nodes: should_delete = False if node.node_type == "project": if node.data_id not in target_set: should_delete = True else: data = node.get_data() or {} project_id = data.get("project_id") if project_id and project_id not in target_set: should_delete = True if should_delete: await self.delete(node.node_id) deleted_count += 1 return deleted_count