strepsiades
541 lines
22 KiB
Python
541 lines
22 KiB
Python
import uuid
|
||
from enum import Enum
|
||
from typing import Dict, List, Optional, Any, Set, Callable, Literal
|
||
from .sync_node import SyncNode
|
||
from .types import SyncStatus
|
||
from .persistence import PersistenceBackend
|
||
|
||
|
||
class DataCollection:
|
||
"""
|
||
DataCollection 管理一组同步节点的内存集合。
|
||
支持快速遍历、CRUD 以及依赖检查。
|
||
|
||
唯一性保证：
|
||
- node_id: 全局唯一，由 Collection.generate_node_id() 生成
|
||
- data_id: 业务主键，允许空字符串但不允许重复（非空时）
|
||
|
||
node_id 生成策略：
|
||
- 由 Collection.generate_node_id() 统一生成
|
||
- Collection.add() 会检查 node_id 唯一性，重复时抛出异常
|
||
"""
|
||
def __init__(
|
||
self,
|
||
collection_id: str,
|
||
persistence: Optional[PersistenceBackend] = None,
|
||
auto_persist: bool = False,
|
||
sm_runtime: Optional[Any] = None,
|
||
):
|
||
self.collection_id = collection_id
|
||
self.persistence = persistence
|
||
self.auto_persist = auto_persist
|
||
self._sm_runtime = sm_runtime
|
||
self._nodes: Dict[str, SyncNode] = {} # 内存缓存
|
||
# data_id 全局唯一（按 collection 维度），用于 O(1) 反查 node_id
|
||
self._data_id_to_node_id: Dict[str, str] = {}
|
||
# auto_persist=False 时，delete() 不会立即落库；这里记录删除集合，persist() 时统一写回
|
||
self._deleted_node_ids: set[str] = set()
|
||
|
||
@staticmethod
|
||
def is_bootstrap_blocked(node: SyncNode) -> bool:
|
||
"""是否为加载异常隔离节点（不参与后续自动流程）。"""
|
||
context = node.context
|
||
if not isinstance(context, dict):
|
||
return False
|
||
return bool(context.get("bootstrap_blocked"))
|
||
|
||
def set_state_machine_runtime(self, runtime: Any) -> None:
|
||
"""设置 Collection 默认状态机 runtime。"""
|
||
self._sm_runtime = runtime
|
||
|
||
async def add(self, node: SyncNode):
|
||
# 检查 node_id 唯一性
|
||
if node.node_id in self._nodes:
|
||
raise ValueError(
|
||
f"Duplicate node_id detected: {node.node_id} already exists in collection"
|
||
)
|
||
|
||
# 检查 data_id 唯一性（允许空字符串）
|
||
if node.data_id and node.data_id != "":
|
||
existing_node_id = self._data_id_to_node_id.get(node.data_id)
|
||
if existing_node_id is not None:
|
||
raise ValueError(
|
||
f"Duplicate data_id detected: {node.data_id} already bound to node_id={existing_node_id}"
|
||
)
|
||
self._data_id_to_node_id[node.data_id] = node.node_id
|
||
|
||
self._nodes[node.node_id] = node
|
||
self._deleted_node_ids.discard(node.node_id)
|
||
if self.persistence and self.auto_persist:
|
||
await self.persistence.save_node(self.collection_id, node)
|
||
|
||
async def update(self, node: SyncNode):
|
||
old_node = self._nodes.get(node.node_id)
|
||
if old_node and old_node.data_id and old_node.data_id != "" and old_node.data_id != node.data_id:
|
||
# node_id 不变但 data_id 变更，清理旧索引
|
||
self._data_id_to_node_id.pop(old_node.data_id, None)
|
||
|
||
if node.data_id and node.data_id != "":
|
||
existing_node_id = self._data_id_to_node_id.get(node.data_id)
|
||
if existing_node_id is not None and existing_node_id != node.node_id:
|
||
raise ValueError(
|
||
f"Duplicate data_id detected: {node.data_id} already bound to node_id={existing_node_id}"
|
||
)
|
||
self._data_id_to_node_id[node.data_id] = node.node_id
|
||
self._nodes[node.node_id] = node
|
||
self._deleted_node_ids.discard(node.node_id)
|
||
if self.persistence and self.auto_persist:
|
||
await self.persistence.save_node(self.collection_id, node)
|
||
|
||
async def delete(self, node_id: str):
|
||
old_node = self._nodes.get(node_id)
|
||
if old_node and old_node.data_id and old_node.data_id != "":
|
||
self._data_id_to_node_id.pop(old_node.data_id, None)
|
||
|
||
if node_id in self._nodes:
|
||
del self._nodes[node_id]
|
||
if self.persistence:
|
||
if self.auto_persist:
|
||
await self.persistence.delete_node(self.collection_id, node_id)
|
||
else:
|
||
self._deleted_node_ids.add(node_id)
|
||
|
||
def get(self, node_id: str) -> Optional[SyncNode]:
|
||
return self._nodes.get(node_id)
|
||
|
||
def get_by_node_id(self, node_id: str) -> Optional[SyncNode]:
|
||
"""别名方法，与 get() 功能相同"""
|
||
return self.get(node_id)
|
||
|
||
def generate_node_id(self) -> str:
|
||
"""生成唯一的 node_id"""
|
||
return str(uuid.uuid4())
|
||
|
||
def get_node_id_by_data_id(self, data_id: str) -> Optional[str]:
|
||
"""按 data_id 反查 node_id（O(1)）。"""
|
||
return self._data_id_to_node_id.get(data_id)
|
||
|
||
def get_by_data_id_any(self, data_id: str) -> Optional[SyncNode]:
|
||
"""按 data_id 查找节点（不要求 node_type）（O(1)）。"""
|
||
node_id = self._data_id_to_node_id.get(data_id)
|
||
if not node_id:
|
||
return None
|
||
return self._nodes.get(node_id)
|
||
|
||
def get_by_data_id(self, node_type: str, data_id: str) -> Optional[SyncNode]:
|
||
"""按 node_type 和 data_id 查找节点"""
|
||
node = self.get_by_data_id_any(data_id)
|
||
if not node:
|
||
return None
|
||
return node if node.node_type == node_type else None
|
||
|
||
def filter(
|
||
self,
|
||
node_type: Optional[str] = None,
|
||
filter: Optional[Dict[str, Any]] = None,
|
||
node_filter: Optional[
|
||
Dict[
|
||
Literal[
|
||
"node_id",
|
||
"data_id",
|
||
"depend_ids",
|
||
"origin_data",
|
||
"action",
|
||
"status",
|
||
"binding_status",
|
||
"error",
|
||
"context",
|
||
"sync_log",
|
||
"node_type",
|
||
],
|
||
Any,
|
||
]
|
||
| Callable[[SyncNode], bool]
|
||
] = None
|
||
) -> List[SyncNode]:
|
||
"""
|
||
获取节点列表。
|
||
- node_type: 按 node_type 过滤。
|
||
- filter: 按 node.data 中的属性过滤 (业务数据)。
|
||
- node_filter: 按 SyncNode 顶级属性过滤
|
||
- 如果是字典：按属性匹配 (如 {"status": SyncStatus.PENDING})
|
||
- 如果是函数：自定义过滤函数 (如 lambda n: n.status == SyncStatus.PENDING)
|
||
"""
|
||
results = self._nodes.values()
|
||
|
||
# 1. 按 node_type 过滤
|
||
if node_type:
|
||
results = [n for n in results if n.node_type == node_type]
|
||
else:
|
||
results = list(results)
|
||
|
||
# 2. 按 node_filter 过滤 (顶级属性)
|
||
if node_filter:
|
||
if callable(node_filter):
|
||
# 函数过滤器
|
||
results = [n for n in results if node_filter(n)]
|
||
else:
|
||
filtered = []
|
||
for node in results:
|
||
match = True
|
||
for k, v in node_filter.items():
|
||
if k == "node_id":
|
||
actual = node.node_id
|
||
elif k == "data_id":
|
||
actual = node.data_id
|
||
elif k == "depend_ids":
|
||
actual = node.depend_ids
|
||
elif k == "origin_data":
|
||
actual = node.origin_data
|
||
elif k == "action":
|
||
actual = node.action
|
||
elif k == "status":
|
||
actual = node.status
|
||
elif k == "binding_status":
|
||
actual = node.binding_status
|
||
elif k == "error":
|
||
actual = node.error
|
||
elif k == "context":
|
||
actual = node.context
|
||
elif k == "sync_log":
|
||
actual = node.sync_log
|
||
elif k == "node_type":
|
||
actual = node.node_type
|
||
else:
|
||
match = False
|
||
break
|
||
if actual != v:
|
||
match = False
|
||
break
|
||
if match:
|
||
filtered.append(node)
|
||
results = filtered
|
||
|
||
# 3. 按 filter 过滤 (data 内部)
|
||
if filter:
|
||
filtered = []
|
||
for node in results:
|
||
node_data = node.get_data()
|
||
if not node_data:
|
||
match = False
|
||
else:
|
||
match = True
|
||
for k, v in filter.items():
|
||
actual_val = node_data.get(k)
|
||
if actual_val != v:
|
||
match = False
|
||
break
|
||
if match:
|
||
filtered.append(node)
|
||
results = filtered
|
||
|
||
return results
|
||
|
||
@staticmethod
|
||
def _expected_matches_actual(expected: Any, actual: Any) -> bool:
|
||
if isinstance(expected, list):
|
||
return any(DataCollection._expected_matches_actual(item, actual) for item in expected)
|
||
|
||
if isinstance(expected, str):
|
||
actual_candidates = set()
|
||
if isinstance(actual, Enum):
|
||
actual_candidates.update({str(actual.name), str(actual.value)})
|
||
else:
|
||
actual_candidates.add(str(actual))
|
||
expected_lower = expected.lower()
|
||
return any(str(candidate).lower() == expected_lower for candidate in actual_candidates)
|
||
|
||
return expected == actual
|
||
|
||
@classmethod
|
||
def _node_matches_state_id(
|
||
cls,
|
||
runtime,
|
||
*,
|
||
node: SyncNode,
|
||
state_id: str,
|
||
action_hint: Optional[str] = None,
|
||
match_fields: Optional[Set[str]] = None,
|
||
) -> bool:
|
||
states = runtime._config.raw.get("states")
|
||
pseudo_states = runtime._config.raw.get("pseudo_states")
|
||
if not isinstance(states, dict):
|
||
raise RuntimeError("invalid state-machine config: missing states")
|
||
if pseudo_states is not None and not isinstance(pseudo_states, dict):
|
||
raise RuntimeError("invalid state-machine config: pseudo_states must be mapping")
|
||
|
||
state_def = states.get(state_id)
|
||
if state_def is None and isinstance(pseudo_states, dict):
|
||
state_def = pseudo_states.get(state_id)
|
||
if not isinstance(state_def, dict):
|
||
return False
|
||
|
||
snapshot = {
|
||
"binding_status": node.binding_status,
|
||
"action": action_hint if action_hint is not None else node.action,
|
||
"status": node.status,
|
||
"data_id_exist": bool(node.data_id),
|
||
}
|
||
selected_fields = match_fields or set(snapshot.keys())
|
||
|
||
for field, actual in snapshot.items():
|
||
if field not in selected_fields:
|
||
continue
|
||
if field not in state_def:
|
||
continue
|
||
expected = state_def[field]
|
||
if not cls._expected_matches_actual(expected, actual):
|
||
return False
|
||
return True
|
||
|
||
def filter_by_state_ids(
|
||
self,
|
||
runtime: Optional[Any] = None,
|
||
*,
|
||
state_ids: List[str],
|
||
node_type: Optional[str] = None,
|
||
include_bootstrap_blocked: bool = False,
|
||
action_hint: Optional[str] = None,
|
||
match_fields: Optional[Set[str]] = None,
|
||
) -> List[SyncNode]:
|
||
"""
|
||
按状态机状态名（Sxx）筛选节点。
|
||
|
||
说明：
|
||
- 状态定义来源于 `runtime` 关联的 YAML 配置；
|
||
- 默认排除 `bootstrap_blocked` 隔离节点；
|
||
- `action_hint` 可用于匹配类似 E40 场景的动作分支。
|
||
"""
|
||
runtime_obj = runtime or self._sm_runtime
|
||
if runtime_obj is None:
|
||
raise RuntimeError("state-machine runtime is not set for collection")
|
||
|
||
candidates = self.filter(node_type=node_type)
|
||
matched: List[SyncNode] = []
|
||
for node in candidates:
|
||
if not include_bootstrap_blocked and self.is_bootstrap_blocked(node):
|
||
continue
|
||
if any(
|
||
self._node_matches_state_id(
|
||
runtime_obj,
|
||
node=node,
|
||
state_id=state_id,
|
||
action_hint=action_hint,
|
||
match_fields=match_fields,
|
||
)
|
||
for state_id in state_ids
|
||
):
|
||
matched.append(node)
|
||
return matched
|
||
|
||
def check_depends(self, node_ids: List[str]) -> bool:
|
||
"""
|
||
检查指定的 node_id 列表是否存在，且状态是否不是 FAILED。
|
||
|
||
Args:
|
||
node_ids: 依赖的 node_id 列表
|
||
|
||
Returns:
|
||
bool: 所有依赖均满足则返回 True
|
||
"""
|
||
for nid in node_ids:
|
||
node = self.get(nid)
|
||
if not node:
|
||
return False
|
||
if node.status == SyncStatus.FAILED:
|
||
return False
|
||
return True
|
||
|
||
async def load_from_persistence(self):
|
||
"""
|
||
从后端加载数据到内存
|
||
|
||
使用 DomainRegistry 恢复正确的 SyncNode 类型。
|
||
如果 node_type 未注册，会抛出 ValueError。
|
||
|
||
**加载阶段**：
|
||
- 原样恢复持久化状态（binding_status/action/status/error）
|
||
- 清空 sync_log（避免跨轮次累积日志）
|
||
- data/origin_data 保留（由 DataSource 覆盖）
|
||
|
||
**初始化归并（E01）**：
|
||
- 由 Pipeline 在加载后调用 run_reset() 触发 E01
|
||
- CREATE 僵尸节点进入 S15 并删除
|
||
- 其余节点进入 S00（UNCHECKED/NONE/PENDING）
|
||
"""
|
||
if not self.persistence:
|
||
return
|
||
|
||
self._nodes = {}
|
||
self._data_id_to_node_id = {}
|
||
self._deleted_node_ids.clear()
|
||
|
||
node_dicts = await self.persistence.load_nodes(self.collection_id)
|
||
|
||
from .types import SyncAction, SyncStatus, BindingStatus
|
||
from .registry import DomainRegistry
|
||
|
||
for node_dict in node_dicts:
|
||
node_type = node_dict["node_type"]
|
||
|
||
# 从注册中心获取正确的 SyncNode 类（未注册会抛出 ValueError）
|
||
node_class = DomainRegistry.get_node_class(node_type)
|
||
assert issubclass(node_class, SyncNode)
|
||
|
||
# 严格恢复 action/status/binding_status；非法持久化值进入隔离态（S17 语义）
|
||
parse_errors: list[str] = []
|
||
|
||
# 原样恢复 action 字段（可能是字符串或枚举）
|
||
raw_action = node_dict.get("action", SyncAction.NONE)
|
||
if isinstance(raw_action, str):
|
||
try:
|
||
action_value = SyncAction(raw_action)
|
||
except ValueError:
|
||
parse_errors.append(f"invalid action={raw_action}")
|
||
action_value = SyncAction.NONE
|
||
elif isinstance(raw_action, SyncAction):
|
||
action_value = raw_action
|
||
else:
|
||
if raw_action is None:
|
||
action_value = SyncAction.NONE
|
||
else:
|
||
parse_errors.append(f"invalid action type={type(raw_action).__name__}")
|
||
action_value = SyncAction.NONE
|
||
|
||
# 原样恢复 status 字段（可能是字符串或枚举）
|
||
raw_status = node_dict.get("status", SyncStatus.PENDING)
|
||
if isinstance(raw_status, str):
|
||
try:
|
||
status_value = SyncStatus(raw_status)
|
||
except ValueError:
|
||
parse_errors.append(f"invalid status={raw_status}")
|
||
status_value = SyncStatus.FAILED
|
||
elif isinstance(raw_status, SyncStatus):
|
||
status_value = raw_status
|
||
else:
|
||
if raw_status is None:
|
||
status_value = SyncStatus.PENDING
|
||
else:
|
||
parse_errors.append(f"invalid status type={type(raw_status).__name__}")
|
||
status_value = SyncStatus.FAILED
|
||
|
||
# 原样恢复 binding_status 字段（可能是字符串或枚举）
|
||
raw_binding_status = node_dict.get("binding_status", BindingStatus.UNCHECKED)
|
||
if isinstance(raw_binding_status, str):
|
||
try:
|
||
binding_status_value = BindingStatus(raw_binding_status)
|
||
except ValueError:
|
||
parse_errors.append(f"invalid binding_status={raw_binding_status}")
|
||
binding_status_value = BindingStatus.ABNORMAL
|
||
elif isinstance(raw_binding_status, BindingStatus):
|
||
binding_status_value = raw_binding_status
|
||
else:
|
||
if raw_binding_status is None:
|
||
binding_status_value = BindingStatus.UNCHECKED
|
||
else:
|
||
parse_errors.append(
|
||
f"invalid binding_status type={type(raw_binding_status).__name__}"
|
||
)
|
||
binding_status_value = BindingStatus.ABNORMAL
|
||
|
||
raw_context = node_dict.get("context", {})
|
||
context_value = raw_context if isinstance(raw_context, dict) else {}
|
||
error_value = node_dict.get("error")
|
||
|
||
if parse_errors:
|
||
context_value = dict(context_value)
|
||
context_value["bootstrap_blocked"] = True
|
||
context_value["bootstrap_block_reason"] = "invalid_persisted_enum"
|
||
context_value["bootstrap_block_errors"] = parse_errors
|
||
binding_status_value = BindingStatus.ABNORMAL
|
||
action_value = SyncAction.NONE
|
||
status_value = SyncStatus.FAILED
|
||
details = "; ".join(parse_errors)
|
||
error_value = f"LOAD_INVALID_ENUM: {details}"
|
||
|
||
# 使用正确的类型创建节点（保留持久化状态，后续由 E01 统一归并）
|
||
try:
|
||
node = node_class(
|
||
node_id=node_dict["node_id"],
|
||
data_id=node_dict.get("data_id", ""),
|
||
depend_ids=node_dict.get("depend_ids", []),
|
||
data=node_dict.get("data"), # 保留，由 DataSource 覆盖
|
||
origin_data=node_dict.get("origin_data"), # 保留，由 DataSource 覆盖
|
||
action=action_value,
|
||
status=status_value,
|
||
binding_status=binding_status_value,
|
||
error=error_value,
|
||
sync_log=None, # 初始化加载时清空上一轮流转日志
|
||
context=context_value,
|
||
)
|
||
node.context["_loaded_from_persistence"] = True
|
||
except Exception:
|
||
import logging
|
||
logger = logging.getLogger(__name__)
|
||
logger.error(
|
||
f"Load failed for node_type={node_type}, node_id={node_dict.get('node_id')}",
|
||
exc_info=True
|
||
)
|
||
logger.error(f"Offending data: {node_dict.get('data')}")
|
||
raise
|
||
|
||
self._nodes[node.node_id] = node
|
||
|
||
if node.data_id and node.data_id != "":
|
||
existing_node_id = self._data_id_to_node_id.get(node.data_id)
|
||
if existing_node_id is not None and existing_node_id != node.node_id:
|
||
raise ValueError(
|
||
f"Duplicate data_id detected while loading: {node.data_id} already bound to node_id={existing_node_id}"
|
||
)
|
||
self._data_id_to_node_id[node.data_id] = node.node_id
|
||
|
||
node_types = {node.node_type for node in self._nodes.values()}
|
||
for node_type in node_types:
|
||
override_map = await self.persistence.load_bind_data_id_overrides(node_type)
|
||
for node in self.filter(node_type=node_type):
|
||
bind_data_id = override_map.get(node.node_id)
|
||
if bind_data_id:
|
||
node.context["bind_data_id"] = bind_data_id
|
||
else:
|
||
node.context.pop("bind_data_id", None)
|
||
|
||
async def persist(self) -> None:
|
||
"""将当前 collection 全量持久化到后端"""
|
||
if not self.persistence:
|
||
return
|
||
|
||
# 先落库删除（用于 auto_persist=False 的场景）
|
||
if self._deleted_node_ids:
|
||
await self.persistence.delete_nodes_bulk(self.collection_id, list(self._deleted_node_ids))
|
||
self._deleted_node_ids.clear()
|
||
|
||
if not self._nodes:
|
||
return
|
||
|
||
await self.persistence.save_nodes_bulk(self.collection_id, list(self._nodes.values()))
|
||
|
||
async def filter_by_project_ids(self, data_id_filter: List[str]) -> int:
|
||
"""按目标项目 data_id 集合过滤当前 collection，返回删除数量。"""
|
||
target_set = set(data_id_filter or [])
|
||
if not target_set:
|
||
return 0
|
||
|
||
nodes = self.filter()
|
||
deleted_count = 0
|
||
for node in nodes:
|
||
should_delete = False
|
||
if node.node_type == "project":
|
||
if node.data_id not in target_set:
|
||
should_delete = True
|
||
else:
|
||
data = node.get_data() or {}
|
||
project_id = data.get("project_id")
|
||
if project_id and project_id not in target_set:
|
||
should_delete = True
|
||
|
||
if should_delete:
|
||
await self.delete(node.node_id)
|
||
deleted_count += 1
|
||
|
||
return deleted_count
|