strepsiades
566 lines
17 KiB
Python
566 lines
17 KiB
Python
"""
|
||
节点处理器 V2 (Node Handler - Pure Execution Layer)
|
||
|
||
Handler 职责：
|
||
1. 执行后端 API 调用（load/create/update/delete）
|
||
2. 返回执行结果（TaskResult）
|
||
3. 提供异步任务轮询（poll）
|
||
4. 提供数据质量检查（validate）
|
||
|
||
Handler 不负责：
|
||
- ❌ 状态管理（由 DataSource 负责）
|
||
- ❌ ID 映射（由 Strategy 负责，已保存在 node.data）
|
||
- ❌ 依赖关系（由 Strategy 负责）
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
from typing import (
|
||
TypeVar, Generic, Type, Protocol, Dict, List, Any, Optional,
|
||
runtime_checkable, TYPE_CHECKING
|
||
)
|
||
from abc import ABC, abstractmethod
|
||
from pydantic import BaseModel
|
||
|
||
from .task_result import TaskResult
|
||
|
||
if TYPE_CHECKING:
|
||
from ..common.sync_node import SyncNode
|
||
from ..common.collection import DataCollection
|
||
from .api.client import ApiClient
|
||
|
||
T = TypeVar("T", bound=BaseModel)
|
||
|
||
|
||
class ValidationResult:
|
||
"""数据质量检查结果"""
|
||
def __init__(self, is_valid: bool, errors: Optional[List[str]] = None):
|
||
self.is_valid = is_valid
|
||
self.errors = errors or []
|
||
|
||
|
||
@runtime_checkable
|
||
class NodeHandler(Protocol[T]):
|
||
"""
|
||
节点处理器协议（纯执行层）
|
||
|
||
Handler 只负责调用后端 API，不管理状态。
|
||
"""
|
||
|
||
@property
|
||
def node_type(self) -> str:
|
||
"""节点类型（如 'contract'）"""
|
||
...
|
||
|
||
@property
|
||
def node_class(self) -> Type["SyncNode[T]"]:
|
||
"""SyncNode 类型"""
|
||
...
|
||
|
||
@property
|
||
def schema(self) -> Type[T]:
|
||
"""Schema 类型"""
|
||
...
|
||
|
||
def set_collection(self, collection: "DataCollection") -> None:
|
||
"""
|
||
设置 Collection 引用
|
||
|
||
Args:
|
||
collection: 数据集合
|
||
"""
|
||
...
|
||
|
||
def set_handler_config(self, config: Dict[str, Any]) -> None:
|
||
"""注入 handler 级别配置。默认可忽略。"""
|
||
...
|
||
|
||
def set_api_client(self, client: "ApiClient") -> None:
|
||
"""注入 API 客户端。默认可忽略。"""
|
||
...
|
||
|
||
def set_poll_mode(self, mode: str) -> None:
|
||
"""设置轮询模式。默认可忽略。"""
|
||
...
|
||
|
||
def get_update_fields(self, *, exclude: frozenset[str] = frozenset({"id"})) -> List[str]:
|
||
"""返回 post-check 可比较字段列表。默认空。"""
|
||
...
|
||
|
||
async def load(self) -> List['SyncNode']:
|
||
"""
|
||
从后端加载全部数据并创建节点
|
||
|
||
注意：调用前需先 set_collection()
|
||
|
||
Returns:
|
||
SyncNode 列表
|
||
|
||
示例：
|
||
# Project Handler（无依赖）
|
||
async def load(self):
|
||
response = await self.api.get_projects()
|
||
nodes = []
|
||
for data in response["data"]:
|
||
node = self.create_node(data)
|
||
node.context["project_id"] = data["id"]
|
||
nodes.append(node)
|
||
return nodes
|
||
|
||
# Contract Handler（依赖 Project）
|
||
async def load(self):
|
||
# 使用 self._collection 查询依赖
|
||
projects = self._collection.filter(node_type="project")
|
||
project_ids = [p.data_id for p in projects if p.data_id]
|
||
|
||
nodes = []
|
||
for project_id in project_ids:
|
||
response = await self.api.get_contracts(project_id=project_id)
|
||
for data in response["data"]:
|
||
node = self.create_node(data)
|
||
node.context["project_id"] = project_id
|
||
node.context["contract_id"] = data["id"]
|
||
nodes.append(node)
|
||
return nodes
|
||
"""
|
||
...
|
||
|
||
async def create_all(
|
||
self,
|
||
nodes: List["SyncNode"]
|
||
) -> List[TaskResult]:
|
||
"""
|
||
批量创建节点
|
||
|
||
Args:
|
||
nodes: 需要创建的节点列表（DataSource 已筛选为 CREATE + PENDING）
|
||
|
||
Returns:
|
||
List[TaskResult]:
|
||
每个 TaskResult 必须包含 node_id，用于定位节点：
|
||
- TaskResult(status=SUCCESS, node_id="xxx", data_id="yyy")
|
||
- TaskResult(status=FAILED, node_id="xxx", error="...")
|
||
- TaskResult(status=IN_PROGRESS, node_id="xxx", task_id="...")
|
||
|
||
示例：
|
||
# JSONL Handler
|
||
async def create_all(self, nodes):
|
||
results = []
|
||
for node in nodes:
|
||
try:
|
||
new_id = str(uuid4())
|
||
data = node.get_data()
|
||
# ... 存储逻辑 ...
|
||
results.append(TaskResult.success(
|
||
node_id=node.node_id,
|
||
data_id=new_id
|
||
))
|
||
except Exception as e:
|
||
results.append(TaskResult.failed(
|
||
node_id=node.node_id,
|
||
error=str(e)
|
||
))
|
||
return results
|
||
|
||
# API Handler - 查询依赖
|
||
async def create_all(self, nodes):
|
||
results = []
|
||
for node in nodes:
|
||
# 查询依赖数据（如合同创建时查项目）
|
||
project_id = node.context.get("project_id")
|
||
projects = self._collection.filter(
|
||
node_type="project",
|
||
node_filter={"data_id": project_id}
|
||
)
|
||
|
||
if not projects:
|
||
results.append(TaskResult.failed(
|
||
node_id=node.node_id,
|
||
error=f"Project {project_id} not found"
|
||
))
|
||
continue
|
||
|
||
# 构造请求数据
|
||
data = node.get_data()
|
||
if not data:
|
||
results.append(TaskResult.failed(
|
||
node_id=node.node_id,
|
||
error="node data is None"
|
||
))
|
||
continue
|
||
|
||
contract_data = {
|
||
"project_id": projects[0].data_id,
|
||
**data
|
||
}
|
||
|
||
# 调用 API
|
||
response = await self.api.create_contract(contract_data)
|
||
results.append(TaskResult.success(
|
||
node_id=node.node_id,
|
||
data_id=response["id"]
|
||
))
|
||
|
||
return results
|
||
"""
|
||
...
|
||
|
||
async def update_all(
|
||
self,
|
||
nodes: List["SyncNode"]
|
||
) -> List[TaskResult]:
|
||
"""
|
||
批量更新节点
|
||
|
||
Args:
|
||
nodes: 需要更新的节点列表（DataSource 已筛选为 UPDATE + PENDING）
|
||
|
||
Returns:
|
||
List[TaskResult]（同 create_all）
|
||
"""
|
||
...
|
||
|
||
async def delete_all(
|
||
self,
|
||
nodes: List["SyncNode"]
|
||
) -> List[TaskResult]:
|
||
"""
|
||
批量删除节点
|
||
|
||
Args:
|
||
nodes: 需要删除的节点列表（DataSource 已筛选为 DELETE + PENDING）
|
||
|
||
Args:
|
||
collection: Collection 上下文
|
||
|
||
Returns:
|
||
List[TaskResult]（同 create_all）
|
||
"""
|
||
...
|
||
|
||
async def poll_tasks(self, task_ids: List[str]) -> Dict[str, TaskResult]:
|
||
"""
|
||
批量轮询异步任务状态
|
||
|
||
Args:
|
||
task_ids: 任务 ID 列表
|
||
|
||
Returns:
|
||
{task_id: TaskResult} 字典
|
||
- SUCCESS: 任务完成，返回 data_id
|
||
- FAILED: 任务失败，返回 error
|
||
- IN_PROGRESS: 仍在进行中
|
||
|
||
示例：
|
||
async def poll_tasks(self, task_ids):
|
||
response = await self.api.batch_query_tasks(task_ids)
|
||
results = {}
|
||
for task in response["tasks"]:
|
||
if task["status"] == "completed":
|
||
results[task["id"]] = TaskResult.success(
|
||
data_id=task["result"]["contract_id"]
|
||
)
|
||
elif task["status"] == "failed":
|
||
results[task["id"]] = TaskResult.failed(
|
||
error=task["error"]
|
||
)
|
||
else: # running
|
||
results[task["id"]] = TaskResult.in_progress(
|
||
task_id=task["id"]
|
||
)
|
||
return results
|
||
"""
|
||
...
|
||
|
||
async def validate(self, data: Dict[str, Any]) -> ValidationResult:
|
||
"""
|
||
数据质量检查
|
||
|
||
Args:
|
||
data: 原始数据
|
||
|
||
Returns:
|
||
ValidationResult（是否有效 + 错误列表）
|
||
|
||
示例：
|
||
async def validate(self, data):
|
||
errors = []
|
||
if not data.get("code"):
|
||
errors.append("合同编码为空")
|
||
if not data.get("project_id"):
|
||
errors.append("项目 ID 为空")
|
||
|
||
return ValidationResult(
|
||
is_valid=len(errors) == 0,
|
||
errors=errors
|
||
)
|
||
"""
|
||
...
|
||
|
||
def extract_id(self, data: Dict[str, Any]) -> str:
|
||
"""
|
||
从原始数据提取 ID
|
||
|
||
Args:
|
||
data: 原始数据
|
||
|
||
Returns:
|
||
业务主键
|
||
"""
|
||
...
|
||
|
||
def _create_node(self, data: Dict[str, Any]) -> "SyncNode[T]":
|
||
"""
|
||
从原始数据创建 SyncNode
|
||
|
||
Args:
|
||
data: 原始数据
|
||
|
||
Returns:
|
||
类型化的 SyncNode
|
||
"""
|
||
...
|
||
|
||
|
||
class BaseNodeHandler(ABC, Generic[T]):
|
||
"""
|
||
节点处理器基类
|
||
|
||
子类需实现：
|
||
- load()
|
||
- create()
|
||
- update()
|
||
- delete()（可选）
|
||
- poll_tasks()（如果支持异步）
|
||
- validate()（可选）
|
||
- extract_id()
|
||
"""
|
||
|
||
def __init__(
|
||
self,
|
||
node_type: str,
|
||
node_class: Type["SyncNode[T]"],
|
||
schema: Type[T]
|
||
):
|
||
self._node_type = node_type
|
||
self._node_class = node_class
|
||
self._schema = schema
|
||
self._collection: Optional["DataCollection"] = None
|
||
self.handler_config: Dict[str, Any] = {}
|
||
|
||
def set_collection(self, collection: "DataCollection") -> None:
|
||
"""
|
||
设置 Collection 引用
|
||
|
||
Args:
|
||
collection: 数据集合
|
||
"""
|
||
self._collection = collection
|
||
|
||
def set_handler_config(self, config: Dict[str, Any]) -> None:
|
||
"""默认保存 handler 级别配置，供子类按需读取。"""
|
||
self.handler_config = dict(config)
|
||
|
||
def get_data_id_filter(self) -> List[str]:
|
||
value = self.handler_config.get("data_id_filter", [])
|
||
if isinstance(value, str):
|
||
return [value] if value else []
|
||
if isinstance(value, list):
|
||
return [str(item) for item in value if str(item)]
|
||
return []
|
||
|
||
def set_api_client(self, client: "ApiClient") -> None:
|
||
"""默认忽略 API 客户端注入。"""
|
||
return
|
||
|
||
def set_poll_mode(self, mode: str) -> None:
|
||
"""默认忽略轮询模式。"""
|
||
return
|
||
|
||
def get_update_fields(self, *, exclude: frozenset[str] = frozenset({"id"})) -> List[str]:
|
||
"""默认无 post-check 字段白名单。"""
|
||
return []
|
||
|
||
@property
|
||
def node_type(self) -> str:
|
||
return self._node_type
|
||
|
||
@property
|
||
def node_class(self) -> Type["SyncNode[T]"]:
|
||
return self._node_class
|
||
|
||
@property
|
||
def schema(self) -> Type[T]:
|
||
return self._schema
|
||
|
||
# ========== 必须实现 ==========
|
||
|
||
@abstractmethod
|
||
async def load(self) -> List['SyncNode']:
|
||
"""
|
||
从数据源加载数据并创建节点
|
||
|
||
Handler 负责：
|
||
1. 加载原始数据
|
||
2. 创建 SyncNode（调用 create_node）
|
||
3. 设置 context（如 project_id, contract_id 等）
|
||
|
||
Returns:
|
||
SyncNode 列表
|
||
"""
|
||
pass
|
||
|
||
@abstractmethod
|
||
async def create_all(
|
||
self,
|
||
nodes: List["SyncNode"]
|
||
) -> List[TaskResult]:
|
||
"""批量创建节点"""
|
||
pass
|
||
|
||
@abstractmethod
|
||
async def update_all(
|
||
self,
|
||
nodes: List["SyncNode"]
|
||
) -> List[TaskResult]:
|
||
"""批量更新节点"""
|
||
pass
|
||
|
||
@abstractmethod
|
||
def extract_id(self, data: Dict[str, Any]) -> str:
|
||
"""提取 ID"""
|
||
pass
|
||
|
||
# ========== 可选实现 ==========
|
||
|
||
async def delete_all(
|
||
self,
|
||
nodes: List["SyncNode"]
|
||
) -> List[TaskResult]:
|
||
"""批量删除节点（默认不支持）"""
|
||
return [
|
||
TaskResult.failed(node_id=node.node_id, error="Delete operation not supported")
|
||
for node in nodes
|
||
]
|
||
|
||
async def poll_tasks(self, task_ids: List[str]) -> Dict[str, TaskResult]:
|
||
"""轮询异步任务（默认不支持）"""
|
||
return {
|
||
task_id: TaskResult.failed(error="Async tasks not supported")
|
||
for task_id in task_ids
|
||
}
|
||
|
||
async def validate(self, data: Dict[str, Any]) -> ValidationResult:
|
||
"""数据质量检查（默认不检查）"""
|
||
return ValidationResult(is_valid=True)
|
||
|
||
def create_node(self, data: Dict[str, Any]) -> "SyncNode[T]":
|
||
"""创建或复用 SyncNode（默认实现）"""
|
||
data_id = self.extract_id(data)
|
||
if self._collection is None:
|
||
raise RuntimeError("Collection not set. Call set_collection() before create_node().")
|
||
|
||
if data_id:
|
||
existing_node = self._collection.get_by_data_id(self.node_type, data_id)
|
||
if existing_node is not None:
|
||
existing_node.set_data(data.copy())
|
||
existing_node.set_origin_data(data.copy())
|
||
return existing_node
|
||
|
||
node_id = self._collection.generate_node_id()
|
||
|
||
return self.node_class(
|
||
node_id=node_id,
|
||
data_id=data_id,
|
||
data=data.copy()
|
||
)
|
||
|
||
def _create_node(self, data: Dict[str, Any]) -> "SyncNode[T]":
|
||
"""兼容 NodeHandler 协议的默认实现"""
|
||
return self.create_node(data)
|
||
|
||
|
||
class CompatibleNodeHandler(BaseNodeHandler[T]):
|
||
def __init__(self, handler: BaseNodeHandler[T]):
|
||
super().__init__(handler.node_type, handler.node_class, handler.schema)
|
||
self._handler = handler
|
||
|
||
def set_collection(self, collection: "DataCollection") -> None:
|
||
super().set_collection(collection)
|
||
self._handler.set_collection(collection)
|
||
|
||
def set_handler_config(self, config: Dict[str, Any]) -> None:
|
||
self._handler.set_handler_config(config)
|
||
self.handler_config = dict(self._handler.handler_config)
|
||
|
||
def set_api_client(self, client: "ApiClient") -> None:
|
||
self._handler.set_api_client(client)
|
||
|
||
def set_poll_mode(self, mode: str) -> None:
|
||
self._handler.set_poll_mode(mode)
|
||
|
||
def get_update_fields(self, *, exclude: frozenset[str] = frozenset({"id"})) -> List[str]:
|
||
return self._handler.get_update_fields(exclude=exclude)
|
||
|
||
async def load(self) -> List['SyncNode']:
|
||
return await self._handler.load()
|
||
|
||
async def create_all(
|
||
self,
|
||
nodes: List["SyncNode"]
|
||
) -> List[TaskResult]:
|
||
return await self._handler.create_all(nodes)
|
||
|
||
async def update_all(
|
||
self,
|
||
nodes: List["SyncNode"]
|
||
) -> List[TaskResult]:
|
||
return await self._handler.update_all(nodes)
|
||
|
||
async def delete_all(
|
||
self,
|
||
nodes: List["SyncNode"]
|
||
) -> List[TaskResult]:
|
||
return await self._handler.delete_all(nodes)
|
||
|
||
async def poll_tasks(self, task_ids: List[str]) -> Dict[str, TaskResult]:
|
||
return await self._handler.poll_tasks(task_ids)
|
||
|
||
async def validate(self, data: Dict[str, Any]) -> ValidationResult:
|
||
return await self._handler.validate(data)
|
||
|
||
def extract_id(self, data: Dict[str, Any]) -> str:
|
||
return self._handler.extract_id(data)
|
||
|
||
|
||
class NodeHandlerRegistry:
|
||
"""节点处理器注册表"""
|
||
|
||
_handlers: Dict[str, NodeHandler] = {}
|
||
|
||
@classmethod
|
||
def register(cls, node_type: str, handler: NodeHandler) -> None:
|
||
"""注册 handler"""
|
||
cls._handlers[node_type] = handler
|
||
|
||
@classmethod
|
||
def get(cls, node_type: str) -> NodeHandler:
|
||
"""获取 handler"""
|
||
if node_type not in cls._handlers:
|
||
raise KeyError(f"Handler for '{node_type}' not registered")
|
||
return cls._handlers[node_type]
|
||
|
||
@classmethod
|
||
def has(cls, node_type: str) -> bool:
|
||
"""检查是否已注册"""
|
||
return node_type in cls._handlers
|
||
|
||
@classmethod
|
||
def clear(cls) -> None:
|
||
"""清空注册表"""
|
||
cls._handlers.clear()
|
||
|
||
@classmethod
|
||
def list_handlers(cls) -> List[str]:
|
||
"""列出所有已注册的 node_type"""
|
||
return list(cls._handlers.keys())
|