整理配置和代码入口,增加部分测试。

This commit is contained in:
strepsiades
2026-03-10 12:22:54 +08:00
parent 515c3d9388
commit a638e4203b
35 changed files with 1314 additions and 199 deletions
@@ -17,6 +17,9 @@ from .task_result import TaskResult, TaskStatus
# API DataSource
from .api import ApiClient, ApiDataSource, BaseApiHandler
# DB DataSource
from .db import DbDataSource, BaseDbHandler
# JSONL DataSource
from .jsonl import JsonlDataSource, BaseJsonlHandler
@@ -34,6 +37,10 @@ __all__ = [
"ApiClient",
"ApiDataSource",
"BaseApiHandler",
# DB DataSource
"DbDataSource",
"BaseDbHandler",
# JSONL DataSource
"JsonlDataSource",
@@ -44,6 +44,7 @@ class BaseApiHandler(NodeHandler[T]):
self.api_client: 'ApiClient' = None # type: ignore # 由 DataSource 注入
self.poll_mode: str = "async" # async | sync
self._collection: 'DataCollection' = None # type: ignore # 由 DataSource 注入
self.handler_config: Dict[str, Any] = {}
# 子类在 __init__ 中设置此列表以声明 create/update 中涉及的 Pydantic schema 类;
# post-check 将只比较这些 schema 覆盖的字段,过滤后端只读的派生字段。
self.update_schemas: List[type] = []
@@ -63,6 +64,13 @@ class BaseApiHandler(NodeHandler[T]):
def set_collection(self, collection: 'DataCollection') -> None:
"""设置 Collection(由 DataSource 调用)"""
self._collection = collection
def set_handler_config(self, config: Dict[str, Any]) -> None:
self.handler_config = dict(config)
def set_target_project_ids(self, target_project_ids: List[str]) -> None:
"""默认忽略项目过滤列表,具体子类按需覆盖。"""
return
@property
def node_type(self) -> str:
@@ -0,0 +1,6 @@
"""DB DataSource 模块。"""
from .datasource import DbDataSource
from .handler import BaseDbHandler
__all__ = ["DbDataSource", "BaseDbHandler"]
@@ -0,0 +1,22 @@
"""DbDataSource - 数据库数据源协调层。"""
from __future__ import annotations
from ..datasource import BaseDataSource
class DbDataSource(BaseDataSource):
"""基于数据库 Handler 的数据源实现。"""
async def close(self) -> None:
return None
def _on_node_loaded(self, handler, node) -> None:
node.append_log(
f"DB链路[load] node_type={node.node_type} data_id={node.data_id or '<empty>'}"
)
def _on_in_progress_result(self, handler, node, result) -> None:
node.append_log(
f"DB链路[submit] action={node.action.value} data_id={node.data_id or '<empty>'}"
)
@@ -0,0 +1,68 @@
"""BaseDbHandler - DB Handler 基类。"""
from __future__ import annotations
from typing import Any, Dict, List, Optional, Set, TypeVar, TYPE_CHECKING
from pydantic import BaseModel
from ...common.type_safety import get_pydantic_model_fields
from ..handler import BaseNodeHandler
if TYPE_CHECKING:
from ...common.sync_node import SyncNode
from .datasource import DbDataSource
T = TypeVar("T", bound=BaseModel)
class BaseDbHandler(BaseNodeHandler[T]):
"""数据库 Handler 基类。"""
def __init__(
self,
datasource: "DbDataSource",
node_type: str,
node_class: type["SyncNode[T]"],
schema: type[T],
):
super().__init__(node_type, node_class, schema)
self.datasource = datasource
self.update_schemas: List[type] = []
self._target_project_ids: Optional[Set[str]] = None
def set_target_project_ids(self, target_project_ids: List[str]) -> None:
if target_project_ids:
self._target_project_ids = {str(item) for item in target_project_ids if str(item)}
else:
self._target_project_ids = None
def get_update_fields(self, *, exclude: frozenset = frozenset({"id"})) -> List[str]:
fields: set[str] = set()
for schema in self.update_schemas:
for field_name in get_pydantic_model_fields(schema):
if field_name not in exclude:
fields.add(field_name)
return sorted(fields)
def _create_basic_node(self, data: Dict[str, Any], depend_ids: Optional[List[str]] = None) -> "SyncNode[T]":
data_id = self.extract_id(data)
if self._collection is None:
raise RuntimeError("Collection not set. Call set_collection() before creating nodes.")
if data_id:
existing_node = self._collection.get_by_data_id(self.node_type, data_id)
if existing_node is not None:
existing_node.depend_ids = list(depend_ids or [])
existing_node.set_data(data.copy())
existing_node.set_origin_data(data.copy())
return existing_node
node_id = self._collection.generate_node_id()
return self.node_class(
node_id=node_id,
data_id=data_id,
data=data.copy(),
depend_ids=depend_ids or [],
origin_data=data.copy(),
)
+14
View File
@@ -70,6 +70,10 @@ class NodeHandler(Protocol[T]):
"""
...
def set_handler_config(self, config: Dict[str, Any]) -> None:
"""注入 handler 级别配置。默认可忽略。"""
...
def set_target_project_ids(self, target_project_ids: List[str]) -> None:
"""注入项目过滤列表。默认可忽略。"""
...
@@ -345,6 +349,7 @@ class BaseNodeHandler(ABC, Generic[T]):
self._node_class = node_class
self._schema = schema
self._collection: Optional["DataCollection"] = None
self.handler_config: Dict[str, Any] = {}
def set_collection(self, collection: "DataCollection") -> None:
"""
@@ -359,6 +364,10 @@ class BaseNodeHandler(ABC, Generic[T]):
"""默认忽略项目过滤能力。"""
return
def set_handler_config(self, config: Dict[str, Any]) -> None:
"""默认保存 handler 级别配置,供子类按需读取。"""
self.handler_config = dict(config)
def set_api_client(self, client: "ApiClient") -> None:
"""默认忽略 API 客户端注入。"""
return
@@ -491,6 +500,11 @@ class CompatibleNodeHandler(Generic[T]):
def set_collection(self, collection: "DataCollection") -> None:
self._handler.set_collection(collection)
def set_handler_config(self, config: Dict[str, Any]) -> None:
method = getattr(self._handler, "set_handler_config", None)
if callable(method):
method(config)
def set_target_project_ids(self, target_project_ids: List[str]) -> None:
method = getattr(self._handler, "set_target_project_ids", None)
if callable(method):