整理配置和代码入口,增加部分测试。
This commit is contained in:
@@ -17,6 +17,9 @@ from .task_result import TaskResult, TaskStatus
|
||||
# API DataSource
|
||||
from .api import ApiClient, ApiDataSource, BaseApiHandler
|
||||
|
||||
# DB DataSource
|
||||
from .db import DbDataSource, BaseDbHandler
|
||||
|
||||
# JSONL DataSource
|
||||
from .jsonl import JsonlDataSource, BaseJsonlHandler
|
||||
|
||||
@@ -34,6 +37,10 @@ __all__ = [
|
||||
"ApiClient",
|
||||
"ApiDataSource",
|
||||
"BaseApiHandler",
|
||||
|
||||
# DB DataSource
|
||||
"DbDataSource",
|
||||
"BaseDbHandler",
|
||||
|
||||
# JSONL DataSource
|
||||
"JsonlDataSource",
|
||||
|
||||
@@ -44,6 +44,7 @@ class BaseApiHandler(NodeHandler[T]):
|
||||
self.api_client: 'ApiClient' = None # type: ignore # 由 DataSource 注入
|
||||
self.poll_mode: str = "async" # async | sync
|
||||
self._collection: 'DataCollection' = None # type: ignore # 由 DataSource 注入
|
||||
self.handler_config: Dict[str, Any] = {}
|
||||
# 子类在 __init__ 中设置此列表以声明 create/update 中涉及的 Pydantic schema 类;
|
||||
# post-check 将只比较这些 schema 覆盖的字段,过滤后端只读的派生字段。
|
||||
self.update_schemas: List[type] = []
|
||||
@@ -63,6 +64,13 @@ class BaseApiHandler(NodeHandler[T]):
|
||||
def set_collection(self, collection: 'DataCollection') -> None:
|
||||
"""设置 Collection(由 DataSource 调用)"""
|
||||
self._collection = collection
|
||||
|
||||
def set_handler_config(self, config: Dict[str, Any]) -> None:
|
||||
self.handler_config = dict(config)
|
||||
|
||||
def set_target_project_ids(self, target_project_ids: List[str]) -> None:
|
||||
"""默认忽略项目过滤列表,具体子类按需覆盖。"""
|
||||
return
|
||||
|
||||
@property
|
||||
def node_type(self) -> str:
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
"""DB DataSource 模块。"""
|
||||
|
||||
from .datasource import DbDataSource
|
||||
from .handler import BaseDbHandler
|
||||
|
||||
__all__ = ["DbDataSource", "BaseDbHandler"]
|
||||
@@ -0,0 +1,22 @@
|
||||
"""DbDataSource - 数据库数据源协调层。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from ..datasource import BaseDataSource
|
||||
|
||||
|
||||
class DbDataSource(BaseDataSource):
|
||||
"""基于数据库 Handler 的数据源实现。"""
|
||||
|
||||
async def close(self) -> None:
|
||||
return None
|
||||
|
||||
def _on_node_loaded(self, handler, node) -> None:
|
||||
node.append_log(
|
||||
f"DB链路[load] node_type={node.node_type} data_id={node.data_id or '<empty>'}"
|
||||
)
|
||||
|
||||
def _on_in_progress_result(self, handler, node, result) -> None:
|
||||
node.append_log(
|
||||
f"DB链路[submit] action={node.action.value} data_id={node.data_id or '<empty>'}"
|
||||
)
|
||||
@@ -0,0 +1,68 @@
|
||||
"""BaseDbHandler - DB Handler 基类。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional, Set, TypeVar, TYPE_CHECKING
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ...common.type_safety import get_pydantic_model_fields
|
||||
from ..handler import BaseNodeHandler
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ...common.sync_node import SyncNode
|
||||
from .datasource import DbDataSource
|
||||
|
||||
T = TypeVar("T", bound=BaseModel)
|
||||
|
||||
|
||||
class BaseDbHandler(BaseNodeHandler[T]):
|
||||
"""数据库 Handler 基类。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
datasource: "DbDataSource",
|
||||
node_type: str,
|
||||
node_class: type["SyncNode[T]"],
|
||||
schema: type[T],
|
||||
):
|
||||
super().__init__(node_type, node_class, schema)
|
||||
self.datasource = datasource
|
||||
self.update_schemas: List[type] = []
|
||||
self._target_project_ids: Optional[Set[str]] = None
|
||||
|
||||
def set_target_project_ids(self, target_project_ids: List[str]) -> None:
|
||||
if target_project_ids:
|
||||
self._target_project_ids = {str(item) for item in target_project_ids if str(item)}
|
||||
else:
|
||||
self._target_project_ids = None
|
||||
|
||||
def get_update_fields(self, *, exclude: frozenset = frozenset({"id"})) -> List[str]:
|
||||
fields: set[str] = set()
|
||||
for schema in self.update_schemas:
|
||||
for field_name in get_pydantic_model_fields(schema):
|
||||
if field_name not in exclude:
|
||||
fields.add(field_name)
|
||||
return sorted(fields)
|
||||
|
||||
def _create_basic_node(self, data: Dict[str, Any], depend_ids: Optional[List[str]] = None) -> "SyncNode[T]":
|
||||
data_id = self.extract_id(data)
|
||||
if self._collection is None:
|
||||
raise RuntimeError("Collection not set. Call set_collection() before creating nodes.")
|
||||
|
||||
if data_id:
|
||||
existing_node = self._collection.get_by_data_id(self.node_type, data_id)
|
||||
if existing_node is not None:
|
||||
existing_node.depend_ids = list(depend_ids or [])
|
||||
existing_node.set_data(data.copy())
|
||||
existing_node.set_origin_data(data.copy())
|
||||
return existing_node
|
||||
|
||||
node_id = self._collection.generate_node_id()
|
||||
return self.node_class(
|
||||
node_id=node_id,
|
||||
data_id=data_id,
|
||||
data=data.copy(),
|
||||
depend_ids=depend_ids or [],
|
||||
origin_data=data.copy(),
|
||||
)
|
||||
@@ -70,6 +70,10 @@ class NodeHandler(Protocol[T]):
|
||||
"""
|
||||
...
|
||||
|
||||
def set_handler_config(self, config: Dict[str, Any]) -> None:
|
||||
"""注入 handler 级别配置。默认可忽略。"""
|
||||
...
|
||||
|
||||
def set_target_project_ids(self, target_project_ids: List[str]) -> None:
|
||||
"""注入项目过滤列表。默认可忽略。"""
|
||||
...
|
||||
@@ -345,6 +349,7 @@ class BaseNodeHandler(ABC, Generic[T]):
|
||||
self._node_class = node_class
|
||||
self._schema = schema
|
||||
self._collection: Optional["DataCollection"] = None
|
||||
self.handler_config: Dict[str, Any] = {}
|
||||
|
||||
def set_collection(self, collection: "DataCollection") -> None:
|
||||
"""
|
||||
@@ -359,6 +364,10 @@ class BaseNodeHandler(ABC, Generic[T]):
|
||||
"""默认忽略项目过滤能力。"""
|
||||
return
|
||||
|
||||
def set_handler_config(self, config: Dict[str, Any]) -> None:
|
||||
"""默认保存 handler 级别配置,供子类按需读取。"""
|
||||
self.handler_config = dict(config)
|
||||
|
||||
def set_api_client(self, client: "ApiClient") -> None:
|
||||
"""默认忽略 API 客户端注入。"""
|
||||
return
|
||||
@@ -491,6 +500,11 @@ class CompatibleNodeHandler(Generic[T]):
|
||||
def set_collection(self, collection: "DataCollection") -> None:
|
||||
self._handler.set_collection(collection)
|
||||
|
||||
def set_handler_config(self, config: Dict[str, Any]) -> None:
|
||||
method = getattr(self._handler, "set_handler_config", None)
|
||||
if callable(method):
|
||||
method(config)
|
||||
|
||||
def set_target_project_ids(self, target_project_ids: List[str]) -> None:
|
||||
method = getattr(self._handler, "set_target_project_ids", None)
|
||||
if callable(method):
|
||||
|
||||
Reference in New Issue
Block a user