first commit
This commit is contained in:
strepsiades
@@ -0,0 +1,538 @@
|
||||
"""
|
||||
BaseApiHandler - API Handler 基类
|
||||
|
||||
职责：
|
||||
- 实现 Handler 接口（load, create, update, delete, poll）
|
||||
- 使用 self.api_client 发送 HTTP 请求
|
||||
- 生成 push_id 和 biz_id（业务标识）
|
||||
- 从 collection 获取依赖节点数据
|
||||
- 管理节点的 context（存储 project_id, contract_id 等）
|
||||
- 实现 poll 方法，调用 api_client.get_push_logs 批量查询
|
||||
- 提取 created_id（因为不同业务的 id 字段名和位置可能不同）
|
||||
- 返回 TaskResult（SUCCESS/FAILED/IN_PROGRESS）
|
||||
"""
|
||||
|
||||
import uuid
|
||||
import asyncio
|
||||
from abc import abstractmethod
|
||||
from typing import ClassVar, Dict, List, Any, Optional, TYPE_CHECKING, TypeVar
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ...common.type_safety import get_pydantic_model_fields
|
||||
from ..handler import NodeHandler
|
||||
from ..task_result import TaskResult
|
||||
from ..handler import ValidationResult # Import ValidationResult
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ...common.sync_node import SyncNode
|
||||
from ...common.collection import DataCollection
|
||||
from .client import ApiClient
|
||||
|
||||
T = TypeVar("T", bound=BaseModel)
|
||||
|
||||
|
||||
class BaseApiHandler(NodeHandler[T]):
|
||||
"""API Handler 基类"""
|
||||
|
||||
# 子类需要设置这些属性
|
||||
_node_type: str
|
||||
_node_class: type['SyncNode']
|
||||
_schema: Any
|
||||
|
||||
def __init__(self):
|
||||
# 子类应该在调用 super().__init__() 之前或之后设置 _node_type, _node_class, _schema
|
||||
self.api_client: 'ApiClient' = None # type: ignore # 由 DataSource 注入
|
||||
self.poll_mode: str = "async" # async | sync
|
||||
self._collection: 'DataCollection' = None # type: ignore # 由 DataSource 注入
|
||||
# 子类在 __init__ 中设置此列表以声明 create/update 中涉及的 Pydantic schema 类；
|
||||
# post-check 将只比较这些 schema 覆盖的字段，过滤后端只读的派生字段。
|
||||
self.update_schemas: List[type] = []
|
||||
|
||||
def get_update_fields(self, *, exclude: frozenset = frozenset({'id'})) -> List[str]:
|
||||
"""从 update_schemas 中提取所有可写字段名，排除标识键（默认排除 id）。"""
|
||||
fields: set = set()
|
||||
for schema in self.update_schemas:
|
||||
for field_name in get_pydantic_model_fields(schema):
|
||||
if field_name not in exclude:
|
||||
fields.add(field_name)
|
||||
return sorted(fields)
|
||||
|
||||
def set_api_client(self, client: 'ApiClient') -> None:
|
||||
self.api_client = client
|
||||
|
||||
def set_collection(self, collection: 'DataCollection') -> None:
|
||||
"""设置 Collection（由 DataSource 调用）"""
|
||||
self._collection = collection
|
||||
|
||||
@property
|
||||
def node_type(self) -> str:
|
||||
"""返回节点类型"""
|
||||
return self._node_type
|
||||
|
||||
@property
|
||||
def node_class(self) -> type['SyncNode']:
|
||||
"""返回节点类"""
|
||||
return self._node_class
|
||||
|
||||
@property
|
||||
def schema(self) -> Any:
|
||||
"""返回 schema（子类可重写）"""
|
||||
return self._schema
|
||||
|
||||
def _generate_push_id(self) -> str:
|
||||
"""
|
||||
生成 push_id（推送唯一标识）
|
||||
|
||||
Returns:
|
||||
UUID 格式的 push_id
|
||||
"""
|
||||
return str(uuid.uuid4())
|
||||
|
||||
def _generate_biz_id(self, node: 'SyncNode') -> str:
|
||||
"""
|
||||
生成 biz_id（业务标识）
|
||||
|
||||
子类可重写此方法实现自定义业务 ID 生成逻辑。
|
||||
默认使用 node_id 作为 biz_id。
|
||||
|
||||
Args:
|
||||
node: 同步节点
|
||||
|
||||
Returns:
|
||||
业务标识符
|
||||
"""
|
||||
return node.node_id
|
||||
|
||||
def _create_basic_node(self, data: Dict[str, Any], depend_ids: Optional[List[str]] = None) -> "SyncNode":
|
||||
"""创建或复用基础节点（默认使用 data['id'] 或 data['_id']）"""
|
||||
data_id = str(data.get("id") or data.get("_id") or "")
|
||||
if self._collection is None:
|
||||
raise RuntimeError("Collection not set. Call set_collection() before creating nodes.")
|
||||
|
||||
if data_id:
|
||||
existing_node = self._collection.get_by_data_id(self.node_type, data_id)
|
||||
if existing_node is not None:
|
||||
existing_node.depend_ids = list(depend_ids or [])
|
||||
existing_node.set_data(data.copy())
|
||||
existing_node.set_origin_data(data.copy())
|
||||
return existing_node
|
||||
|
||||
node_id = self._collection.generate_node_id()
|
||||
node = self.node_class(
|
||||
node_id=node_id,
|
||||
data_id=data_id,
|
||||
data=data.copy(),
|
||||
depend_ids=depend_ids or [],
|
||||
origin_data=data.copy(),
|
||||
)
|
||||
return node
|
||||
|
||||
def _create_node(self, data: Dict[str, Any]) -> "SyncNode":
|
||||
"""默认节点创建（子类可覆盖并补充 context）"""
|
||||
return self._create_basic_node(data)
|
||||
|
||||
def extract_id(self, data: Dict[str, Any]) -> str:
|
||||
"""从原始数据提取 ID（默认优先 id，其次 _id）"""
|
||||
return str(data.get("id") or data.get("_id") or "")
|
||||
|
||||
def extract_created_id(self, push_log: Dict[str, Any]) -> Optional[str]:
|
||||
"""
|
||||
从 push_log 提取 created_id
|
||||
|
||||
子类可重写此方法以适配不同的响应结构。
|
||||
默认从 push_log['data']['biz_id'] 提取。
|
||||
|
||||
Args:
|
||||
push_log: Push Log 响应数据
|
||||
|
||||
Returns:
|
||||
创建的对象 ID，未找到返回 None
|
||||
"""
|
||||
try:
|
||||
data = push_log.get('data', {})
|
||||
if isinstance(data, dict):
|
||||
return data.get('biz_id')
|
||||
return None
|
||||
except (KeyError, AttributeError):
|
||||
return None
|
||||
|
||||
def set_poll_mode(self, mode: str) -> None:
|
||||
"""设置轮询模式（sync/async）"""
|
||||
if mode not in {"sync", "async"}:
|
||||
raise ValueError(f"Unsupported poll mode: {mode}")
|
||||
self.poll_mode = mode
|
||||
|
||||
def _extract_created_id_from_response(self, response: Dict[str, Any]) -> Optional[str]:
|
||||
"""从同步创建响应提取创建的 ID（默认 data.biz_id）"""
|
||||
data = response.get("data") if isinstance(response, dict) else None
|
||||
if isinstance(data, dict):
|
||||
return data.get("biz_id")
|
||||
return None
|
||||
|
||||
def _build_create_task_result(
|
||||
self,
|
||||
node_id: str,
|
||||
response: Optional[Dict[str, Any]] = None,
|
||||
task_id: Optional[str] = None,
|
||||
fallback_data_id: Optional[str] = None,
|
||||
) -> TaskResult:
|
||||
"""根据轮询模式构建 create 的 TaskResult"""
|
||||
if self.poll_mode == "sync":
|
||||
created_id = None
|
||||
if response is not None:
|
||||
created_id = self._extract_created_id_from_response(response)
|
||||
if not created_id:
|
||||
created_id = fallback_data_id
|
||||
return TaskResult.success(node_id=node_id, data_id=created_id)
|
||||
|
||||
if not task_id:
|
||||
return TaskResult.failed(node_id=node_id, error="Missing task_id for async create")
|
||||
return TaskResult.in_progress(node_id=node_id, task_id=task_id)
|
||||
|
||||
# ========== 通用辅助方法 ==========
|
||||
|
||||
def _validate_schema(self, data: Dict[str, Any], schema: Any) -> Optional[str]:
|
||||
"""
|
||||
验证数据是否符合 schema
|
||||
|
||||
Args:
|
||||
data: 待验证数据
|
||||
schema: Pydantic schema
|
||||
|
||||
Returns:
|
||||
错误信息，无错误返回 None
|
||||
"""
|
||||
try:
|
||||
schema.model_validate(data)
|
||||
return None
|
||||
except Exception as e:
|
||||
return str(e)
|
||||
|
||||
def _get_schema_diff(
|
||||
self,
|
||||
schema,
|
||||
data: Dict[str, Any],
|
||||
origin_data: Dict[str, Any],
|
||||
node_id: Optional[str] = None
|
||||
) -> Dict[str, Any] | None:
|
||||
"""
|
||||
获取数据在指定 schema 下的差异字段
|
||||
|
||||
Args:
|
||||
schema: Pydantic schema 类
|
||||
data: 新数据
|
||||
origin_data: 原始数据
|
||||
node_id: 节点ID（用于日志）
|
||||
|
||||
Returns:
|
||||
Dict[str, Any] | None: 有差异返回差异字段字典，无差异或字段不全返回 None
|
||||
"""
|
||||
from pydantic import ValidationError
|
||||
|
||||
try:
|
||||
new_model = schema.model_validate(data)
|
||||
origin_model = schema.model_validate(origin_data)
|
||||
|
||||
new_dict = new_model.model_dump(exclude_unset=True)
|
||||
origin_dict = origin_model.model_dump(exclude_unset=True)
|
||||
|
||||
if new_dict == origin_dict:
|
||||
return None
|
||||
|
||||
# 返回有差异的字段
|
||||
diff_fields = {k: v for k, v in new_dict.items() if origin_dict.get(k) != v}
|
||||
|
||||
if diff_fields:
|
||||
# 打印差异详情
|
||||
node_info = f"[{node_id}]" if node_id else ""
|
||||
print(f" [DIFF] {self.node_type}{node_info} schema={schema.__name__}")
|
||||
for k, v in diff_fields.items():
|
||||
orig_v = origin_dict.get(k)
|
||||
print(f" • {k}: {orig_v!r} → {v!r}")
|
||||
|
||||
return diff_fields if diff_fields else None
|
||||
|
||||
except ValidationError:
|
||||
# schema 字段不全，视为无差异
|
||||
return None
|
||||
|
||||
def _filter_update_data(
|
||||
self,
|
||||
new_data: Dict[str, Any],
|
||||
original_data: Dict[str, Any],
|
||||
schema: Optional[Any] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
过滤出需要更新的字段
|
||||
|
||||
规则：
|
||||
1. 提取 schema 定义的所有字段（包括必填）
|
||||
2. 检查是否有任何字段发生变化
|
||||
3. 如果有变化，返回所有 schema 字段；否则返回空
|
||||
|
||||
Args:
|
||||
new_data: 新数据
|
||||
original_data: 原始数据
|
||||
schema: 更新 schema（可选）
|
||||
|
||||
Returns:
|
||||
包含 schema 所有字段的数据（如果有变化），否则返回空字典
|
||||
"""
|
||||
schema_fields = get_pydantic_model_fields(schema)
|
||||
if not schema_fields:
|
||||
# 无 schema，按原逻辑处理
|
||||
update_data = {}
|
||||
for key, new_value in new_data.items():
|
||||
if new_value == "" or new_value is None:
|
||||
continue
|
||||
original_value = original_data.get(key)
|
||||
if new_value != original_value:
|
||||
update_data[key] = new_value
|
||||
return update_data
|
||||
|
||||
# 1. 提取 schema 定义的所有字段
|
||||
schema_fields = set(schema_fields.keys())
|
||||
schema_data = {}
|
||||
has_change = False
|
||||
|
||||
for key in schema_fields:
|
||||
new_value = new_data.get(key)
|
||||
|
||||
# 排除空值
|
||||
if new_value == "" or new_value is None:
|
||||
continue
|
||||
|
||||
schema_data[key] = new_value
|
||||
|
||||
# 检查是否变化
|
||||
original_value = original_data.get(key)
|
||||
if new_value != original_value:
|
||||
# 记录具体差异以便排查
|
||||
# print(f" [DEBUG] Field '{key}' changed: '{original_value}' -> '{new_value}'")
|
||||
has_change = True
|
||||
|
||||
# 2. 只有存在变化时才返回数据
|
||||
return schema_data if has_change else {}
|
||||
|
||||
def _schema_changed_fields(
|
||||
self,
|
||||
schema: Any,
|
||||
new_data: Dict[str, Any],
|
||||
origin_data: Dict[str, Any],
|
||||
) -> List[str]:
|
||||
schema_fields = get_pydantic_model_fields(schema)
|
||||
if not schema_fields:
|
||||
return []
|
||||
changed: List[str] = []
|
||||
for field_name in schema_fields.keys():
|
||||
if new_data.get(field_name) != origin_data.get(field_name):
|
||||
changed.append(field_name)
|
||||
return changed
|
||||
|
||||
def _build_update_schema_status(
|
||||
self,
|
||||
*,
|
||||
schema_name: str,
|
||||
schema: Any,
|
||||
new_data: Dict[str, Any],
|
||||
origin_data: Dict[str, Any],
|
||||
will_update: bool,
|
||||
update_error: Optional[str] = None,
|
||||
) -> str:
|
||||
new_error = self._validate_schema(new_data, schema)
|
||||
origin_error = self._validate_schema(origin_data, schema)
|
||||
changed_fields = self._schema_changed_fields(schema, new_data, origin_data)
|
||||
|
||||
if new_error:
|
||||
compact = new_error.replace("\n", " ").strip()
|
||||
if len(compact) > 220:
|
||||
compact = compact[:220] + "..."
|
||||
return f"{schema_name}: NEW_INVALID({compact})"
|
||||
|
||||
if not changed_fields:
|
||||
return f"{schema_name}: CONSISTENT"
|
||||
|
||||
changed_text = ",".join(changed_fields)
|
||||
if update_error:
|
||||
compact = update_error.replace("\n", " ").strip()
|
||||
if len(compact) > 220:
|
||||
compact = compact[:220] + "..."
|
||||
return f"{schema_name}: DIFF[{changed_text}] -> UPDATE_FAILED({compact})"
|
||||
|
||||
if will_update:
|
||||
if origin_error:
|
||||
compact = origin_error.replace("\n", " ").strip()
|
||||
if len(compact) > 160:
|
||||
compact = compact[:160] + "..."
|
||||
return f"{schema_name}: DIFF[{changed_text}] -> UPDATE_TRIGGERED (origin_invalid={compact})"
|
||||
return f"{schema_name}: DIFF[{changed_text}] -> UPDATE_TRIGGERED"
|
||||
|
||||
if origin_error:
|
||||
compact = origin_error.replace("\n", " ").strip()
|
||||
if len(compact) > 160:
|
||||
compact = compact[:160] + "..."
|
||||
return f"{schema_name}: DIFF[{changed_text}] -> NOT_TRIGGERED (origin_invalid={compact})"
|
||||
return f"{schema_name}: DIFF[{changed_text}] -> NOT_TRIGGERED"
|
||||
|
||||
async def _execute_with_retry(
|
||||
self,
|
||||
operation,
|
||||
max_retries: int = 3,
|
||||
retry_delay: float = 1.0
|
||||
) -> Any:
|
||||
"""
|
||||
通用重试逻辑
|
||||
|
||||
子类可重写此方法实现自定义重试策略。
|
||||
默认策略：网络错误/5xx 重试，4xx 不重试。
|
||||
|
||||
Args:
|
||||
operation: 异步操作（协程函数）
|
||||
max_retries: 最大重试次数
|
||||
retry_delay: 重试延迟（秒）
|
||||
|
||||
Returns:
|
||||
操作执行结果
|
||||
|
||||
Raises:
|
||||
最后一次重试的异常
|
||||
"""
|
||||
import httpx
|
||||
|
||||
last_error = None
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
return await operation()
|
||||
except httpx.HTTPStatusError as e:
|
||||
# 4xx 客户端错误不重试
|
||||
if 400 <= e.response.status_code < 500:
|
||||
raise
|
||||
last_error = e
|
||||
if attempt < max_retries - 1:
|
||||
await asyncio.sleep(retry_delay * (attempt + 1))
|
||||
except (httpx.NetworkError, httpx.TimeoutException) as e:
|
||||
# 网络错误重试
|
||||
last_error = e
|
||||
if attempt < max_retries - 1:
|
||||
await asyncio.sleep(retry_delay * (attempt + 1))
|
||||
|
||||
# 重试次数用尽
|
||||
if last_error:
|
||||
raise last_error
|
||||
else:
|
||||
raise RuntimeError("Retry failed with unknown error")
|
||||
|
||||
# ========== 批量接口（子类必须实现） ==========
|
||||
|
||||
@abstractmethod
|
||||
async def create_all(self, nodes: List['SyncNode']) -> List[TaskResult]:
|
||||
"""
|
||||
批量创建节点
|
||||
|
||||
Args:
|
||||
nodes: 需要创建的节点列表（DataSource 已筛选为 CREATE + PENDING）
|
||||
|
||||
Returns:
|
||||
List[TaskResult]，每个结果关联 node_id
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def update_all(self, nodes: List['SyncNode']) -> List[TaskResult]:
|
||||
"""
|
||||
批量更新节点
|
||||
|
||||
Args:
|
||||
nodes: 需要更新的节点列表（DataSource 已筛选为 UPDATE + PENDING）
|
||||
|
||||
Returns:
|
||||
List[TaskResult]，每个结果关联 node_id
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def delete_all(self, nodes: List['SyncNode']) -> List[TaskResult]:
|
||||
"""
|
||||
批量删除节点
|
||||
|
||||
Args:
|
||||
nodes: 需要删除的节点列表（DataSource 已筛选为 DELETE + PENDING）
|
||||
|
||||
Returns:
|
||||
List[TaskResult]，每个结果关联 node_id
|
||||
"""
|
||||
pass
|
||||
|
||||
async def poll_tasks(self, task_ids: List[str]) -> Dict[str, TaskResult]:
|
||||
"""
|
||||
批量轮询异步任务状态
|
||||
|
||||
Args:
|
||||
task_ids: 任务 ID 列表（push_id 列表）
|
||||
|
||||
Returns:
|
||||
{task_id: TaskResult} 映射
|
||||
"""
|
||||
if not task_ids:
|
||||
return {}
|
||||
if not self.api_client:
|
||||
raise RuntimeError("api_client not injected")
|
||||
|
||||
# 批量查询 push_logs
|
||||
try:
|
||||
push_logs = await self.api_client.get_push_logs(task_ids)
|
||||
except Exception as e:
|
||||
# 查询失败，所有任务标记为失败
|
||||
return {
|
||||
task_id: TaskResult.failed(error=f"Poll failed: {str(e)}")
|
||||
for task_id in task_ids
|
||||
}
|
||||
|
||||
results = {}
|
||||
for task_id in task_ids:
|
||||
push_log = push_logs.get(task_id, {})
|
||||
|
||||
if 'error' in push_log:
|
||||
results[task_id] = TaskResult.failed(error=push_log['error'])
|
||||
elif push_log.get('code') == 200: # 通过code==200判断成功
|
||||
created_id = self.extract_created_id(push_log)
|
||||
results[task_id] = TaskResult.success(data_id=created_id)
|
||||
elif push_log.get('code') and push_log.get('code') != 200: # code存在但不是200表示失败
|
||||
results[task_id] = TaskResult.failed(error=push_log.get('message', 'Unknown error'))
|
||||
else:
|
||||
# 仍在进行中
|
||||
results[task_id] = TaskResult.in_progress(task_id=task_id)
|
||||
|
||||
return results
|
||||
|
||||
async def validate(self, data: Dict[str, Any]) -> ValidationResult:
|
||||
"""
|
||||
数据质量检查（使用 schema 验证）
|
||||
|
||||
Args:
|
||||
data: 原始数据
|
||||
|
||||
Returns:
|
||||
ValidationResult（是否有效 + 错误列表）
|
||||
"""
|
||||
error = self._validate_schema(data, self.schema)
|
||||
if error:
|
||||
return ValidationResult(is_valid=False, errors=[error])
|
||||
return ValidationResult(is_valid=True)
|
||||
|
||||
# ========== 抽象方法：子类必须实现 ==========
|
||||
|
||||
@abstractmethod
|
||||
async def load(self) -> List['SyncNode']:
|
||||
"""
|
||||
从 API 加载数据并创建节点
|
||||
|
||||
Handler 负责：
|
||||
1. 加载原始数据（可使用 self._collection 查询依赖）
|
||||
2. 创建 SyncNode（调用 self.create_node）
|
||||
3. 设置 context（如 project_id, contract_id 等）
|
||||
|
||||
Returns:
|
||||
SyncNode 列表
|
||||
"""
|
||||
pass
|
||||
Reference in New Issue
Block a user