strepsiades
68 lines
2.1 KiB
Python
68 lines
2.1 KiB
Python
"""
|
||
JsonlDataSource - JSONL 文件数据源
|
||
|
||
职责：
|
||
- 提供 JSONL 文件目录管理
|
||
- 实现 DataSource 接口（load_all, sync_all）
|
||
- 管理内存缓存和文件写入
|
||
- 统一异常处理和状态管理
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from pathlib import Path
|
||
from typing import Dict, Any, Set
|
||
|
||
from ..datasource import BaseDataSource
|
||
|
||
|
||
|
||
|
||
class JsonlDataSource(BaseDataSource):
|
||
"""
|
||
JSONL 文件数据源
|
||
|
||
提供文件路径和基础功能，具体的文件查找和读取由各个 Handler 负责。
|
||
|
||
Args:
|
||
dir_path: JSONL 文件目录
|
||
read_only: 是否只读模式
|
||
|
||
示例：
|
||
datasource = JsonlDataSource(Path("filtered_datasource/datasource/ecm-2025-12-02"))
|
||
datasource.register_handler(ContractJsonlHandler(datasource))
|
||
|
||
collection = DataCollection()
|
||
datasource.set_collection(collection)
|
||
await datasource.load_all(order=["project", "contract"])
|
||
await datasource.sync_all(order=["project", "contract"])
|
||
"""
|
||
|
||
def __init__(self, dir_path: Path, read_only: bool = False):
|
||
super().__init__()
|
||
self.dir_path = Path(dir_path)
|
||
self.read_only = read_only
|
||
|
||
# 用于 create/update/delete 的内存缓存
|
||
self._data: Dict[str, Dict[str, Dict[str, Any]]] = {} # {node_type: {id: data}}
|
||
self._dirty_types: Set[str] = set()
|
||
|
||
def _on_node_loaded(self, handler, node) -> None:
|
||
node.append_log(
|
||
f"JSONL链路[load] dir={self.dir_path.as_posix()} node_type={node.node_type} data_id={node.data_id or '<empty>'}"
|
||
)
|
||
|
||
def _on_in_progress_result(self, handler, node, result) -> None:
|
||
node.append_log(
|
||
f"JSONL链路[submit] action={node.action.value} data_id={node.data_id or '<empty>'}"
|
||
)
|
||
|
||
def _on_poll_result(self, handler, node, task_id: str, result) -> None:
|
||
node.append_log(
|
||
f"JSONL链路[poll] task_id={task_id} status={result.status.value}"
|
||
)
|
||
|
||
# load_all 和 sync_all 继承自 BaseDataSource
|
||
# 保存逻辑在 BaseJsonlHandler 中实现
|
||
|