整理了初始化过程,现在更容易被继承了。

This commit is contained in:
strepsiades
2026-03-18 16:48:53 +08:00
parent 84b3b7652a
commit 841986740c
22 changed files with 432 additions and 277 deletions
@@ -1,13 +1,53 @@
from __future__ import annotations
"""
JSONL DataSource 模块
提供基于 JSONL 文件的数据源实现,用于测试和本地开发。
"""
from pathlib import Path
from typing import TYPE_CHECKING
from ..type_registry import register_datasource_type
from .datasource import JsonlDataSource
from .handler import BaseJsonlHandler
from .path_utils import prepare_remote_jsonl_dir
if TYPE_CHECKING:
from ...config.datasource_config import JsonlDataSourceConfig
def build_jsonl_datasource(
config: JsonlDataSourceConfig,
endpoint_name: str,
*,
project_root: Path | None = None,
):
dir_path = Path(config.jsonl_dir)
if endpoint_name == "remote":
dir_path = prepare_remote_jsonl_dir(dir_path, project_root=project_root or Path.cwd())
elif not dir_path.exists() or not dir_path.is_dir():
raise FileNotFoundError(
f"{endpoint_name}_datasource.jsonl_dir does not exist or is not a directory: {dir_path}"
)
return JsonlDataSource(dir_path, read_only=config.read_only)
def register(*, replace: bool = False) -> None:
from ...config.datasource_config import JsonlDataSourceConfig
register_datasource_type(
"jsonl",
config_model=JsonlDataSourceConfig,
factory=build_jsonl_datasource,
replace=replace,
)
__all__ = [
"JsonlDataSource",
"BaseJsonlHandler",
"build_jsonl_datasource",
"prepare_remote_jsonl_dir",
"register",
]
@@ -50,24 +50,22 @@ class BaseJsonlHandler(BaseNodeHandler):
):
super().__init__(node_type, node_class, schema)
self.datasource = datasource
self._target_project_ids: Optional[Set[str]] = None
self._target_project_ids: List[str] = []
def set_target_project_ids(self, target_project_ids: List[str]) -> None:
"""注入项目过滤列表,供 load 阶段预过滤使用"""
if target_project_ids:
self._target_project_ids = {str(pid) for pid in target_project_ids if str(pid)}
else:
self._target_project_ids = None
"""注入项目过滤列表的本地副本。真正的过滤来源仍然是 datasource"""
self._target_project_ids = [str(pid) for pid in target_project_ids if str(pid)]
def _should_skip_item_by_project_filter(self, item: Dict[str, Any], item_id: str) -> bool:
if not self._target_project_ids:
target_project_ids = set(self.datasource.target_project_ids)
if not target_project_ids:
return False
if self.node_type == "project":
return item_id not in self._target_project_ids
return item_id not in target_project_ids
project_id = item.get("project_id")
if project_id:
return str(project_id) not in self._target_project_ids
return str(project_id) not in target_project_ids
return False
async def load(self) -> List['SyncNode']:
@@ -0,0 +1,31 @@
from __future__ import annotations
import logging
from pathlib import Path
logger = logging.getLogger(__name__)
def prepare_remote_jsonl_dir(remote_dir: Path, *, project_root: Path) -> Path:
project_root_path = project_root.resolve()
resolved_remote_dir = remote_dir.resolve(strict=False)
if resolved_remote_dir.exists():
if not resolved_remote_dir.is_dir():
raise FileNotFoundError(
f"remote_datasource.jsonl_dir exists but is not a directory: {resolved_remote_dir}"
)
return resolved_remote_dir
try:
resolved_remote_dir.relative_to(project_root_path)
except ValueError as exc:
raise FileNotFoundError(
"remote_datasource.jsonl_dir does not exist and auto-create is only allowed "
f"under project root: {resolved_remote_dir} (project_root={project_root_path})"
) from exc
resolved_remote_dir.mkdir(parents=True, exist_ok=True)
logger.info("Created missing remote JSONL directory: %s", resolved_remote_dir)
return resolved_remote_dir