调整了pipeline执行顺序。现在按domain顺序读取数据源数据,防止前面domain的更新影响后面domain.

将target_project_ids替换为更通用的data_id_filter
This commit is contained in:
strepsiades
2026-03-20 14:38:56 +08:00
parent f9f175ee79
commit 2c09c61165
29 changed files with 296 additions and 172 deletions
+12 -20
View File
@@ -277,9 +277,6 @@ async def create_pipeline_from_config(
project_root=_project_root(),
)
await _initialize_datasource(remote_ds, endpoint_name="remote")
local_ds.set_target_project_ids([str(pid) for pid in config.local_datasource.target_project_ids if str(pid)])
remote_ds.set_target_project_ids([str(pid) for pid in config.remote_datasource.target_project_ids if str(pid)])
_register_handlers(config, local_ds, remote_ds, all_types)
local_collection = DataCollection("local", persistence, auto_persist=False)
@@ -434,8 +431,6 @@ async def create_jsonl_to_api_pipeline(
api_secret: str,
# 节点类型
node_types: List[str],
# 项目过滤
target_project_ids: Optional[List[str]] = None,
strategy_overrides: Optional[Dict[str, Dict[str, Any]]] = None,
# Pipeline 行为
enable_persistence: bool = True,
@@ -446,8 +441,8 @@ async def create_jsonl_to_api_pipeline(
poll_interval: float = 0.5,
api_rate_limit_max_requests: int = 30,
api_rate_limit_window_seconds: float = 10.0,
# Handler 自定义配置(可选,作用于 API 端)
handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
local_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
remote_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
# Logger 配置
initialize_logger: bool = True,
logger_file_path: Optional[str] = None,
@@ -459,7 +454,7 @@ async def create_jsonl_to_api_pipeline(
type="jsonl",
jsonl_dir=local_jsonl_dir,
read_only=False,
target_project_ids=target_project_ids or [],
handler_configs=local_handler_configs or {},
),
remote_datasource=ApiDataSourceConfig(
type="api",
@@ -471,8 +466,7 @@ async def create_jsonl_to_api_pipeline(
poll_interval=poll_interval,
api_rate_limit_max_requests=api_rate_limit_max_requests,
api_rate_limit_window_seconds=api_rate_limit_window_seconds,
target_project_ids=target_project_ids or [],
handler_configs=handler_configs or {},
handler_configs=remote_handler_configs or {},
),
strategies=[],
persist=PersistConfig(
@@ -502,8 +496,6 @@ async def create_api_to_jsonl_pipeline(
persistence_db: str,
# 节点类型
node_types: List[str],
# 项目过滤
target_project_ids: Optional[List[str]] = None,
strategy_overrides: Optional[Dict[str, Dict[str, Any]]] = None,
# Pipeline 行为
enable_persistence: bool = True,
@@ -514,8 +506,8 @@ async def create_api_to_jsonl_pipeline(
poll_interval: float = 0.5,
api_rate_limit_max_requests: int = 30,
api_rate_limit_window_seconds: float = 10.0,
# Handler 自定义配置(可选,作用于 API 端)
handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
local_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
remote_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
# Logger 配置
initialize_logger: bool = True,
logger_file_path: Optional[str] = None,
@@ -533,14 +525,13 @@ async def create_api_to_jsonl_pipeline(
poll_interval=poll_interval,
api_rate_limit_max_requests=api_rate_limit_max_requests,
api_rate_limit_window_seconds=api_rate_limit_window_seconds,
target_project_ids=target_project_ids or [],
handler_configs=handler_configs or {},
handler_configs=local_handler_configs or {},
),
remote_datasource=JsonlDataSourceConfig(
type="jsonl",
jsonl_dir=remote_jsonl_dir,
read_only=False,
target_project_ids=target_project_ids or [],
handler_configs=remote_handler_configs or {},
),
strategies=[],
persist=PersistConfig(
@@ -565,10 +556,11 @@ async def create_jsonl_to_jsonl_pipeline(
remote_jsonl_dir: str,
persistence_db: str,
node_types: List[str],
target_project_ids: Optional[List[str]] = None,
enable_persistence: bool = True,
wipe_persistence_on_start: bool = False,
strategy_overrides: Optional[Dict[str, Dict[str, Any]]] = None,
local_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
remote_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
initialize_logger: bool = True,
logger_file_path: Optional[str] = None,
logger_level: int = logging.INFO,
@@ -579,13 +571,13 @@ async def create_jsonl_to_jsonl_pipeline(
type="jsonl",
jsonl_dir=local_jsonl_dir,
read_only=False,
target_project_ids=target_project_ids or [],
handler_configs=local_handler_configs or {},
),
remote_datasource=JsonlDataSourceConfig(
type="jsonl",
jsonl_dir=remote_jsonl_dir,
read_only=False,
target_project_ids=target_project_ids or [],
handler_configs=remote_handler_configs or {},
),
strategies=[],
persist=PersistConfig(