调整了pipeline执行顺序。现在按domain顺序读取数据源数据,防止前面domain的更新影响后面domain.

将target_project_ids替换为更通用的data_id_filter
This commit is contained in:
strepsiades
2026-03-20 14:38:56 +08:00
parent f9f175ee79
commit 2c09c61165
29 changed files with 296 additions and 172 deletions
+10 -8
View File
@@ -22,10 +22,11 @@ local_datasource:
type: jsonl
jsonl_dir: ${PROJECT_ROOT}/filtered_datasource/datasource/filtered
read_only: false
target_project_ids:
- "0f9c3e22-f4bb-4803-825a-4f238eeb06d5"
- "f3e02e84-e81c-4aa6-88d4-f5aa108c8227"
handler_configs: {}
handler_configs:
project:
data_id_filter:
- "0f9c3e22-f4bb-4803-825a-4f238eeb06d5"
- "f3e02e84-e81c-4aa6-88d4-f5aa108c8227"
remote_datasource:
type: api
api_base_url: http://localhost:8000/api/third/v2
@@ -36,11 +37,12 @@ remote_datasource:
api_rate_limit_window_seconds: 10.0
poll_max_retries: 1
poll_interval: 0.5
target_project_ids:
# 请替换为实际的项目ID,至少需要一个
- "0f9c3e22-f4bb-4803-825a-4f238eeb06d5"
- "f3e02e84-e81c-4aa6-88d4-f5aa108c8227"
handler_configs:
project:
data_id_filter:
# 请替换为实际的项目ID,至少需要一个
- "0f9c3e22-f4bb-4803-825a-4f238eeb06d5"
- "f3e02e84-e81c-4aa6-88d4-f5aa108c8227"
supplier: {}
persist:
backend: sqlite
@@ -23,13 +23,11 @@ local_datasource:
type: jsonl
jsonl_dir: ${PROJECT_ROOT}/filtered_datasource/datasource/filtered
read_only: false
target_project_ids: []
handler_configs: {}
remote_datasource:
type: jsonl
jsonl_dir: ${PROJECT_ROOT}/remote_datasource/datasource
read_only: false
target_project_ids: []
handler_configs: {}
persist:
backend: sqlite