调整了pipeline执行顺序。现在按domain顺序读取数据源数据,防止前面domain的更新影响后面domain.
将target_project_ids替换为更通用的data_id_filter
This commit is contained in:
@@ -22,10 +22,11 @@ local_datasource:
|
||||
type: jsonl
|
||||
jsonl_dir: ${PROJECT_ROOT}/filtered_datasource/datasource/filtered
|
||||
read_only: false
|
||||
target_project_ids:
|
||||
- "0f9c3e22-f4bb-4803-825a-4f238eeb06d5"
|
||||
- "f3e02e84-e81c-4aa6-88d4-f5aa108c8227"
|
||||
handler_configs: {}
|
||||
handler_configs:
|
||||
project:
|
||||
data_id_filter:
|
||||
- "0f9c3e22-f4bb-4803-825a-4f238eeb06d5"
|
||||
- "f3e02e84-e81c-4aa6-88d4-f5aa108c8227"
|
||||
remote_datasource:
|
||||
type: api
|
||||
api_base_url: http://localhost:8000/api/third/v2
|
||||
@@ -36,11 +37,12 @@ remote_datasource:
|
||||
api_rate_limit_window_seconds: 10.0
|
||||
poll_max_retries: 1
|
||||
poll_interval: 0.5
|
||||
target_project_ids:
|
||||
# 请替换为实际的项目ID,至少需要一个
|
||||
- "0f9c3e22-f4bb-4803-825a-4f238eeb06d5"
|
||||
- "f3e02e84-e81c-4aa6-88d4-f5aa108c8227"
|
||||
handler_configs:
|
||||
project:
|
||||
data_id_filter:
|
||||
# 请替换为实际的项目ID,至少需要一个
|
||||
- "0f9c3e22-f4bb-4803-825a-4f238eeb06d5"
|
||||
- "f3e02e84-e81c-4aa6-88d4-f5aa108c8227"
|
||||
supplier: {}
|
||||
persist:
|
||||
backend: sqlite
|
||||
|
||||
@@ -23,13 +23,11 @@ local_datasource:
|
||||
type: jsonl
|
||||
jsonl_dir: ${PROJECT_ROOT}/filtered_datasource/datasource/filtered
|
||||
read_only: false
|
||||
target_project_ids: []
|
||||
handler_configs: {}
|
||||
remote_datasource:
|
||||
type: jsonl
|
||||
jsonl_dir: ${PROJECT_ROOT}/remote_datasource/datasource
|
||||
read_only: false
|
||||
target_project_ids: []
|
||||
handler_configs: {}
|
||||
persist:
|
||||
backend: sqlite
|
||||
|
||||
Reference in New Issue
Block a user