调整了pipeline执行顺序。现在按domain顺序读取数据源数据,防止前面domain的更新影响后面domain.
将target_project_ids替换为更通用的data_id_filter
This commit is contained in:
+23
-4
@@ -200,14 +200,30 @@
|
||||
function toCsv(list) { return (list || []).join(","); }
|
||||
function fromCsv(text) { return (text || "").split(",").map(s => s.trim()).filter(Boolean); }
|
||||
|
||||
function buildProjectFilterHandlerConfigs(baseConfigs, projectIds) {
|
||||
const nextConfigs = { ...(baseConfigs || {}) };
|
||||
const ids = Array.isArray(projectIds) ? projectIds : [];
|
||||
if (!ids.length) return nextConfigs;
|
||||
nextConfigs.project = {
|
||||
...(nextConfigs.project || {}),
|
||||
data_id_filter: ids,
|
||||
};
|
||||
return nextConfigs;
|
||||
}
|
||||
|
||||
function readProjectFilter(cfg) {
|
||||
return cfg?.handler_configs?.project?.data_id_filter || [];
|
||||
}
|
||||
|
||||
function readForm() {
|
||||
const mode = el.mode.value;
|
||||
const handlerConfigs = safeJsonParse(el.handlerConfigs.value, {});
|
||||
const projectIds = fromCsv(el.targetProjectIds.value);
|
||||
const handlerConfigs = buildProjectFilterHandlerConfigs(safeJsonParse(el.handlerConfigs.value, {}), projectIds);
|
||||
const localDs = {
|
||||
type: "jsonl",
|
||||
jsonl_dir: el.localJsonlDir.value,
|
||||
read_only: false,
|
||||
handler_configs: {},
|
||||
handler_configs: buildProjectFilterHandlerConfigs({}, projectIds),
|
||||
};
|
||||
const remoteDs = mode === "jsonl_to_api"
|
||||
? {
|
||||
@@ -244,7 +260,6 @@
|
||||
file_path: el.logFilePath.value || null,
|
||||
level: 20,
|
||||
},
|
||||
target_project_ids: fromCsv(el.targetProjectIds.value),
|
||||
strategies: safeJsonParse(el.strategies.value, {}),
|
||||
};
|
||||
}
|
||||
@@ -257,7 +272,11 @@
|
||||
el.persistBackend.value = cfg.persist?.backend || "sqlite";
|
||||
el.persistenceDb.value = cfg.persist?.db_path || "";
|
||||
el.backendOptions.value = JSON.stringify(cfg.persist?.backend_options || {}, null, 2);
|
||||
el.targetProjectIds.value = toCsv(cfg.target_project_ids);
|
||||
el.targetProjectIds.value = toCsv(
|
||||
readProjectFilter(cfg.remote_datasource).length
|
||||
? readProjectFilter(cfg.remote_datasource)
|
||||
: readProjectFilter(cfg.local_datasource)
|
||||
);
|
||||
if (cfg.remote_datasource?.type === "api") {
|
||||
el.apiBaseUrl.value = cfg.remote_datasource.api_base_url || "";
|
||||
el.apiUid.value = cfg.remote_datasource.api_uid || "";
|
||||
|
||||
Reference in New Issue
Block a user