调整了pipeline执行顺序。现在按domain顺序读取数据源数据,防止前面domain的更新影响后面domain.

将target_project_ids替换为更通用的data_id_filter
This commit is contained in:
strepsiades
2026-03-20 14:38:56 +08:00
parent f9f175ee79
commit 2c09c61165
29 changed files with 296 additions and 172 deletions
+23 -4
View File
@@ -200,14 +200,30 @@
function toCsv(list) { return (list || []).join(","); }
function fromCsv(text) { return (text || "").split(",").map(s => s.trim()).filter(Boolean); }
function buildProjectFilterHandlerConfigs(baseConfigs, projectIds) {
const nextConfigs = { ...(baseConfigs || {}) };
const ids = Array.isArray(projectIds) ? projectIds : [];
if (!ids.length) return nextConfigs;
nextConfigs.project = {
...(nextConfigs.project || {}),
data_id_filter: ids,
};
return nextConfigs;
}
function readProjectFilter(cfg) {
return cfg?.handler_configs?.project?.data_id_filter || [];
}
function readForm() {
const mode = el.mode.value;
const handlerConfigs = safeJsonParse(el.handlerConfigs.value, {});
const projectIds = fromCsv(el.targetProjectIds.value);
const handlerConfigs = buildProjectFilterHandlerConfigs(safeJsonParse(el.handlerConfigs.value, {}), projectIds);
const localDs = {
type: "jsonl",
jsonl_dir: el.localJsonlDir.value,
read_only: false,
handler_configs: {},
handler_configs: buildProjectFilterHandlerConfigs({}, projectIds),
};
const remoteDs = mode === "jsonl_to_api"
? {
@@ -244,7 +260,6 @@
file_path: el.logFilePath.value || null,
level: 20,
},
target_project_ids: fromCsv(el.targetProjectIds.value),
strategies: safeJsonParse(el.strategies.value, {}),
};
}
@@ -257,7 +272,11 @@
el.persistBackend.value = cfg.persist?.backend || "sqlite";
el.persistenceDb.value = cfg.persist?.db_path || "";
el.backendOptions.value = JSON.stringify(cfg.persist?.backend_options || {}, null, 2);
el.targetProjectIds.value = toCsv(cfg.target_project_ids);
el.targetProjectIds.value = toCsv(
readProjectFilter(cfg.remote_datasource).length
? readProjectFilter(cfg.remote_datasource)
: readProjectFilter(cfg.local_datasource)
);
if (cfg.remote_datasource?.type === "api") {
el.apiBaseUrl.value = cfg.remote_datasource.api_base_url || "";
el.apiUid.value = cfg.remote_datasource.api_uid || "";