调整了pipeline执行顺序。现在按domain顺序读取数据源数据,防止前面domain的更新影响后面domain.

将target_project_ids替换为更通用的data_id_filter
This commit is contained in:
strepsiades
2026-03-20 14:38:56 +08:00
parent f9f175ee79
commit 2c09c61165
29 changed files with 296 additions and 172 deletions
@@ -50,22 +50,17 @@ class BaseJsonlHandler(BaseNodeHandler):
):
super().__init__(node_type, node_class, schema)
self.datasource = datasource
self._target_project_ids: List[str] = []
def set_target_project_ids(self, target_project_ids: List[str]) -> None:
"""注入项目过滤列表的本地副本。真正的过滤来源仍然是 datasource。"""
self._target_project_ids = [str(pid) for pid in target_project_ids if str(pid)]
def _should_skip_item_by_project_filter(self, item: Dict[str, Any], item_id: str) -> bool:
target_project_ids = set(self.datasource.target_project_ids)
if not target_project_ids:
data_id_filter = set(self.get_data_id_filter())
if not data_id_filter:
return False
if self.node_type == "project":
return item_id not in target_project_ids
return item_id not in data_id_filter
project_id = item.get("project_id")
if project_id:
return str(project_id) not in target_project_ids
return str(project_id) not in data_id_filter
return False
async def load(self) -> List['SyncNode']: