调整了pipeline执行顺序。现在按domain顺序读取数据源数据,防止前面domain的更新影响后面domain.
将target_project_ids替换为更通用的data_id_filter
This commit is contained in:
@@ -50,22 +50,17 @@ class BaseJsonlHandler(BaseNodeHandler):
|
||||
):
|
||||
super().__init__(node_type, node_class, schema)
|
||||
self.datasource = datasource
|
||||
self._target_project_ids: List[str] = []
|
||||
|
||||
def set_target_project_ids(self, target_project_ids: List[str]) -> None:
|
||||
"""注入项目过滤列表的本地副本。真正的过滤来源仍然是 datasource。"""
|
||||
self._target_project_ids = [str(pid) for pid in target_project_ids if str(pid)]
|
||||
|
||||
def _should_skip_item_by_project_filter(self, item: Dict[str, Any], item_id: str) -> bool:
|
||||
target_project_ids = set(self.datasource.target_project_ids)
|
||||
if not target_project_ids:
|
||||
data_id_filter = set(self.get_data_id_filter())
|
||||
if not data_id_filter:
|
||||
return False
|
||||
if self.node_type == "project":
|
||||
return item_id not in target_project_ids
|
||||
return item_id not in data_id_filter
|
||||
|
||||
project_id = item.get("project_id")
|
||||
if project_id:
|
||||
return str(project_id) not in target_project_ids
|
||||
return str(project_id) not in data_id_filter
|
||||
return False
|
||||
|
||||
async def load(self) -> List['SyncNode']:
|
||||
|
||||
Reference in New Issue
Block a user