多pipeline同步支持

sync_state_machine/datasource/jsonl/handler.py

@@ -12,7 +12,6 @@ BaseJsonlHandler - JSONL Handler 基类
 from __future__ import annotations
 
 import json
-import re
 from typing import Dict, List, Any, TYPE_CHECKING, Optional, Set
 from uuid import uuid4
 
@@ -64,40 +63,26 @@ class BaseJsonlHandler(BaseNodeHandler):
         
         if not self.datasource.dir_path.exists():
             return nodes
-        
-        # 查找所有匹配的文件：{node_type}_N.jsonl
-        pattern = re.compile(rf"^{re.escape(self.node_type)}_\d+\.jsonl$")
-        
-        for file_path in self.datasource.dir_path.iterdir():
-            if not pattern.match(file_path.name):
-                continue
-            
-            # 读取文件
-            with open(file_path, "r", encoding="utf-8") as f:
-                for line_num, line in enumerate(f, 1):
-                    line = line.strip()
-                    if not line:
-                        continue
-                    
-                    try:
-                        item = json.loads(line)
-                        item_id = self.extract_id(item)
 
-                        if self.should_skip_by_data_id_filter(item, item_id):
-                            continue
-                        
-                        # 创建节点
-                        node = self.create_node(item)
-                        nodes.append(node)
-                        
-                        # 同时加载到缓存
-                        if item_id:
-                            bucket = self.datasource._data.setdefault(self.node_type, {})
-                            bucket[item_id] = item
-                            
-                    except json.JSONDecodeError as e:
-                        print(f"Warning: Invalid JSON in {file_path.name}:{line_num}: {e}")
-                        # 继续处理下一行
+        data_id_filter = self.get_data_id_filter()
+        items = self.datasource.get_items_for_load(
+            node_type=self.node_type,
+            extract_id=self.extract_id,
+            data_id_filter=data_id_filter,
+            filter_on_project_id=self.node_type != "project",
+        )
+
+        for item in items:
+            item_id = self.extract_id(item)
+            if self.should_skip_by_data_id_filter(item, item_id):
+                continue
+
+            node = self.create_node(item)
+            nodes.append(node)
+
+            if item_id:
+                bucket = self.datasource._data.setdefault(self.node_type, {})
+                bucket[item_id] = item
         
         return nodes
     
@@ -276,6 +261,7 @@ class BaseJsonlHandler(BaseNodeHandler):
             
             # 清除该类型的 dirty 标记
             self.datasource._dirty_types.discard(self.node_type)
+            self.datasource.invalidate_node_cache(self.node_type)
             
         except Exception as e:
             print(f"Error saving {self.node_type}: {str(e)}")