多pipeline同步支持

sync_state_machine/datasource/handler.py

@@ -361,11 +361,15 @@ class BaseNodeHandler(ABC, Generic[T]):
         self.handler_config = dict(config)
 
     def get_data_id_filter(self) -> List[str]:
-        value = self.handler_config.get("data_id_filter", [])
-        if isinstance(value, str):
-            return [value] if value else []
-        if isinstance(value, list):
-            return [str(item) for item in value if str(item)]
+        if "data_id_filter" in self.handler_config:
+            value = self.handler_config.get("data_id_filter", [])
+            if isinstance(value, str):
+                return [value] if value else []
+            if isinstance(value, list):
+                return [str(item) for item in value if str(item)]
+            return []
+        if self.node_type != "project" and self._collection is not None:
+            return [node.data_id for node in self._collection.filter(node_type="project") if node.data_id]
         return []
 
     def should_skip_by_data_id_filter(self, item: Dict[str, Any], item_id: str) -> bool:

sync_state_machine/datasource/jsonl/datasource.py

@@ -10,8 +10,10 @@ JsonlDataSource - JSONL 文件数据源
 
 from __future__ import annotations
 
+import json
+import re
 from pathlib import Path
-from typing import Dict, Any, Set
+from typing import Dict, Any, Set, Callable, List, Optional
 
 from ..datasource import BaseDataSource
 
@@ -46,6 +48,7 @@ class JsonlDataSource(BaseDataSource):
         # 用于 create/update/delete 的内存缓存
         self._data: Dict[str, Dict[str, Dict[str, Any]]] = {}  # {node_type: {id: data}}
         self._dirty_types: Set[str] = set()
+        self._node_items_cache: Dict[str, Dict[str, Any]] = {}
 
     def _on_node_loaded(self, handler, node) -> None:
         node.append_log(
@@ -61,6 +64,98 @@ class JsonlDataSource(BaseDataSource):
         node.append_log(
             f"JSONL链路[poll] task_id={task_id} status={result.status.value}"
         )
+
+    def invalidate_node_cache(self, node_type: str) -> None:
+        self._node_items_cache.pop(node_type, None)
+
+    def _matching_files(self, node_type: str) -> List[Path]:
+        if not self.dir_path.exists():
+            return []
+        pattern = re.compile(rf"^{re.escape(node_type)}_\d+\.jsonl$")
+        return sorted(
+            file_path
+            for file_path in self.dir_path.iterdir()
+            if pattern.match(file_path.name)
+        )
+
+    def _build_cache_entry(
+        self,
+        *,
+        node_type: str,
+        extract_id: Callable[[Dict[str, Any]], str],
+    ) -> Dict[str, Any]:
+        files = self._matching_files(node_type)
+        signature = tuple(
+            (file_path.name, file_path.stat().st_mtime_ns, file_path.stat().st_size)
+            for file_path in files
+        )
+        cached = self._node_items_cache.get(node_type)
+        if cached and cached.get("signature") == signature:
+            return cached
+
+        items: List[Dict[str, Any]] = []
+        data_id_index: Dict[str, List[Dict[str, Any]]] = {}
+        project_index: Dict[str, List[Dict[str, Any]]] = {}
+        no_project_items: List[Dict[str, Any]] = []
+
+        for file_path in files:
+            with open(file_path, "r", encoding="utf-8") as f:
+                for line_num, line in enumerate(f, 1):
+                    line = line.strip()
+                    if not line:
+                        continue
+
+                    try:
+                        item = json.loads(line)
+                    except json.JSONDecodeError:
+                        print(f"Warning: Invalid JSON in {file_path.name}:{line_num}")
+                        continue
+
+                    items.append(item)
+
+                    item_id = extract_id(item)
+                    if item_id:
+                        data_id_index.setdefault(str(item_id), []).append(item)
+
+                    project_id = item.get("project_id")
+                    if project_id:
+                        project_index.setdefault(str(project_id), []).append(item)
+                    else:
+                        no_project_items.append(item)
+
+        entry = {
+            "signature": signature,
+            "items": items,
+            "data_id_index": data_id_index,
+            "project_index": project_index,
+            "no_project_items": no_project_items,
+        }
+        self._node_items_cache[node_type] = entry
+        return entry
+
+    def get_items_for_load(
+        self,
+        *,
+        node_type: str,
+        extract_id: Callable[[Dict[str, Any]], str],
+        data_id_filter: Optional[List[str]] = None,
+        filter_on_project_id: bool = False,
+    ) -> List[Dict[str, Any]]:
+        entry = self._build_cache_entry(node_type=node_type, extract_id=extract_id)
+        filters = [str(item) for item in (data_id_filter or []) if str(item)]
+        if not filters:
+            return list(entry["items"])
+
+        if not filter_on_project_id:
+            matched: List[Dict[str, Any]] = []
+            for data_id in filters:
+                matched.extend(entry["data_id_index"].get(data_id, []))
+            return matched
+
+        matched = list(entry["no_project_items"])
+        for project_id in filters:
+            matched.extend(entry["project_index"].get(project_id, []))
+        return matched
     
     # load_all 和 sync_all 继承自 BaseDataSource
     # 保存逻辑在 BaseJsonlHandler 中实现

sync_state_machine/datasource/jsonl/handler.py

@@ -12,7 +12,6 @@ BaseJsonlHandler - JSONL Handler 基类
 from __future__ import annotations
 
 import json
-import re
 from typing import Dict, List, Any, TYPE_CHECKING, Optional, Set
 from uuid import uuid4
 
@@ -64,40 +63,26 @@ class BaseJsonlHandler(BaseNodeHandler):
         
         if not self.datasource.dir_path.exists():
             return nodes
-        
-        # 查找所有匹配的文件：{node_type}_N.jsonl
-        pattern = re.compile(rf"^{re.escape(self.node_type)}_\d+\.jsonl$")
-        
-        for file_path in self.datasource.dir_path.iterdir():
-            if not pattern.match(file_path.name):
-                continue
-            
-            # 读取文件
-            with open(file_path, "r", encoding="utf-8") as f:
-                for line_num, line in enumerate(f, 1):
-                    line = line.strip()
-                    if not line:
-                        continue
-                    
-                    try:
-                        item = json.loads(line)
-                        item_id = self.extract_id(item)
 
-                        if self.should_skip_by_data_id_filter(item, item_id):
-                            continue
-                        
-                        # 创建节点
-                        node = self.create_node(item)
-                        nodes.append(node)
-                        
-                        # 同时加载到缓存
-                        if item_id:
-                            bucket = self.datasource._data.setdefault(self.node_type, {})
-                            bucket[item_id] = item
-                            
-                    except json.JSONDecodeError as e:
-                        print(f"Warning: Invalid JSON in {file_path.name}:{line_num}: {e}")
-                        # 继续处理下一行
+        data_id_filter = self.get_data_id_filter()
+        items = self.datasource.get_items_for_load(
+            node_type=self.node_type,
+            extract_id=self.extract_id,
+            data_id_filter=data_id_filter,
+            filter_on_project_id=self.node_type != "project",
+        )
+
+        for item in items:
+            item_id = self.extract_id(item)
+            if self.should_skip_by_data_id_filter(item, item_id):
+                continue
+
+            node = self.create_node(item)
+            nodes.append(node)
+
+            if item_id:
+                bucket = self.datasource._data.setdefault(self.node_type, {})
+                bucket[item_id] = item
         
         return nodes
     
@@ -276,6 +261,7 @@ class BaseJsonlHandler(BaseNodeHandler):
             
             # 清除该类型的 dirty 标记
             self.datasource._dirty_types.discard(self.node_type)
+            self.datasource.invalidate_node_cache(self.node_type)
             
         except Exception as e:
             print(f"Error saving {self.node_type}: {str(e)}")