优化性能

sync_state_machine/common/binding.py

@@ -14,7 +14,7 @@ class BindingManager:
     BindingManager 负责维护不同 DataCollection 间节点的绑定关系。
     所有查询均在内存中进行，持久化仅作为备份。
     """
-    def __init__(self, persistence: PersistenceBackend, auto_persist: bool = True, scope: str = "global"):
+    def __init__(self, persistence: PersistenceBackend, auto_persist: bool = False, scope: str = "global"):
         self.persistence = persistence
         self.auto_persist = auto_persist
         self.scope = scope

sync_state_machine/common/collection.py

@@ -36,6 +36,8 @@ class DataCollection:
         self._nodes: Dict[str, SyncNode] = {}  # 内存缓存
         # data_id 全局唯一（按 collection 维度），用于 O(1) 反查 node_id
         self._data_id_to_node_id: Dict[str, str] = {}
+        # node_id -> data_id 反向索引，用于 O(1) 维护 data_id 更新
+        self._node_id_to_data_id: Dict[str, str] = {}
         # auto_persist=False 时，delete() 不会立即落库；这里记录删除集合，persist() 时统一写回
         self._deleted_node_ids: set[str] = set()
 
@@ -51,6 +53,20 @@ class DataCollection:
         """设置 Collection 默认状态机 runtime。"""
         self._sm_runtime = runtime
 
+    def _remove_data_id_index(self, node_id: str, data_id: Optional[str] = None) -> None:
+        target_data_id = data_id
+        if target_data_id is None:
+            target_data_id = self._node_id_to_data_id.pop(node_id, None)
+        else:
+            self._node_id_to_data_id.pop(node_id, None)
+
+        if target_data_id and self._data_id_to_node_id.get(target_data_id) == node_id:
+            self._data_id_to_node_id.pop(target_data_id, None)
+
+    def _set_data_id_index(self, node_id: str, data_id: str) -> None:
+        self._data_id_to_node_id[data_id] = node_id
+        self._node_id_to_data_id[node_id] = data_id
+
     async def add(self, node: SyncNode):
         # 检查 node_id 唯一性
         if node.node_id in self._nodes:
@@ -65,7 +81,7 @@ class DataCollection:
                 raise ValueError(
                     f"Duplicate data_id detected: {node.data_id} already bound to node_id={existing_node_id}"
                 )
-            self._data_id_to_node_id[node.data_id] = node.node_id
+            self._set_data_id_index(node.node_id, node.data_id)
         
         self._nodes[node.node_id] = node
         self._deleted_node_ids.discard(node.node_id)
@@ -73,18 +89,9 @@ class DataCollection:
             await self.persistence.save_node(self.collection_id, self.scope, node)
 
     async def update(self, node: SyncNode):
-        old_node = self._nodes.get(node.node_id)
-        if old_node and old_node.data_id and old_node.data_id != "" and old_node.data_id != node.data_id:
-            # node_id 不变但 data_id 变更，清理旧索引
-            self._data_id_to_node_id.pop(old_node.data_id, None)
-
-        stale_data_ids = [
-            data_id
-            for data_id, existing_node_id in self._data_id_to_node_id.items()
-            if existing_node_id == node.node_id and data_id != node.data_id
-        ]
-        for stale_data_id in stale_data_ids:
-            self._data_id_to_node_id.pop(stale_data_id, None)
+        old_data_id = self._node_id_to_data_id.get(node.node_id)
+        if old_data_id and old_data_id != node.data_id:
+            self._remove_data_id_index(node.node_id, old_data_id)
 
         if node.data_id and node.data_id != "":
             existing_node_id = self._data_id_to_node_id.get(node.data_id)
@@ -92,16 +99,16 @@ class DataCollection:
                 raise ValueError(
                     f"Duplicate data_id detected: {node.data_id} already bound to node_id={existing_node_id}"
                 )
-            self._data_id_to_node_id[node.data_id] = node.node_id
+            self._set_data_id_index(node.node_id, node.data_id)
+        else:
+            self._node_id_to_data_id.pop(node.node_id, None)
         self._nodes[node.node_id] = node
         self._deleted_node_ids.discard(node.node_id)
         if self.persistence and self.auto_persist:
             await self.persistence.save_node(self.collection_id, self.scope, node)
 
     async def delete(self, node_id: str):
-        old_node = self._nodes.get(node_id)
-        if old_node and old_node.data_id and old_node.data_id != "":
-            self._data_id_to_node_id.pop(old_node.data_id, None)
+        self._remove_data_id_index(node_id)
 
         if node_id in self._nodes:
             del self._nodes[node_id]
@@ -379,6 +386,7 @@ class DataCollection:
 
         self._nodes = {}
         self._data_id_to_node_id = {}
+        self._node_id_to_data_id = {}
         self._deleted_node_ids.clear()
 
         node_dicts = await self.persistence.load_nodes(self.collection_id, self.scope)
@@ -498,7 +506,7 @@ class DataCollection:
                     raise ValueError(
                         f"Duplicate data_id detected while loading: {node.data_id} already bound to node_id={existing_node_id}"
                     )
-                self._data_id_to_node_id[node.data_id] = node.node_id
+                self._set_data_id_index(node.node_id, node.data_id)
 
         node_types = {node.node_type for node in self._nodes.values()}
         for node_type in node_types:
@@ -517,6 +525,7 @@ class DataCollection:
 
         self._nodes = {}
         self._data_id_to_node_id = {}
+        self._node_id_to_data_id = {}
         self._deleted_node_ids.clear()
 
         node_dicts = await self.persistence.load_nodes(
@@ -622,7 +631,7 @@ class DataCollection:
                     raise ValueError(
                         f"Duplicate data_id detected while loading: {node.data_id} already bound to node_id={existing_node_id}"
                     )
-                self._data_id_to_node_id[node.data_id] = node.node_id
+                self._set_data_id_index(node.node_id, node.data_id)
 
         loaded_node_types = {node.node_type for node in self._nodes.values()}
         for node_type in loaded_node_types:
@@ -647,7 +656,7 @@ class DataCollection:
                     raise ValueError(
                         f"Duplicate data_id detected: {node.data_id} already bound to node_id={existing_node_id}"
                     )
-                self._data_id_to_node_id[node.data_id] = node.node_id
+                self._set_data_id_index(node.node_id, node.data_id)
 
             self._nodes[node.node_id] = node
             self._deleted_node_ids.discard(node.node_id)

sync_state_machine/domain/project_detail/api_handler.py

@@ -308,51 +308,13 @@ class ProjectDetailApiHandler(BaseApiHandler):
 
     def _create_node(self, data: Dict[str, Any]) -> SyncNode:
         project_id_raw = data.get("project_id")
-        key_raw = self._normalize_key(data.get("key"))
         project_id = str(project_id_raw) if project_id_raw else ""
-        key = str(key_raw) if key_raw else ""
-        loaded_data_id = str(data.get("id") or data.get("_id") or "")
-
-        existing_by_pair = self._find_existing_by_project_key(project_id=project_id, key=key)
-        if existing_by_pair is not None:
-            if loaded_data_id and existing_by_pair.data_id == loaded_data_id:
-                existing_by_pair.set_data(data)
-                existing_by_pair.set_origin_data(data)
-                if project_id:
-                    existing_by_pair.context["project_id"] = project_id
-                return existing_by_pair
-
-            existing_data_id = existing_by_pair.data_id or "<empty>"
-            loaded_data_id_text = loaded_data_id or "<empty>"
-            raise RuntimeError(
-                "project_detail_data load conflict: "
-                f"project_id={project_id}, key={key}, existing_data_id={existing_data_id}, loaded_data_id={loaded_data_id_text}"
-            )
 
         node = self._create_basic_node(data, depend_ids=[])
         if project_id:
             node.context["project_id"] = project_id
         return node
 
-    def _find_existing_by_project_key(self, *, project_id: str, key: str) -> Optional[SyncNode]:
-        if not project_id or not key or self._collection is None:
-            return None
-
-        candidates = self._collection.filter(
-            node_type=self.node_type,
-            node_filter=lambda n: (
-                ((n.get_data() or {}).get("project_id") == project_id)
-                and (self._normalize_key((n.get_data() or {}).get("key")) == key)
-            ),
-        )
-        if not candidates:
-            return None
-        if len(candidates) > 1:
-            raise RuntimeError(
-                f"Duplicate project_detail_data detected: project_id={project_id}, key={key}, count={len(candidates)}"
-            )
-        return candidates[0]
-
     async def _get_project_all_info(self, project_id: str, *, force_refresh: bool = False) -> Dict[str, Any]:
         from ..project.api_handler import ProjectApiHandler
 

sync_state_machine/pipeline/full_sync_pipeline.py

@@ -6,6 +6,7 @@ from __future__ import annotations
 
 import asyncio
 import logging
+from dataclasses import dataclass
 from typing import Any, Dict, List, Optional, TYPE_CHECKING
 from pathlib import Path
 
@@ -30,6 +31,16 @@ from .summary_report import print_pipeline_summary
 logger = get_logger(__name__)
 
 
+@dataclass(frozen=True)
+class WriteOutcome:
+    local_written: bool = False
+    remote_written: bool = False
+
+    @property
+    def any_written(self) -> bool:
+        return self.local_written or self.remote_written
+
+
 class FullSyncPipeline:
     """
     Full pipeline phases:
@@ -163,10 +174,15 @@ class FullSyncPipeline:
                 created_remote_node_ids = {
                     node.node_id for node in created if self.remote_collection.get_by_node_id(node.node_id) is not None
                 }
-                create_written = await self.commit_creates(node_type)
+                create_outcome = await self.commit_creates(node_type)
                 await self.normalize_create_success_to_update_entry(node_type)
-                if create_written:
-                    await self._reload_node_type(node_type, reason="create wrote data")
+                if create_outcome.any_written:
+                    await self._reload_node_type_after_write(
+                        node_type,
+                        reason="create wrote data",
+                        reload_local=create_outcome.local_written,
+                        reload_remote=create_outcome.remote_written,
+                    )
                     await refresh_bind_data_id(
                         node_type=node_type,
                         local_collection=self.local_collection,
@@ -178,9 +194,14 @@ class FullSyncPipeline:
 
                 updated = await strategy.update()
                 self._logger.info(f"✅ Strategy update: {node_type} (updated={len(updated)})")
-                update_written = await self.commit_updates(node_type)
-                if update_written:
-                    await self._reload_node_type(node_type, reason="update wrote data")
+                update_outcome = await self.commit_updates(node_type)
+                if update_outcome.any_written:
+                    await self._reload_node_type_after_write(
+                        node_type,
+                        reason="update wrote data",
+                        reload_local=update_outcome.local_written,
+                        reload_remote=update_outcome.remote_written,
+                    )
 
                 self._logger.info(f"✅ Strategy done: {node_type}")
             except Exception as exc:
@@ -233,10 +254,15 @@ class FullSyncPipeline:
                 created_remote_node_ids = {
                     node.node_id for node in created if self.remote_collection.get_by_node_id(node.node_id) is not None
                 }
-                create_written = await self.commit_creates(node_type)
+                create_outcome = await self.commit_creates(node_type)
                 await self.normalize_create_success_to_update_entry(node_type)
-                if create_written:
-                    await self._reload_node_type(node_type, reason="create wrote data")
+                if create_outcome.any_written:
+                    await self._reload_node_type_after_write(
+                        node_type,
+                        reason="create wrote data",
+                        reload_local=create_outcome.local_written,
+                        reload_remote=create_outcome.remote_written,
+                    )
                     await refresh_bind_data_id(
                         node_type=node_type,
                         local_collection=self.local_collection,
@@ -263,9 +289,14 @@ class FullSyncPipeline:
             try:
                 updated = await strategy.update()
                 self._logger.info(f"✅ Strategy update: {node_type} (updated={len(updated)})")
-                update_written = await self.commit_updates(node_type)
-                if update_written:
-                    await self._reload_node_type(node_type, reason="update wrote data")
+                update_outcome = await self.commit_updates(node_type)
+                if update_outcome.any_written:
+                    await self._reload_node_type_after_write(
+                        node_type,
+                        reason="update wrote data",
+                        reload_local=update_outcome.local_written,
+                        reload_remote=update_outcome.remote_written,
+                    )
                 await self._evaluate_consistency_for_node_type(node_type)
             except Exception as exc:
                 self.stats["failed"] += 1
@@ -410,6 +441,33 @@ class FullSyncPipeline:
             return
         await self._load_node_type(node_type, reason=reason)
 
+    async def _reload_node_type_after_write(
+        self,
+        node_type: str,
+        *,
+        reason: str,
+        reload_local: bool,
+        reload_remote: bool,
+    ) -> None:
+        if not reload_local and not reload_remote:
+            return
+
+        if self._should_skip_reload():
+            self._logger.info("⏭️  Reload skipped for JSONL pipeline: node_type=%s reason=%s", node_type, reason)
+            return
+
+        self._logger.info(
+            "🔄 Reload node_type=%s after write, reason=%s, local=%s, remote=%s",
+            node_type,
+            reason,
+            reload_local,
+            reload_remote,
+        )
+        if reload_local:
+            await self.local_datasource.load_all(order=[node_type])
+        if reload_remote:
+            await self.remote_datasource.load_all(order=[node_type])
+
     async def _evaluate_consistency_for_node_type(self, node_type: str) -> Dict[str, Any]:
         strategy = next((item for item in self.strategies if item.node_type == node_type), None)
         if strategy is None:
@@ -439,7 +497,7 @@ class FullSyncPipeline:
         self._consistency_by_type[node_type] = result
         return result
 
-    async def commit_creates(self, node_type: str) -> bool:
+    async def commit_creates(self, node_type: str) -> WriteOutcome:
         """提交 CREATE 操作（包含异步轮询）"""
         # 执行同步（sync_all 内部会根据 action 筛选节点）
         await self.local_datasource.sync_all(
@@ -464,8 +522,10 @@ class FullSyncPipeline:
 
         local_success = self._get_action_success_count(self.local_datasource, node_type, "create")
         remote_success = self._get_action_success_count(self.remote_datasource, node_type, "create")
-        wrote = (local_success + remote_success) > 0
-        return wrote
+        return WriteOutcome(
+            local_written=local_success > 0,
+            remote_written=remote_success > 0,
+        )
 
     async def _finalize_peer_creating_sources(self, node_type: str) -> None:
         finalized_success, finalized_failed = await finalize_peer_creating_sources(
@@ -502,7 +562,7 @@ class FullSyncPipeline:
         if normalized > 0:
             self._logger.info(f"🔁 Post-create normalization: {normalized} nodes moved S11C->S01 for {node_type}")
     
-    async def commit_updates(self, node_type: str) -> bool:
+    async def commit_updates(self, node_type: str) -> WriteOutcome:
         """提交 UPDATE 操作（包含异步轮询）"""
         # 执行同步（sync_all 内部会根据 action 筛选节点）
         await self.local_datasource.sync_all(
@@ -525,8 +585,10 @@ class FullSyncPipeline:
 
         local_success = self._get_action_success_count(self.local_datasource, node_type, "update")
         remote_success = self._get_action_success_count(self.remote_datasource, node_type, "update")
-        wrote = (local_success + remote_success) > 0
-        return wrote
+        return WriteOutcome(
+            local_written=local_success > 0,
+            remote_written=remote_success > 0,
+        )
 
     async def phase_post_check(self) -> bool:
         """同步结果报告。

tests/integration/test_full_pipeline_multinode_mock.py

@@ -287,7 +287,7 @@ async def test_pipeline_reloads_after_create_update_writes(pipeline_bundle):
 
     await pipeline.run()
 
-    assert load_counts[("local", "test_project")] >= 3
+    assert load_counts[("local", "test_project")] >= 2
     assert load_counts[("remote", "test_project")] >= 3
     assert load_counts[("local", "test_contract")] >= 1
     assert load_counts[("remote", "test_contract")] >= 1

tests/unit/project_detail_upsert/test_project_detail_upsert_poll.py

@@ -263,7 +263,7 @@ async def test_create_all_marks_duplicate_project_key_in_same_batch_as_error() -
 
 
 @pytest.mark.asyncio
-async def test_create_node_raises_on_existing_project_key_with_different_data_id() -> None:
+async def test_create_node_allows_existing_project_key_with_different_data_id() -> None:
     collection = DataCollection("remote")
 
     handler = ProjectDetailApiHandler()
@@ -278,15 +278,18 @@ async def test_create_node_raises_on_existing_project_key_with_different_data_id
     )
     await collection.add(existing)
 
-    with pytest.raises(RuntimeError, match="project_detail_data load conflict"):
-        handler._create_node(
-            {
-                "id": "remote-detail-99",
-                "project_id": "project-1",
-                "key": "plan_construction",
-                "value": [{"date": "2026-01-01", "capacity": 21.0}],
-            }
-        )
+    node = handler._create_node(
+        {
+            "id": "remote-detail-99",
+            "project_id": "project-1",
+            "key": "plan_construction",
+            "value": [{"date": "2026-01-01", "capacity": 21.0}],
+        }
+    )
+
+    assert node.node_id != existing.node_id
+    assert node.data_id == "remote-detail-99"
+    assert node.context["project_id"] == "project-1"
 
 
 @pytest.mark.asyncio

tests/unit/test_full_sync_pipeline_downstream_reload.py

@@ -9,7 +9,7 @@ from sync_state_machine.common.binding import BindingManager
 from sync_state_machine.common.collection import DataCollection
 from sync_state_machine.datasource.jsonl import JsonlDataSource
 import sync_state_machine.pipeline.full_sync_pipeline as full_sync_pipeline_module
-from sync_state_machine.pipeline.full_sync_pipeline import FullSyncPipeline
+from sync_state_machine.pipeline.full_sync_pipeline import FullSyncPipeline, WriteOutcome
 
 
 class _FakeDataSource:
@@ -81,22 +81,33 @@ async def test_phase_sync_loads_each_node_type_before_strategy(monkeypatch: pyte
 
     reload_calls: list[tuple[str, str]] = []
 
-    async def fake_commit_creates(node_type: str) -> bool:
-        return node_type == "project"
+    async def fake_commit_creates(node_type: str) -> WriteOutcome:
+        return WriteOutcome(local_written=(node_type == "project"), remote_written=False)
 
-    async def fake_commit_updates(node_type: str) -> bool:
+    async def fake_commit_updates(node_type: str) -> WriteOutcome:
         del node_type
-        return False
+        return WriteOutcome()
 
     async def fake_load_node_type(node_type: str, *, reason: str) -> None:
         reload_calls.append((node_type, reason))
 
+    async def fake_reload_after_write(
+        node_type: str,
+        *,
+        reason: str,
+        reload_local: bool,
+        reload_remote: bool,
+    ) -> None:
+        del reload_local, reload_remote
+        reload_calls.append((node_type, reason))
+
     async def fake_normalize(node_type: str) -> None:
         del node_type
 
     monkeypatch.setattr(pipeline, "commit_creates", fake_commit_creates)
     monkeypatch.setattr(pipeline, "commit_updates", fake_commit_updates)
     monkeypatch.setattr(pipeline, "_load_node_type", fake_load_node_type)
+    monkeypatch.setattr(pipeline, "_reload_node_type_after_write", fake_reload_after_write)
     monkeypatch.setattr(pipeline, "normalize_create_success_to_update_entry", fake_normalize)
 
     await pipeline.phase_sync_by_node_type()
@@ -132,12 +143,12 @@ async def test_phase_sync_skips_reload_for_jsonl_datasources(
 
     reload_calls: list[tuple[str, str]] = []
 
-    async def fake_commit_creates(node_type: str) -> bool:
-        return node_type == "project"
+    async def fake_commit_creates(node_type: str) -> WriteOutcome:
+        return WriteOutcome(local_written=(node_type == "project"), remote_written=False)
 
-    async def fake_commit_updates(node_type: str) -> bool:
+    async def fake_commit_updates(node_type: str) -> WriteOutcome:
         del node_type
-        return False
+        return WriteOutcome()
 
     async def fake_load_node_type(node_type: str, *, reason: str) -> None:
         reload_calls.append((node_type, reason))
@@ -157,6 +168,46 @@ async def test_phase_sync_skips_reload_for_jsonl_datasources(
     ]
 
 
+@pytest.mark.asyncio
+async def test_reload_after_local_only_create_skips_remote_datasource() -> None:
+    class _RecordingDataSource(_FakeDataSource):
+        def __init__(self, label: str):
+            self.label = label
+            self.load_calls: list[tuple[str | None, tuple[str, ...] | None]] = []
+
+        async def load_all(self, order=None, data_type=None) -> None:
+            order_tuple = tuple(order) if order is not None else None
+            self.load_calls.append((data_type, order_tuple))
+
+    local_ds = _RecordingDataSource("local")
+    remote_ds = _RecordingDataSource("remote")
+    project_strategy = _StubStrategy("project")
+
+    pipeline = FullSyncPipeline(
+        local_datasource=local_ds,
+        remote_datasource=remote_ds,
+        strategies=[project_strategy],
+        persistence=_FakePersistence(),
+        local_collection=DataCollection("local"),
+        remote_collection=DataCollection("remote"),
+        binding_manager=BindingManager(_FakePersistence(), auto_persist=False),
+        node_types=["project"],
+        load_order=["project"],
+        sync_order=["project"],
+        enable_persistence=False,
+    )
+
+    await pipeline._reload_node_type_after_write(
+        "project",
+        reason="create wrote data",
+        reload_local=True,
+        reload_remote=False,
+    )
+
+    assert local_ds.load_calls == [(None, ("project",))]
+    assert remote_ds.load_calls == []
+
+
 @pytest.mark.asyncio
 async def test_evaluate_consistency_merges_compare_and_post_check_ignore_fields() -> None:
     project_strategy = _StubStrategy("project")

tests/unit/test_project_detail_api_handler.py

@@ -96,7 +96,7 @@ async def test_project_detail_create_failure_preserves_push_id_and_biz_id() -> N
     assert result.metadata["key"] == "production"
 
 
-def test_project_detail_load_conflict_raises_on_same_project_key_different_data_id() -> None:
+def test_project_detail_load_allows_same_project_key_with_different_data_id() -> None:
     collection = DataCollection("remote")
 
     project = _ProjectNode(
@@ -119,16 +119,20 @@ def test_project_detail_load_conflict_raises_on_same_project_key_different_data_
     )
     collection._nodes[existing_detail.node_id] = existing_detail
     collection._data_id_to_node_id[existing_detail.data_id] = existing_detail.node_id
+    collection._node_id_to_data_id[existing_detail.node_id] = existing_detail.data_id
 
     handler = ProjectDetailApiHandler()
     handler.set_collection(collection)
 
-    with pytest.raises(RuntimeError, match="project_detail_data load conflict"):
-        handler._create_node(
-            {
-                "id": "detail-new",
-                "project_id": "project-1",
-                "key": "actual_construction",
-                "value": [{"date": "2026-01-02", "capacity": 2.0}],
-            }
-        )
+    node = handler._create_node(
+        {
+            "id": "detail-new",
+            "project_id": "project-1",
+            "key": "actual_construction",
+            "value": [{"date": "2026-01-02", "capacity": 2.0}],
+        }
+    )
+
+    assert node.node_id != existing_detail.node_id
+    assert node.data_id == "detail-new"
+    assert node.context["project_id"] == "project-1"