优化性能

2026-04-01 18:13:18 +08:00
parent 9a3a34c58b
commit f93a6c5c68
8 changed files with 198 additions and 107 deletions
@@ -6,6 +6,7 @@ from __future__ import annotations

 import asyncio
 import logging
+from dataclasses import dataclass
 from typing import Any, Dict, List, Optional, TYPE_CHECKING
 from pathlib import Path

@@ -30,6 +31,16 @@ from .summary_report import print_pipeline_summary
 logger = get_logger(__name__)


+@dataclass(frozen=True)
+class WriteOutcome:
+    local_written: bool = False
+    remote_written: bool = False
+
+    @property
+    def any_written(self) -> bool:
+        return self.local_written or self.remote_written
+
+
 class FullSyncPipeline:
    """
    Full pipeline phases:
@@ -163,10 +174,15 @@ class FullSyncPipeline:
                created_remote_node_ids = {
                    node.node_id for node in created if self.remote_collection.get_by_node_id(node.node_id) is not None
                }
-                create_written = await self.commit_creates(node_type)
+                create_outcome = await self.commit_creates(node_type)
                await self.normalize_create_success_to_update_entry(node_type)
-                if create_written:
-                    await self._reload_node_type(node_type, reason="create wrote data")
+                if create_outcome.any_written:
+                    await self._reload_node_type_after_write(
+                        node_type,
+                        reason="create wrote data",
+                        reload_local=create_outcome.local_written,
+                        reload_remote=create_outcome.remote_written,
+                    )
                    await refresh_bind_data_id(
                        node_type=node_type,
                        local_collection=self.local_collection,
@@ -178,9 +194,14 @@ class FullSyncPipeline:

                updated = await strategy.update()
                self._logger.info(f"✅ Strategy update: {node_type} (updated={len(updated)})")
-                update_written = await self.commit_updates(node_type)
-                if update_written:
-                    await self._reload_node_type(node_type, reason="update wrote data")
+                update_outcome = await self.commit_updates(node_type)
+                if update_outcome.any_written:
+                    await self._reload_node_type_after_write(
+                        node_type,
+                        reason="update wrote data",
+                        reload_local=update_outcome.local_written,
+                        reload_remote=update_outcome.remote_written,
+                    )

                self._logger.info(f"✅ Strategy done: {node_type}")
            except Exception as exc:
@@ -233,10 +254,15 @@ class FullSyncPipeline:
                created_remote_node_ids = {
                    node.node_id for node in created if self.remote_collection.get_by_node_id(node.node_id) is not None
                }
-                create_written = await self.commit_creates(node_type)
+                create_outcome = await self.commit_creates(node_type)
                await self.normalize_create_success_to_update_entry(node_type)
-                if create_written:
-                    await self._reload_node_type(node_type, reason="create wrote data")
+                if create_outcome.any_written:
+                    await self._reload_node_type_after_write(
+                        node_type,
+                        reason="create wrote data",
+                        reload_local=create_outcome.local_written,
+                        reload_remote=create_outcome.remote_written,
+                    )
                    await refresh_bind_data_id(
                        node_type=node_type,
                        local_collection=self.local_collection,
@@ -263,9 +289,14 @@ class FullSyncPipeline:
            try:
                updated = await strategy.update()
                self._logger.info(f"✅ Strategy update: {node_type} (updated={len(updated)})")
-                update_written = await self.commit_updates(node_type)
-                if update_written:
-                    await self._reload_node_type(node_type, reason="update wrote data")
+                update_outcome = await self.commit_updates(node_type)
+                if update_outcome.any_written:
+                    await self._reload_node_type_after_write(
+                        node_type,
+                        reason="update wrote data",
+                        reload_local=update_outcome.local_written,
+                        reload_remote=update_outcome.remote_written,
+                    )
                await self._evaluate_consistency_for_node_type(node_type)
            except Exception as exc:
                self.stats["failed"] += 1
@@ -410,6 +441,33 @@ class FullSyncPipeline:
            return
        await self._load_node_type(node_type, reason=reason)

+    async def _reload_node_type_after_write(
+        self,
+        node_type: str,
+        *,
+        reason: str,
+        reload_local: bool,
+        reload_remote: bool,
+    ) -> None:
+        if not reload_local and not reload_remote:
+            return
+
+        if self._should_skip_reload():
+            self._logger.info("⏭️  Reload skipped for JSONL pipeline: node_type=%s reason=%s", node_type, reason)
+            return
+
+        self._logger.info(
+            "🔄 Reload node_type=%s after write, reason=%s, local=%s, remote=%s",
+            node_type,
+            reason,
+            reload_local,
+            reload_remote,
+        )
+        if reload_local:
+            await self.local_datasource.load_all(order=[node_type])
+        if reload_remote:
+            await self.remote_datasource.load_all(order=[node_type])
+
    async def _evaluate_consistency_for_node_type(self, node_type: str) -> Dict[str, Any]:
        strategy = next((item for item in self.strategies if item.node_type == node_type), None)
        if strategy is None:
@@ -439,7 +497,7 @@ class FullSyncPipeline:
        self._consistency_by_type[node_type] = result
        return result

-    async def commit_creates(self, node_type: str) -> bool:
+    async def commit_creates(self, node_type: str) -> WriteOutcome:
        """提交 CREATE 操作（包含异步轮询）"""
        # 执行同步（sync_all 内部会根据 action 筛选节点）
        await self.local_datasource.sync_all(
@@ -464,8 +522,10 @@ class FullSyncPipeline:

        local_success = self._get_action_success_count(self.local_datasource, node_type, "create")
        remote_success = self._get_action_success_count(self.remote_datasource, node_type, "create")
-        wrote = (local_success + remote_success) > 0
-        return wrote
+        return WriteOutcome(
+            local_written=local_success > 0,
+            remote_written=remote_success > 0,
+        )

    async def _finalize_peer_creating_sources(self, node_type: str) -> None:
        finalized_success, finalized_failed = await finalize_peer_creating_sources(
@@ -502,7 +562,7 @@ class FullSyncPipeline:
        if normalized > 0:
            self._logger.info(f"🔁 Post-create normalization: {normalized} nodes moved S11C->S01 for {node_type}")
    
-    async def commit_updates(self, node_type: str) -> bool:
+    async def commit_updates(self, node_type: str) -> WriteOutcome:
        """提交 UPDATE 操作（包含异步轮询）"""
        # 执行同步（sync_all 内部会根据 action 筛选节点）
        await self.local_datasource.sync_all(
@@ -525,8 +585,10 @@ class FullSyncPipeline:

        local_success = self._get_action_success_count(self.local_datasource, node_type, "update")
        remote_success = self._get_action_success_count(self.remote_datasource, node_type, "update")
-        wrote = (local_success + remote_success) > 0
-        return wrote
+        return WriteOutcome(
+            local_written=local_success > 0,
+            remote_written=remote_success > 0,
+        )

    async def phase_post_check(self) -> bool:
        """同步结果报告。