解决了部分detail load效率问题和load失败终止pipeline问题

sync_state_machine/pipeline/full_sync_pipeline.py

@@ -31,6 +31,10 @@ from .summary_report import print_pipeline_summary
 logger = get_logger(__name__)
 
 
+class NodeTypeLoadError(RuntimeError):
+    """Raised when a datasource load for a node type fails and the pipeline must stop."""
+
+
 @dataclass(frozen=True)
 class WriteOutcome:
     local_written: bool = False
@@ -127,6 +131,7 @@ class FullSyncPipeline:
         return StateMachineRuntime(StateMachineConfig.load(cfg_path))
 
     async def run(self) -> Dict[str, Any]:
+        persistence_started = False
         try:
             await self.phase_bootstrap()
             self._logger.info("✅ Phase bootstrap completed (%s)", self.pipeline_name)
@@ -138,12 +143,41 @@ class FullSyncPipeline:
             self.stats["post_check_passed"] = post_ok
             self._logger.info("✅ Phase post-check completed (%s)", self.pipeline_name)
 
+            persistence_started = True
             await self.phase_persistence()
             self._logger.info("✅ Phase persistence completed (%s)", self.pipeline_name)
             return self.stats
+        except Exception as exc:
+            if not persistence_started:
+                await self._persist_after_failure(exc)
+            raise
         finally:
             await self.close()
 
+    async def _persist_after_failure(self, cause: Exception) -> None:
+        if not self.enable_persistence:
+            self._logger.info(
+                "⏭️  Persistence disabled after pipeline failure: %s (%s)",
+                type(cause).__name__,
+                self.pipeline_name,
+            )
+            return
+
+        self._logger.info(
+            "🔄 Pipeline failed before persistence, persisting current state: %s | %s",
+            type(cause).__name__,
+            cause,
+        )
+        try:
+            await self.phase_persistence()
+            self._logger.info("✅ Phase persistence completed after failure (%s)", self.pipeline_name)
+        except Exception as persist_exc:
+            self._logger.error(
+                "❌ Phase persistence failed after pipeline failure: %s | %s",
+                type(persist_exc).__name__,
+                persist_exc,
+            )
+
     async def phase_sync_by_node_type(self) -> None:
         self._prepare_datasources()
         strategy_map = {s.node_type: s for s in self.strategies}
@@ -159,6 +193,7 @@ class FullSyncPipeline:
 
                 await self._load_node_type(node_type, reason="pre-strategy refresh")
 
+                # Load failure is a hard stop: create/update cannot safely treat missing data as empty data.
                 await strategy.bind()
                 self._logger.info(f"✅ Strategy bind: {node_type}")
 
@@ -204,6 +239,8 @@ class FullSyncPipeline:
                     )
 
                 self._logger.info(f"✅ Strategy done: {node_type}")
+            except NodeTypeLoadError:
+                raise
             except Exception as exc:
                 self.stats["failed"] += 1
                 self._logger.error(f"❌ Strategy failed but pipeline continues: {node_type} | {type(exc).__name__}: {exc}")
@@ -417,8 +454,18 @@ class FullSyncPipeline:
 
     async def _load_node_type(self, node_type: str, *, reason: str) -> None:
         self._logger.info(f"🔄 Load node_type={node_type}, reason={reason}")
-        await self.local_datasource.load_all(order=[node_type])
-        await self.remote_datasource.load_all(order=[node_type])
+        await self._load_node_type_from_datasource(
+            self.local_datasource,
+            datasource_role="local",
+            node_type=node_type,
+            reason=reason,
+        )
+        await self._load_node_type_from_datasource(
+            self.remote_datasource,
+            datasource_role="remote",
+            node_type=node_type,
+            reason=reason,
+        )
 
         if node_type in self._loaded_node_types:
             return
@@ -427,6 +474,21 @@ class FullSyncPipeline:
         self.stats["loaded"] += len(self.local_collection.filter(node_type=node_type))
         self.stats["loaded"] += len(self.remote_collection.filter(node_type=node_type))
 
+    async def _load_node_type_from_datasource(
+        self,
+        datasource: "BaseDataSource",
+        *,
+        datasource_role: str,
+        node_type: str,
+        reason: str,
+    ) -> None:
+        try:
+            await datasource.load_all(order=[node_type], raise_on_error=True)
+        except Exception as exc:
+            raise NodeTypeLoadError(
+                f"Load failed for node_type={node_type}, datasource={datasource_role}, reason={reason}: {exc}"
+            ) from exc
+
     def _get_action_success_count(self, datasource: "BaseDataSource", node_type: str, action_key: str) -> int:
         summary = datasource.get_action_summary().get(node_type, {})
         action_summary = summary.get(action_key, {}) if isinstance(summary, dict) else {}
@@ -464,9 +526,19 @@ class FullSyncPipeline:
             reload_remote,
         )
         if reload_local:
-            await self.local_datasource.load_all(order=[node_type])
+            await self._load_node_type_from_datasource(
+                self.local_datasource,
+                datasource_role="local",
+                node_type=node_type,
+                reason=reason,
+            )
         if reload_remote:
-            await self.remote_datasource.load_all(order=[node_type])
+            await self._load_node_type_from_datasource(
+                self.remote_datasource,
+                datasource_role="remote",
+                node_type=node_type,
+                reason=reason,
+            )
 
     async def _evaluate_consistency_for_node_type(self, node_type: str) -> Dict[str, Any]:
         strategy = next((item for item in self.strategies if item.node_type == node_type), None)