解决了部分detail load效率问题和load失败终止pipeline问题

2026-04-02 14:35:28 +08:00
parent bb8ea931b5
commit 74df53889c
14 changed files with 626 additions and 64 deletions
@@ -1,6 +1,8 @@
 from __future__ import annotations

 import asyncio
+import hmac
+import hashlib
 import logging
 import time

@@ -24,9 +26,11 @@ class _FakeAsyncClient:
    def __init__(self) -> None:
        self.call_times: list[float] = []
        self.closed = False
+        self.last_kwargs: dict | None = None

    async def request(self, **kwargs):
        self.call_times.append(time.monotonic())
+        self.last_kwargs = kwargs
        return _FakeResponse()

    async def aclose(self) -> None:
@@ -114,3 +118,49 @@ async def test_api_client_logs_one_line_request_summary_in_debug_mode(caplog: py
        await client.request("GET", "/ping")

    assert "🔎 ApiClient request: method=GET endpoint=/ping elapsed=" in caplog.text
+
+
+@pytest.mark.asyncio
+async def test_api_client_get_signature_keeps_list_params_in_signature_source() -> None:
+    client = ApiClient(
+        base_url="http://example.com",
+        uid="uid",
+        secret="secret",
+    )
+    fake_client = _FakeAsyncClient()
+    client._client = fake_client
+
+    await client.request(
+        "GET",
+        "/ping",
+        params={"domains": ["a", "b", "c"], "project_id": "project-1"},
+    )
+
+    assert fake_client.last_kwargs is not None
+    sent_params = fake_client.last_kwargs["params"]
+    assert sent_params["domains"] == ["a", "b", "c"]
+
+    sign = sent_params["sign"]
+    sign_source = {k: v for k, v in sent_params.items() if k != "sign"}
+    items = []
+    for key, value in sorted(sign_source.items()):
+        if value is None:
+            value_str = "null"
+        elif isinstance(value, bool):
+            value_str = "true" if value else "false"
+        elif isinstance(value, str):
+            value_str = value
+        elif isinstance(value, (int, float)):
+            value_str = str(value)
+        elif isinstance(value, (list, dict)):
+            value_str = __import__("json").dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+        else:
+            value_str = str(value)
+        items.append(f"{key}={value_str}")
+
+    expected_sign = hmac.new(
+        b"secret",
+        "&".join(items).encode("utf-8"),
+        hashlib.sha256,
+    ).hexdigest()
+    assert sign == expected_sign
@@ -108,3 +108,14 @@ async def test_load_all_continue_when_one_handler_fails(caplog) -> None:

    assert "[fail] 加载失败" in caplog.text
    assert "Loaded nodes for ok: 0" in caplog.text
+
+
+@pytest.mark.asyncio
+async def test_load_all_raise_on_error_stops_on_first_failure() -> None:
+    ds = _DS()
+    ds.add_handler(_FailHandler())
+    ds.add_handler(_OkHandler())
+    ds.set_collection(DataCollection("local"))
+
+    with pytest.raises(RuntimeError, match=r"\[fail\] 加载失败: boom"):
+        await ds.load_all(order=["fail", "ok"], raise_on_error=True)
@@ -9,7 +9,7 @@ from sync_state_machine.common.binding import BindingManager
 from sync_state_machine.common.collection import DataCollection
 from sync_state_machine.datasource.jsonl import JsonlDataSource
 import sync_state_machine.pipeline.full_sync_pipeline as full_sync_pipeline_module
-from sync_state_machine.pipeline.full_sync_pipeline import FullSyncPipeline, WriteOutcome
+from sync_state_machine.pipeline.full_sync_pipeline import FullSyncPipeline, NodeTypeLoadError, WriteOutcome


 class _FakeDataSource:
@@ -19,13 +19,16 @@ class _FakeDataSource:
    def set_collection(self, collection) -> None:
        self.collection = collection

-    async def load_all(self, order=None, data_type=None) -> None:
-        del order, data_type
+    async def load_all(self, order=None, data_type=None, raise_on_error=False) -> None:
+        del order, data_type, raise_on_error

    async def sync_all(self, order=None, data_type=None, poll_async_tasks=True):
        del order, data_type, poll_async_tasks
        return {}

+    async def close(self) -> None:
+        return None
+

 class _StubStrategy:
    def __init__(self, node_type: str):
@@ -55,7 +58,11 @@ class _StubStrategy:


 class _FakePersistence:
+    def __init__(self) -> None:
+        self.close_calls = 0
+
    async def close(self) -> None:
+        self.close_calls += 1
        return None


@@ -175,8 +182,9 @@ async def test_reload_after_local_only_create_skips_remote_datasource() -> None:
            self.label = label
            self.load_calls: list[tuple[str | None, tuple[str, ...] | None]] = []

-        async def load_all(self, order=None, data_type=None) -> None:
+        async def load_all(self, order=None, data_type=None, raise_on_error=False) -> None:
            order_tuple = tuple(order) if order is not None else None
+            del raise_on_error
            self.load_calls.append((data_type, order_tuple))

    local_ds = _RecordingDataSource("local")
@@ -300,4 +308,44 @@ async def test_phase_post_check_skips_configured_node_types(monkeypatch: pytest.
        "checked_pairs": 0,
        "mismatch_count": 0,
        "skipped": True,
-    }
+    }
+
+
+@pytest.mark.asyncio
+async def test_run_persists_before_reraising_load_failure(monkeypatch: pytest.MonkeyPatch) -> None:
+    persistence = _FakePersistence()
+    pipeline = FullSyncPipeline(
+        local_datasource=_FakeDataSource(),
+        remote_datasource=_FakeDataSource(),
+        strategies=[_StubStrategy("project")],
+        persistence=persistence,
+        local_collection=DataCollection("local"),
+        remote_collection=DataCollection("remote"),
+        binding_manager=BindingManager(_FakePersistence(), auto_persist=False),
+        node_types=["project"],
+        load_order=["project"],
+        sync_order=["project"],
+        enable_persistence=True,
+    )
+
+    calls: list[str] = []
+
+    async def fake_bootstrap() -> None:
+        calls.append("bootstrap")
+
+    async def fake_sync() -> None:
+        calls.append("sync")
+        raise NodeTypeLoadError("load failed")
+
+    async def fake_persist() -> None:
+        calls.append("persist")
+
+    monkeypatch.setattr(pipeline, "phase_bootstrap", fake_bootstrap)
+    monkeypatch.setattr(pipeline, "phase_sync_by_node_type", fake_sync)
+    monkeypatch.setattr(pipeline, "phase_persistence", fake_persist)
+
+    with pytest.raises(NodeTypeLoadError, match="load failed"):
+        await pipeline.run()
+
+    assert calls == ["bootstrap", "sync", "persist"]
+    assert persistence.close_calls == 1
@@ -5,8 +5,10 @@ from typing import Any
 import pytest
 from pydantic import BaseModel

+from sync_state_machine.common.collection import DataCollection
 from sync_state_machine.common.sync_node import SyncNode
 from sync_state_machine.domain.material_detail.api_handler import MaterialDetailApiHandler
+from sync_state_machine.domain.project.api_handler import ProjectApiHandler
 from sync_state_machine.datasource.task_result import TaskStatus


@@ -25,6 +27,15 @@ class _MaterialDetailNode(SyncNode[_MaterialDetailSchema]):
    schema = _MaterialDetailSchema


+class _ProjectSchema(BaseModel):
+    id: str
+
+
+class _ProjectNode(SyncNode[_ProjectSchema]):
+    node_type = "project"
+    schema = _ProjectSchema
+
+
 class _MockApiClient:
    def __init__(self) -> None:
        self.requests: list[dict[str, Any]] = []
@@ -127,3 +138,97 @@ async def test_material_detail_update_failure_preserves_push_id_for_trace_lookup
    assert results[0].status == TaskStatus.FAILED
    assert results[0].push_id
    assert handler.api_client.requests[0]["endpoint"] == "/material/detail/update"
+
+
+@pytest.mark.asyncio
+async def test_material_detail_load_uses_project_aggregate_when_enabled(monkeypatch: pytest.MonkeyPatch) -> None:
+    collection = DataCollection("remote")
+    project = _ProjectNode(
+        node_id="project-node-1",
+        data_id="project-1",
+        data={"id": "project-1"},
+    )
+    await collection.add(project)
+
+    handler = MaterialDetailApiHandler()
+    handler.set_collection(collection)
+    handler.set_handler_config({"load_method": "aggregate"})
+    handler.api_client = _MockApiClient()
+
+    async def _fake_get_aggregate(client, project_id: str, *, domains: list[str], limit: int = 10000):
+        assert project_id == "project-1"
+        assert domains == ["material_details"]
+        assert limit == 10000
+        return {
+            "meta": {
+                "project_id": project_id,
+                "requested_domains": domains,
+                "domain_limit": limit,
+                "truncated_domains": [],
+            },
+            "data": {
+                "material_details": [
+                    {
+                        "id": "detail-1",
+                        "project_id": project_id,
+                        "parent_id": "material-1",
+                        "date": "2025-03-01",
+                        "quantity": 2.0,
+                        "remark": None,
+                        "is_complete": False,
+                    }
+                ],
+                "construction_details": [],
+                "contract_settlement_details": [],
+                "power_details": [],
+            },
+        }
+
+    monkeypatch.setattr(ProjectApiHandler, "get_aggregate", _fake_get_aggregate)
+
+    nodes = await handler.load()
+
+    assert len(nodes) == 1
+    assert nodes[0].data_id == "detail-1"
+    detail_data = nodes[0].get_data()
+    assert detail_data is not None
+    assert detail_data["project_id"] == "project-1"
+    assert handler.api_client.requests == []
+
+
+@pytest.mark.asyncio
+async def test_material_detail_load_aggregate_raises_on_truncated_domain(monkeypatch: pytest.MonkeyPatch) -> None:
+    collection = DataCollection("remote")
+    project = _ProjectNode(
+        node_id="project-node-1",
+        data_id="project-1",
+        data={"id": "project-1"},
+    )
+    await collection.add(project)
+
+    handler = MaterialDetailApiHandler()
+    handler.set_collection(collection)
+    handler.set_handler_config({"load_method": "aggregate"})
+    handler.api_client = _MockApiClient()
+
+    async def _fake_get_aggregate(client, project_id: str, *, domains: list[str], limit: int = 10000):
+        assert domains == ["material_details"]
+        return {
+            "meta": {
+                "project_id": project_id,
+                "requested_domains": domains,
+                "domain_limit": limit,
+                "truncated_domains": ["material_details"],
+            },
+            "data": {
+                "material_details": [],
+                "construction_details": [],
+                "contract_settlement_details": [],
+                "power_details": [],
+            },
+        }
+
+    monkeypatch.setattr(ProjectApiHandler, "get_aggregate", _fake_get_aggregate)
+
+    with pytest.raises(RuntimeError, match="truncated domain material_details"):
+        await handler.load()
@@ -9,7 +9,7 @@ from sync_state_machine.common.collection import DataCollection
 from sync_state_machine.common.persistence import PersistenceBackend
 from sync_state_machine.common.sqlite_backend import SQLitePersistenceBackend
 from sync_state_machine.datasource.datasource import BaseDataSource
-from sync_state_machine.pipeline.full_sync_pipeline import FullSyncPipeline
+from sync_state_machine.pipeline.full_sync_pipeline import FullSyncPipeline, NodeTypeLoadError
 from sync_state_machine.sync_system.strategy import DefaultSyncStrategy


@@ -141,3 +141,47 @@ async def test_phase_sync_by_node_type_skip_sync_keeps_bind_but_skips_writes(tmp
    assert strategy.update_calls == 0

    await pipeline.close()
+
+
+@pytest.mark.asyncio
+async def test_phase_sync_by_node_type_aborts_on_load_failure(tmp_path: Path) -> None:
+    db_path = tmp_path / "state.db"
+    persistence = SQLitePersistenceBackend(str(db_path))
+    await persistence.initialize()
+
+    local_collection = DataCollection("local", persistence=persistence, auto_persist=False)
+    remote_collection = DataCollection("remote", persistence=persistence, auto_persist=False)
+    binding_manager = BindingManager(persistence, auto_persist=False)
+
+    blocked_strategy = _OkStrategy("blocked", local_collection, remote_collection, binding_manager)
+    downstream_strategy = _OkStrategy("downstream", local_collection, remote_collection, binding_manager)
+    downstream_strategy.skip_sync = True
+
+    pipeline = FullSyncPipeline(
+        local_datasource=_DS(),
+        remote_datasource=_DS(),
+        strategies=[blocked_strategy, downstream_strategy],
+        persistence=persistence,
+        local_collection=local_collection,
+        remote_collection=remote_collection,
+        binding_manager=binding_manager,
+        node_types=["blocked", "downstream"],
+        load_order=["blocked", "downstream"],
+        sync_order=["blocked", "downstream"],
+        enable_persistence=False,
+    )
+
+    async def _fail_load(node_type: str, reason: str = "") -> None:
+        del reason
+        if node_type == "blocked":
+            raise NodeTypeLoadError("Load failed for node_type=blocked, datasource=remote, reason=pre-strategy refresh: boom")
+
+    pipeline._load_node_type = _fail_load  # type: ignore[method-assign]
+
+    with pytest.raises(NodeTypeLoadError, match="node_type=blocked"):
+        await pipeline.phase_sync_by_node_type()
+
+    assert blocked_strategy.bound is False
+    assert downstream_strategy.bound is False
+
+    await pipeline.close()
@@ -94,3 +94,65 @@ async def test_project_api_handler_load_requires_api_client() -> None:

    with pytest.raises(RuntimeError, match="API client not set"):
        await handler.load()
+
+
+class _AggregateRecordingClient:
+    def __init__(self) -> None:
+        self.calls: list[dict] = []
+
+    async def request(self, method, endpoint, params=None, data=None, **kwargs):
+        self.calls.append(
+            {
+                "method": method,
+                "endpoint": endpoint,
+                "params": params,
+                "data": data,
+                "kwargs": kwargs,
+            }
+        )
+        return {
+            "code": 200,
+            "message": "OK",
+            "data": {
+                "meta": {
+                    "project_id": data["project_id"],
+                    "requested_domains": data["domains"],
+                    "domain_limit": data["limit"],
+                    "truncated_domains": [],
+                },
+                "data": {
+                    "material_details": [],
+                    "construction_details": [],
+                    "contract_settlement_details": [],
+                    "power_details": [],
+                },
+            },
+        }
+
+
+@pytest.mark.asyncio
+async def test_project_aggregate_requests_are_not_shared_across_calls() -> None:
+    ProjectApiHandler.clear_cache()
+    client = _AggregateRecordingClient()
+
+    first = await ProjectApiHandler.get_aggregate(
+        client,
+        "project-1",
+        domains=["material_details"],
+    )
+    second = await ProjectApiHandler.get_aggregate(
+        client,
+        "project-1",
+        domains=["material_details"],
+    )
+
+    assert first["meta"]["project_id"] == "project-1"
+    assert second["meta"]["project_id"] == "project-1"
+    assert len(client.calls) == 2
+    assert client.calls[0]["method"] == "POST"
+    assert client.calls[0]["endpoint"] == "/project/aggregate"
+    assert client.calls[0]["data"]["project_id"] == "project-1"
+    assert client.calls[0]["data"]["domains"] == ["material_details"]
+    assert client.calls[1]["data"]["domains"] == ["material_details"]
+
+    ProjectApiHandler.clear_cache()