strepsiades/ecm_sync_system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

解决了部分detail load效率问题和load失败终止pipeline问题

Browse Source
This commit is contained in:
strepsiades
2026-04-02 14:35:28 +08:00
parent bb8ea931b5
commit 74df53889c
14 changed files with 626 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sync_state_machine/datasource/api/client.py
+2 -2
View File
@@ -435,10 +435,10 @@ class ApiClient:
if params is None:
params = {}
params['uid'] = self._uid
# 转换所有参数值为字符串（除了 list/dict）
params = {k: str(v) if not isinstance(v, (list, dict)) else v for k, v in params.items()}
sign_params = params.copy()
# 生成签名并附加到 params
signature = self._generate_signature(sign_params)
sync_state_machine/datasource/datasource.py
+5 -1
View File
@@ -419,7 +419,9 @@ class BaseDataSource(ABC):
async def load_all(
self,
order: Optional[List[str]] = None,
data_type: Optional[str] = None
data_type: Optional[str] = None,
*,
raise_on_error: bool = False,
) -> None:
"""
按依赖顺序加载所有数据
@@ -470,6 +472,8 @@ class BaseDataSource(ABC):
self._stats[node_type]["loaded"] += len(nodes)
except Exception as exc:
logger.error("❌ [%s] 加载失败: %s", node_type, exc)
if raise_on_error:
raise RuntimeError(f"[{node_type}] 加载失败: {exc}") from exc
continue
# ========== 同步执行 ==========
sync_state_machine/domain/construction_task_detail/api_handler.py
+41 -9
View File
@@ -17,39 +17,71 @@ from schemas.construction.construction_detail import (
ConstructionDetailCreate,
ConstructionDetailUpdate
)
from ..project.api_handler import ProjectApiHandler
class ConstructionTaskDetailApiHandler(BaseApiHandler):
"""施工明细 API Handler"""
def __init__(self):
super().__init__()
def __init__(self, **handler_config: Any):
super().__init__(**handler_config)
self._node_type = "construction_task_detail"
self._node_class = ConstructionTaskDetailSyncNode
self._schema = ConstructionDetailResponse
self.update_schemas = [ConstructionDetailCreate, ConstructionDetailUpdate]
async def load(self) -> List[SyncNode]:
"""加载施工明细列表（依赖 ConstructionTask）"""
def _use_aggregate_load(self) -> bool:
return str(self.handler_config.get("load_method", "legacy")).lower() == "aggregate"
async def _load_via_project_aggregate(self) -> List[SyncNode]:
projects = self._collection.filter(node_type="project")
project_ids = [project.data_id for project in projects if project.data_id]
if not project_ids:
return []
all_details: List[SyncNode] = []
aggregate_limit = int(self.handler_config.get("aggregate_limit", 10000))
for project_id in project_ids:
aggregate = await ProjectApiHandler.get_aggregate(
self.api_client,
project_id,
domains=["construction_details"],
limit=aggregate_limit,
)
truncated = set(aggregate.get("meta", {}).get("truncated_domains", []))
if "construction_details" in truncated:
raise RuntimeError(
f"project aggregate truncated domain construction_details for project {project_id}"
)
for detail_data in aggregate.get("data", {}).get("construction_details", []):
all_details.append(self._create_node(dict(detail_data)))
return all_details
async def _load_via_legacy_list(self) -> List[SyncNode]:
tasks = self._collection.filter(node_type="construction_task")
task_ids = [t.data_id for t in tasks if t.data_id]
if not task_ids:
return []
all_details = []
for task_id in task_ids:
try:
details = await api_get_construction_task_detail_list(self.api_client, task_id)
for detail_model in details:
# 已验证的 Pydantic model，转换为 dict
detail_data = detail_model.model_dump()
all_details.append(self._create_node(detail_data))
except Exception as e:
print(f"❌ Failed to load construction details for task {task_id}: {e}")
return all_details
async def load(self) -> List[SyncNode]:
"""加载施工明细列表（依赖 ConstructionTask）"""
if self._use_aggregate_load():
return await self._load_via_project_aggregate()
return await self._load_via_legacy_list()
async def create_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
"""批量创建施工明细"""
from pydantic import ValidationError
sync_state_machine/domain/contract_settlement_detail/api_handler.py
+41 -8
View File
@@ -17,26 +17,53 @@ from schemas.contract_settlement.contract_settlement_detail import (
ContractSettlementDetailCreate,
ContractSettlementDetailUpdate
)
from ..project.api_handler import ProjectApiHandler
class ContractSettlementDetailApiHandler(BaseApiHandler):
"""合同结算明细 API Handler"""
def __init__(self):
super().__init__()
def __init__(self, **handler_config: Any):
super().__init__(**handler_config)
self._node_type = "contract_settlement_detail"
self._node_class = ContractSettlementDetailSyncNode
self._schema = ContractSettlementDetailResponse
self.update_schemas = [ContractSettlementDetailCreate, ContractSettlementDetailUpdate]
async def load(self) -> List[SyncNode]:
"""加载合同结算明细列表（依赖 ContractSettlement）"""
def _use_aggregate_load(self) -> bool:
return str(self.handler_config.get("load_method", "legacy")).lower() == "aggregate"
async def _load_via_project_aggregate(self) -> List[SyncNode]:
projects = self._collection.filter(node_type="project")
project_ids = [project.data_id for project in projects if project.data_id]
if not project_ids:
return []
all_details: List[SyncNode] = []
aggregate_limit = int(self.handler_config.get("aggregate_limit", 10000))
for project_id in project_ids:
aggregate = await ProjectApiHandler.get_aggregate(
self.api_client,
project_id,
domains=["contract_settlement_details"],
limit=aggregate_limit,
)
truncated = set(aggregate.get("meta", {}).get("truncated_domains", []))
if "contract_settlement_details" in truncated:
raise RuntimeError(
f"project aggregate truncated domain contract_settlement_details for project {project_id}"
)
for detail_data in aggregate.get("data", {}).get("contract_settlement_details", []):
all_details.append(self._create_node(dict(detail_data)))
return all_details
async def _load_via_legacy_list(self) -> List[SyncNode]:
settlements = self._collection.filter(node_type="contract_settlement")
settlement_ids = [s.data_id for s in settlements if s.data_id]
if not settlement_ids:
return []
all_details = []
for settlement_id in settlement_ids:
try:
@@ -46,9 +73,15 @@ class ContractSettlementDetailApiHandler(BaseApiHandler):
all_details.append(self._create_node(detail_data))
except Exception as e:
print(f"❌ Failed to load settlement details for {settlement_id}: {e}")
return all_details
async def load(self) -> List[SyncNode]:
"""加载合同结算明细列表（依赖 ContractSettlement）"""
if self._use_aggregate_load():
return await self._load_via_project_aggregate()
return await self._load_via_legacy_list()
async def create_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
"""批量创建合同结算明细"""
from pydantic import ValidationError
sync_state_machine/domain/material_detail/api_handler.py
+54 -26
View File
@@ -32,44 +32,72 @@ from schemas.material.material_detail import (
MaterialDetailCreate,
MaterialDetailUpdate
)
from ..project.api_handler import ProjectApiHandler
class MaterialDetailApiHandler(BaseApiHandler):
"""物资明细 API Handler"""
def __init__(self):
super().__init__()
def __init__(self, **handler_config: Any):
super().__init__(**handler_config)
self._node_type = "material_detail"
self._node_class = MaterialDetailSyncNode
self._schema = MaterialDetailResponse
self.update_schemas = [MaterialDetailCreate, MaterialDetailUpdate]
def _use_aggregate_load(self) -> bool:
return str(self.handler_config.get("load_method", "legacy")).lower() == "aggregate"
async def _load_via_project_aggregate(self) -> List[SyncNode]:
projects = self._collection.filter(node_type="project")
project_ids = [project.data_id for project in projects if project.data_id]
if not project_ids:
return []
all_nodes: List[SyncNode] = []
aggregate_limit = int(self.handler_config.get("aggregate_limit", 10000))
for project_id in project_ids:
aggregate = await ProjectApiHandler.get_aggregate(
self.api_client,
project_id,
domains=["material_details"],
limit=aggregate_limit,
)
truncated = set(aggregate.get("meta", {}).get("truncated_domains", []))
if "material_details" in truncated:
raise RuntimeError(
f"project aggregate truncated domain material_details for project {project_id}"
)
for detail_data in aggregate.get("data", {}).get("material_details", []):
node = self._create_node(dict(detail_data))
all_nodes.append(node)
return all_nodes
async def _load_via_legacy_list(self) -> List[SyncNode]:
materials = self._collection.filter(node_type="material")
material_ids = [m.data_id for m in materials if m.data_id]
if not material_ids:
return []
all_nodes = []
for material_id in material_ids:
try:
details = await api_get_material_detail_list(self.api_client, material_id)
for detail_model in details:
detail_data = detail_model.model_dump()
node = self._create_node(detail_data)
all_nodes.append(node)
except Exception as e:
print(f"❌ Failed to load material details for {material_id}: {e}")
return all_nodes
async def load(self) -> List[SyncNode]:
"""加载物资明细列表（依赖 Material）"""
try:
materials = self._collection.filter(node_type="material")
material_ids = [m.data_id for m in materials if m.data_id]
if not material_ids:
return []
all_nodes = []
for material_id in material_ids:
try:
details = await api_get_material_detail_list(self.api_client, material_id)
for detail_model in details:
# 已验证的 Pydantic model，转换为 dict
detail_data = detail_model.model_dump()
node = self._create_node(detail_data)
all_nodes.append(node)
except Exception as e:
print(f"❌ Failed to load material details for {material_id}: {e}")
return all_nodes
except Exception as e:
print(f"❌ Failed to load material details: {e}")
return []
if self._use_aggregate_load():
return await self._load_via_project_aggregate()
return await self._load_via_legacy_list()
async def create_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
"""批量创建物资明细"""
sync_state_machine/domain/personnel_detail/api_handler.py
+42 -8
View File
@@ -13,26 +13,54 @@ from ...common.sync_node import SyncNode
from ...datasource.api.errors import ERROR_NODE_DATA_NONE, ERROR_ID_NOT_FOUND
from .sync_node import PersonnelDetailSyncNode
from schemas.power.power_detail import PowerDetail, PowerDetailCreate, PowerDetailUpdate
from ..project.api_handler import ProjectApiHandler
class PersonnelDetailApiHandler(BaseApiHandler):
"""人员明细 API Handler"""
def __init__(self):
super().__init__()
def __init__(self, **handler_config: Any):
super().__init__(**handler_config)
self._node_type = "personnel_detail"
self._node_class = PersonnelDetailSyncNode
self._schema = PowerDetail
self.update_schemas = [PowerDetailCreate, PowerDetailUpdate]
async def load(self) -> List[SyncNode]:
"""加载人员明细列表（依赖 Personnel）"""
def _use_aggregate_load(self) -> bool:
return str(self.handler_config.get("load_method", "legacy")).lower() == "aggregate"
async def _load_via_project_aggregate(self) -> List[SyncNode]:
projects = self._collection.filter(node_type="project")
project_ids = [project.data_id for project in projects if project.data_id]
if not project_ids:
return []
all_details: List[SyncNode] = []
aggregate_limit = int(self.handler_config.get("aggregate_limit", 10000))
for project_id in project_ids:
aggregate = await ProjectApiHandler.get_aggregate(
self.api_client,
project_id,
domains=["power_details"],
limit=aggregate_limit,
)
truncated = set(aggregate.get("meta", {}).get("truncated_domains", []))
if "power_details" in truncated:
raise RuntimeError(
f"project aggregate truncated domain power_details for project {project_id}"
)
for detail_data in aggregate.get("data", {}).get("power_details", []):
validated = PowerDetail.model_validate(detail_data)
all_details.append(self._create_node(validated.model_dump()))
return all_details
async def _load_via_legacy_list(self) -> List[SyncNode]:
personnel_list = self._collection.filter(node_type="personnel")
personnel_ids = [p.data_id for p in personnel_list if p.data_id]
if not personnel_ids:
return []
all_details = []
for personnel_id in personnel_ids:
try:
@@ -42,9 +70,15 @@ class PersonnelDetailApiHandler(BaseApiHandler):
all_details.append(self._create_node(validated.model_dump()))
except Exception as e:
print(f"❌ Failed to load personnel details for {personnel_id}: {e}")
return all_details
async def load(self) -> List[SyncNode]:
"""加载人员明细列表（依赖 Personnel）"""
if self._use_aggregate_load():
return await self._load_via_project_aggregate()
return await self._load_via_legacy_list()
async def create_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
"""批量创建人员明细"""
results = []
sync_state_machine/domain/project/api_handler.py
+39
View File
@@ -34,6 +34,7 @@ from schemas.project.project_base import (
ProjectSettlementInvestment
)
from schemas.project.project import ProjectInfoResponse
from schemas.project.project_aggregate import ProjectAggregateResponse
_PROJECT_EXTENSION_FIELD_NAMES = {
@@ -94,6 +95,23 @@ class ProjectApiHandler(BaseApiHandler):
all_info = await api_get_project_all_info(client, project_id)
cls._all_info_cache[project_id] = all_info
return all_info
@classmethod
async def get_aggregate(
cls,
client,
project_id: str,
*,
domains: List[str],
limit: int = 10000,
) -> Dict[str, Any]:
normalized_domains = tuple(dict.fromkeys(domains))
return await api_get_project_aggregate(
client,
project_id,
domains=list(normalized_domains),
limit=limit,
)
@classmethod
def clear_cache(cls):
@@ -407,6 +425,27 @@ async def api_get_project_all_info(client, project_id: str) -> Dict[str, Any]:
return validated.model_dump()
async def api_get_project_aggregate(
client,
project_id: str,
*,
domains: List[str],
limit: int = 10000,
) -> Dict[str, Any]:
"""POST /project/aggregate - 获取项目聚合数据（校验为 ProjectAggregateResponse）"""
response = await client.request(
method="POST",
endpoint="/project/aggregate",
data={"project_id": project_id, "domains": domains, "limit": limit},
)
code = response.get("code")
if code != 200:
raise Exception(f"API error {code}: {response.get('message', 'Unknown error')}")
validated = ProjectAggregateResponse.model_validate(response.get("data", {}))
return validated.model_dump()
async def api_update_project_base_info(client, data: Dict[str, Any], push_id: str, biz_id: str, *, steps: List[Dict[str, Any]] | None = None) -> None:
"""PUT /project - 更新项目基本信息"""
ProjectBaseInfoUpdate.model_validate(data)
sync_state_machine/pipeline/full_sync_pipeline.py
+76 -4
View File
@@ -31,6 +31,10 @@ from .summary_report import print_pipeline_summary
logger = get_logger(__name__)
class NodeTypeLoadError(RuntimeError):
"""Raised when a datasource load for a node type fails and the pipeline must stop."""
@dataclass(frozen=True)
class WriteOutcome:
local_written: bool = False
@@ -127,6 +131,7 @@ class FullSyncPipeline:
return StateMachineRuntime(StateMachineConfig.load(cfg_path))
async def run(self) -> Dict[str, Any]:
persistence_started = False
try:
await self.phase_bootstrap()
self._logger.info("✅ Phase bootstrap completed (%s)", self.pipeline_name)
@@ -138,12 +143,41 @@ class FullSyncPipeline:
self.stats["post_check_passed"] = post_ok
self._logger.info("✅ Phase post-check completed (%s)", self.pipeline_name)
persistence_started = True
await self.phase_persistence()
self._logger.info("✅ Phase persistence completed (%s)", self.pipeline_name)
return self.stats
except Exception as exc:
if not persistence_started:
await self._persist_after_failure(exc)
raise
finally:
await self.close()
async def _persist_after_failure(self, cause: Exception) -> None:
if not self.enable_persistence:
self._logger.info(
"⏭️ Persistence disabled after pipeline failure: %s (%s)",
type(cause).__name__,
self.pipeline_name,
)
return
self._logger.info(
"🔄 Pipeline failed before persistence, persisting current state: %s | %s",
type(cause).__name__,
cause,
)
try:
await self.phase_persistence()
self._logger.info("✅ Phase persistence completed after failure (%s)", self.pipeline_name)
except Exception as persist_exc:
self._logger.error(
"❌ Phase persistence failed after pipeline failure: %s | %s",
type(persist_exc).__name__,
persist_exc,
)
async def phase_sync_by_node_type(self) -> None:
self._prepare_datasources()
strategy_map = {s.node_type: s for s in self.strategies}
@@ -159,6 +193,7 @@ class FullSyncPipeline:
await self._load_node_type(node_type, reason="pre-strategy refresh")
# Load failure is a hard stop: create/update cannot safely treat missing data as empty data.
await strategy.bind()
self._logger.info(f"✅ Strategy bind: {node_type}")
@@ -204,6 +239,8 @@ class FullSyncPipeline:
)
self._logger.info(f"✅ Strategy done: {node_type}")
except NodeTypeLoadError:
raise
except Exception as exc:
self.stats["failed"] += 1
self._logger.error(f"❌ Strategy failed but pipeline continues: {node_type} | {type(exc).__name__}: {exc}")
@@ -417,8 +454,18 @@ class FullSyncPipeline:
async def _load_node_type(self, node_type: str, *, reason: str) -> None:
self._logger.info(f"🔄 Load node_type={node_type}, reason={reason}")
await self.local_datasource.load_all(order=[node_type])
await self.remote_datasource.load_all(order=[node_type])
await self._load_node_type_from_datasource(
self.local_datasource,
datasource_role="local",
node_type=node_type,
reason=reason,
)
await self._load_node_type_from_datasource(
self.remote_datasource,
datasource_role="remote",
node_type=node_type,
reason=reason,
)
if node_type in self._loaded_node_types:
return
@@ -427,6 +474,21 @@ class FullSyncPipeline:
self.stats["loaded"] += len(self.local_collection.filter(node_type=node_type))
self.stats["loaded"] += len(self.remote_collection.filter(node_type=node_type))
async def _load_node_type_from_datasource(
self,
datasource: "BaseDataSource",
*,
datasource_role: str,
node_type: str,
reason: str,
) -> None:
try:
await datasource.load_all(order=[node_type], raise_on_error=True)
except Exception as exc:
raise NodeTypeLoadError(
f"Load failed for node_type={node_type}, datasource={datasource_role}, reason={reason}: {exc}"
) from exc
def _get_action_success_count(self, datasource: "BaseDataSource", node_type: str, action_key: str) -> int:
summary = datasource.get_action_summary().get(node_type, {})
action_summary = summary.get(action_key, {}) if isinstance(summary, dict) else {}
@@ -464,9 +526,19 @@ class FullSyncPipeline:
reload_remote,
)
if reload_local:
await self.local_datasource.load_all(order=[node_type])
await self._load_node_type_from_datasource(
self.local_datasource,
datasource_role="local",
node_type=node_type,
reason=reason,
)
if reload_remote:
await self.remote_datasource.load_all(order=[node_type])
await self._load_node_type_from_datasource(
self.remote_datasource,
datasource_role="remote",
node_type=node_type,
reason=reason,
)
async def _evaluate_consistency_for_node_type(self, node_type: str) -> Dict[str, Any]:
strategy = next((item for item in self.strategies if item.node_type == node_type), None)
tests/unit/test_api_client_rate_limit.py
+50
View File
@@ -1,6 +1,8 @@
from __future__ import annotations
import asyncio
import hmac
import hashlib
import logging
import time
@@ -24,9 +26,11 @@ class _FakeAsyncClient:
def __init__(self) -> None:
self.call_times: list[float] = []
self.closed = False
self.last_kwargs: dict | None = None
async def request(self, **kwargs):
self.call_times.append(time.monotonic())
self.last_kwargs = kwargs
return _FakeResponse()
async def aclose(self) -> None:
@@ -114,3 +118,49 @@ async def test_api_client_logs_one_line_request_summary_in_debug_mode(caplog: py
await client.request("GET", "/ping")
assert "🔎 ApiClient request: method=GET endpoint=/ping elapsed=" in caplog.text
@pytest.mark.asyncio
async def test_api_client_get_signature_keeps_list_params_in_signature_source() -> None:
client = ApiClient(
base_url="http://example.com",
uid="uid",
secret="secret",
)
fake_client = _FakeAsyncClient()
client._client = fake_client
await client.request(
"GET",
"/ping",
params={"domains": ["a", "b", "c"], "project_id": "project-1"},
)
assert fake_client.last_kwargs is not None
sent_params = fake_client.last_kwargs["params"]
assert sent_params["domains"] == ["a", "b", "c"]
sign = sent_params["sign"]
sign_source = {k: v for k, v in sent_params.items() if k != "sign"}
items = []
for key, value in sorted(sign_source.items()):
if value is None:
value_str = "null"
elif isinstance(value, bool):
value_str = "true" if value else "false"
elif isinstance(value, str):
value_str = value
elif isinstance(value, (int, float)):
value_str = str(value)
elif isinstance(value, (list, dict)):
value_str = __import__("json").dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
else:
value_str = str(value)
items.append(f"{key}={value_str}")
expected_sign = hmac.new(
b"secret",
"&".join(items).encode("utf-8"),
hashlib.sha256,
).hexdigest()
assert sign == expected_sign
tests/unit/test_datasource_load_continue_on_error.py
+11
View File
@@ -108,3 +108,14 @@ async def test_load_all_continue_when_one_handler_fails(caplog) -> None:
assert "[fail] 加载失败" in caplog.text
assert "Loaded nodes for ok: 0" in caplog.text
@pytest.mark.asyncio
async def test_load_all_raise_on_error_stops_on_first_failure() -> None:
ds = _DS()
ds.add_handler(_FailHandler())
ds.add_handler(_OkHandler())
ds.set_collection(DataCollection("local"))
with pytest.raises(RuntimeError, match=r"\[fail\] 加载失败: boom"):
await ds.load_all(order=["fail", "ok"], raise_on_error=True)
tests/unit/test_full_sync_pipeline_downstream_reload.py
+53 -5
View File
@@ -9,7 +9,7 @@ from sync_state_machine.common.binding import BindingManager
from sync_state_machine.common.collection import DataCollection
from sync_state_machine.datasource.jsonl import JsonlDataSource
import sync_state_machine.pipeline.full_sync_pipeline as full_sync_pipeline_module
from sync_state_machine.pipeline.full_sync_pipeline import FullSyncPipeline, WriteOutcome
from sync_state_machine.pipeline.full_sync_pipeline import FullSyncPipeline, NodeTypeLoadError, WriteOutcome
class _FakeDataSource:
@@ -19,13 +19,16 @@ class _FakeDataSource:
def set_collection(self, collection) -> None:
self.collection = collection
async def load_all(self, order=None, data_type=None) -> None:
del order, data_type
async def load_all(self, order=None, data_type=None, raise_on_error=False) -> None:
del order, data_type, raise_on_error
async def sync_all(self, order=None, data_type=None, poll_async_tasks=True):
del order, data_type, poll_async_tasks
return {}
async def close(self) -> None:
return None
class _StubStrategy:
def __init__(self, node_type: str):
@@ -55,7 +58,11 @@ class _StubStrategy:
class _FakePersistence:
def __init__(self) -> None:
self.close_calls = 0
async def close(self) -> None:
self.close_calls += 1
return None
@@ -175,8 +182,9 @@ async def test_reload_after_local_only_create_skips_remote_datasource() -> None:
self.label = label
self.load_calls: list[tuple[str | None, tuple[str, ...] | None]] = []
async def load_all(self, order=None, data_type=None) -> None:
async def load_all(self, order=None, data_type=None, raise_on_error=False) -> None:
order_tuple = tuple(order) if order is not None else None
del raise_on_error
self.load_calls.append((data_type, order_tuple))
local_ds = _RecordingDataSource("local")
@@ -300,4 +308,44 @@ async def test_phase_post_check_skips_configured_node_types(monkeypatch: pytest.
"checked_pairs": 0,
"mismatch_count": 0,
"skipped": True,
}
}
@pytest.mark.asyncio
async def test_run_persists_before_reraising_load_failure(monkeypatch: pytest.MonkeyPatch) -> None:
persistence = _FakePersistence()
pipeline = FullSyncPipeline(
local_datasource=_FakeDataSource(),
remote_datasource=_FakeDataSource(),
strategies=[_StubStrategy("project")],
persistence=persistence,
local_collection=DataCollection("local"),
remote_collection=DataCollection("remote"),
binding_manager=BindingManager(_FakePersistence(), auto_persist=False),
node_types=["project"],
load_order=["project"],
sync_order=["project"],
enable_persistence=True,
)
calls: list[str] = []
async def fake_bootstrap() -> None:
calls.append("bootstrap")
async def fake_sync() -> None:
calls.append("sync")
raise NodeTypeLoadError("load failed")
async def fake_persist() -> None:
calls.append("persist")
monkeypatch.setattr(pipeline, "phase_bootstrap", fake_bootstrap)
monkeypatch.setattr(pipeline, "phase_sync_by_node_type", fake_sync)
monkeypatch.setattr(pipeline, "phase_persistence", fake_persist)
with pytest.raises(NodeTypeLoadError, match="load failed"):
await pipeline.run()
assert calls == ["bootstrap", "sync", "persist"]
assert persistence.close_calls == 1
tests/unit/test_material_detail_api_handler.py
+105
View File
@@ -5,8 +5,10 @@ from typing import Any
import pytest
from pydantic import BaseModel
from sync_state_machine.common.collection import DataCollection
from sync_state_machine.common.sync_node import SyncNode
from sync_state_machine.domain.material_detail.api_handler import MaterialDetailApiHandler
from sync_state_machine.domain.project.api_handler import ProjectApiHandler
from sync_state_machine.datasource.task_result import TaskStatus
@@ -25,6 +27,15 @@ class _MaterialDetailNode(SyncNode[_MaterialDetailSchema]):
schema = _MaterialDetailSchema
class _ProjectSchema(BaseModel):
id: str
class _ProjectNode(SyncNode[_ProjectSchema]):
node_type = "project"
schema = _ProjectSchema
class _MockApiClient:
def __init__(self) -> None:
self.requests: list[dict[str, Any]] = []
@@ -127,3 +138,97 @@ async def test_material_detail_update_failure_preserves_push_id_for_trace_lookup
assert results[0].status == TaskStatus.FAILED
assert results[0].push_id
assert handler.api_client.requests[0]["endpoint"] == "/material/detail/update"
@pytest.mark.asyncio
async def test_material_detail_load_uses_project_aggregate_when_enabled(monkeypatch: pytest.MonkeyPatch) -> None:
collection = DataCollection("remote")
project = _ProjectNode(
node_id="project-node-1",
data_id="project-1",
data={"id": "project-1"},
)
await collection.add(project)
handler = MaterialDetailApiHandler()
handler.set_collection(collection)
handler.set_handler_config({"load_method": "aggregate"})
handler.api_client = _MockApiClient()
async def _fake_get_aggregate(client, project_id: str, *, domains: list[str], limit: int = 10000):
assert project_id == "project-1"
assert domains == ["material_details"]
assert limit == 10000
return {
"meta": {
"project_id": project_id,
"requested_domains": domains,
"domain_limit": limit,
"truncated_domains": [],
},
"data": {
"material_details": [
{
"id": "detail-1",
"project_id": project_id,
"parent_id": "material-1",
"date": "2025-03-01",
"quantity": 2.0,
"remark": None,
"is_complete": False,
}
],
"construction_details": [],
"contract_settlement_details": [],
"power_details": [],
},
}
monkeypatch.setattr(ProjectApiHandler, "get_aggregate", _fake_get_aggregate)
nodes = await handler.load()
assert len(nodes) == 1
assert nodes[0].data_id == "detail-1"
detail_data = nodes[0].get_data()
assert detail_data is not None
assert detail_data["project_id"] == "project-1"
assert handler.api_client.requests == []
@pytest.mark.asyncio
async def test_material_detail_load_aggregate_raises_on_truncated_domain(monkeypatch: pytest.MonkeyPatch) -> None:
collection = DataCollection("remote")
project = _ProjectNode(
node_id="project-node-1",
data_id="project-1",
data={"id": "project-1"},
)
await collection.add(project)
handler = MaterialDetailApiHandler()
handler.set_collection(collection)
handler.set_handler_config({"load_method": "aggregate"})
handler.api_client = _MockApiClient()
async def _fake_get_aggregate(client, project_id: str, *, domains: list[str], limit: int = 10000):
assert domains == ["material_details"]
return {
"meta": {
"project_id": project_id,
"requested_domains": domains,
"domain_limit": limit,
"truncated_domains": ["material_details"],
},
"data": {
"material_details": [],
"construction_details": [],
"contract_settlement_details": [],
"power_details": [],
},
}
monkeypatch.setattr(ProjectApiHandler, "get_aggregate", _fake_get_aggregate)
with pytest.raises(RuntimeError, match="truncated domain material_details"):
await handler.load()
tests/unit/test_pipeline_strategy_continue_on_error.py
+45 -1
View File
@@ -9,7 +9,7 @@ from sync_state_machine.common.collection import DataCollection
from sync_state_machine.common.persistence import PersistenceBackend
from sync_state_machine.common.sqlite_backend import SQLitePersistenceBackend
from sync_state_machine.datasource.datasource import BaseDataSource
from sync_state_machine.pipeline.full_sync_pipeline import FullSyncPipeline
from sync_state_machine.pipeline.full_sync_pipeline import FullSyncPipeline, NodeTypeLoadError
from sync_state_machine.sync_system.strategy import DefaultSyncStrategy
@@ -141,3 +141,47 @@ async def test_phase_sync_by_node_type_skip_sync_keeps_bind_but_skips_writes(tmp
assert strategy.update_calls == 0
await pipeline.close()
@pytest.mark.asyncio
async def test_phase_sync_by_node_type_aborts_on_load_failure(tmp_path: Path) -> None:
db_path = tmp_path / "state.db"
persistence = SQLitePersistenceBackend(str(db_path))
await persistence.initialize()
local_collection = DataCollection("local", persistence=persistence, auto_persist=False)
remote_collection = DataCollection("remote", persistence=persistence, auto_persist=False)
binding_manager = BindingManager(persistence, auto_persist=False)
blocked_strategy = _OkStrategy("blocked", local_collection, remote_collection, binding_manager)
downstream_strategy = _OkStrategy("downstream", local_collection, remote_collection, binding_manager)
downstream_strategy.skip_sync = True
pipeline = FullSyncPipeline(
local_datasource=_DS(),
remote_datasource=_DS(),
strategies=[blocked_strategy, downstream_strategy],
persistence=persistence,
local_collection=local_collection,
remote_collection=remote_collection,
binding_manager=binding_manager,
node_types=["blocked", "downstream"],
load_order=["blocked", "downstream"],
sync_order=["blocked", "downstream"],
enable_persistence=False,
)
async def _fail_load(node_type: str, reason: str = "") -> None:
del reason
if node_type == "blocked":
raise NodeTypeLoadError("Load failed for node_type=blocked, datasource=remote, reason=pre-strategy refresh: boom")
pipeline._load_node_type = _fail_load # type: ignore[method-assign]
with pytest.raises(NodeTypeLoadError, match="node_type=blocked"):
await pipeline.phase_sync_by_node_type()
assert blocked_strategy.bound is False
assert downstream_strategy.bound is False
await pipeline.close()
tests/unit/test_project_api_handler_normalization.py
+62
View File
@@ -94,3 +94,65 @@ async def test_project_api_handler_load_requires_api_client() -> None:
with pytest.raises(RuntimeError, match="API client not set"):
await handler.load()
class _AggregateRecordingClient:
def __init__(self) -> None:
self.calls: list[dict] = []
async def request(self, method, endpoint, params=None, data=None, **kwargs):
self.calls.append(
{
"method": method,
"endpoint": endpoint,
"params": params,
"data": data,
"kwargs": kwargs,
}
)
return {
"code": 200,
"message": "OK",
"data": {
"meta": {
"project_id": data["project_id"],
"requested_domains": data["domains"],
"domain_limit": data["limit"],
"truncated_domains": [],
},
"data": {
"material_details": [],
"construction_details": [],
"contract_settlement_details": [],
"power_details": [],
},
},
}
@pytest.mark.asyncio
async def test_project_aggregate_requests_are_not_shared_across_calls() -> None:
ProjectApiHandler.clear_cache()
client = _AggregateRecordingClient()
first = await ProjectApiHandler.get_aggregate(
client,
"project-1",
domains=["material_details"],
)
second = await ProjectApiHandler.get_aggregate(
client,
"project-1",
domains=["material_details"],
)
assert first["meta"]["project_id"] == "project-1"
assert second["meta"]["project_id"] == "project-1"
assert len(client.calls) == 2
assert client.calls[0]["method"] == "POST"
assert client.calls[0]["endpoint"] == "/project/aggregate"
assert client.calls[0]["data"]["project_id"] == "project-1"
assert client.calls[0]["data"]["domains"] == ["material_details"]
assert client.calls[1]["data"]["domains"] == ["material_details"]
ProjectApiHandler.clear_cache()