strepsiades/ecm_sync_system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

优化大量推送问题

Browse Source
This commit is contained in:
strepsiades
2026-04-02 11:14:44 +08:00
parent f93a6c5c68
commit bb8ea931b5
8 changed files with 343 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
schemas/project_extensions/preparation.py
+1 -1
View File
@@ -36,4 +36,4 @@ class PreparationDetail(BaseResponseWithID):
# approval_status: Optional[str] = Field(None, description="审批状态")
name: Optional[str] = Field(None, description="里程碑名称")
group: Optional[str] = Field(None, description="分组条件", examples=['KGTJLS'])
is_acquired: Optional[int] = Field(0, description="是否取得")
# is_acquired: Optional[int] = Field(0, description="是否取得")
schemas/project_extensions/production.py
+1 -1
View File
@@ -36,4 +36,4 @@ class ProductionDetail(BaseResponseWithID):
is_exist: Optional[bool] = Field(None, description="是否取得")
# approval_status: Optional[str] = Field(None, description="审批状态")
name: Optional[str] = Field(None, description="投产节点名称")
is_acquired: Optional[int] = Field(0, description="是否取得")
# is_acquired: Optional[int] = Field(0, description="是否取得")
sync_state_machine/domain/material_detail/api_handler.py
+9 -2
View File
@@ -75,6 +75,7 @@ class MaterialDetailApiHandler(BaseApiHandler):
"""批量创建物资明细"""
results = []
for node in nodes:
push_id: str | None = None
try:
data = node.get_data()
if not data:
@@ -118,7 +119,13 @@ class MaterialDetailApiHandler(BaseApiHandler):
else:
results.append(TaskResult.in_progress(node_id=node.node_id, task_id=push_id))
except Exception as e:
results.append(TaskResult.failed(node_id=node.node_id, error=f"Create failed: {str(e)}"))
results.append(
TaskResult.failed(
node_id=node.node_id,
error=f"Create failed: {str(e)}",
push_id=push_id,
)
)
return results
@@ -209,7 +216,7 @@ class MaterialDetailApiHandler(BaseApiHandler):
# 整组失败
error_msg = str(e)
for node, _ in group_items:
results.append(TaskResult.failed(node_id=node.node_id, error=error_msg))
results.append(TaskResult.failed(node_id=node.node_id, error=error_msg, push_id=push_id))
return results
sync_state_machine/domain/project_detail/api_handler.py
+3 -2
View File
@@ -40,8 +40,6 @@ from schemas.project.project_base import (
_PRODUCTION_PROJECT_FIELDS = [
"plan_production_date",
"plan_full_production_date",
"ic_plan_first_production_date",
"ic_plan_full_production_date",
"actual_production_date",
"actual_full_production_date",
]
@@ -185,6 +183,9 @@ class ProjectDetailApiHandler(BaseApiHandler):
except ValidationError as e:
results.append(TaskResult.failed(node_id=node.node_id, error=f"{ERROR_VALIDATION_FAILED}: {e}"))
continue
except Exception as e:
results.append(TaskResult.failed(node_id=node.node_id, error=str(e)))
continue
push_id = self._generate_push_id()
biz_id = project_id
sync_state_machine/domain/project_detail/sync_strategy.py
+58
View File
@@ -1,3 +1,5 @@
from typing import Any
from pydantic import BaseModel, Field
from ...common.sync_node import SyncNode
@@ -36,6 +38,12 @@ class ProjectDetailSyncStrategy(DefaultSyncStrategy[ProjectDetailProject]):
"climb_production",
"challenge_production",
}
_PRODUCTION_PROJECT_FIELDS = (
"plan_production_date",
"plan_full_production_date",
"actual_production_date",
"actual_full_production_date",
)
default_config = StrategyConfig(
auto_bind=True,
@@ -153,6 +161,11 @@ class ProjectDetailSyncStrategy(DefaultSyncStrategy[ProjectDetailProject]):
def domain_option(self) -> ProjectDetailDomainOption:
return ProjectDetailDomainOption.model_validate(self.config.domain_option)
async def create_for_direction(self, *, source_is_local: bool) -> list[SyncNode]:
created_nodes = await super().create_for_direction(source_is_local=source_is_local)
await self._enrich_created_production_nodes(created_nodes, source_is_local=source_is_local)
return created_nodes
def normalize_compare_payload(self, data: dict | None, data_id_map=None) -> dict:
normalized = super().normalize_compare_payload(data, data_id_map)
key = normalized.get("key")
@@ -223,3 +236,48 @@ class ProjectDetailSyncStrategy(DefaultSyncStrategy[ProjectDetailProject]):
return key
return None
async def _enrich_created_production_nodes(self, created_nodes: list[SyncNode], *, source_is_local: bool) -> None:
if not created_nodes:
return
source_collection = self.local_collection if source_is_local else self.remote_collection
for created_node in created_nodes:
created_data = created_node.get_data() or {}
if self._extract_node_key(created_node) != "production":
continue
if all(created_data.get(field) not in (None, "") for field in self._PRODUCTION_PROJECT_FIELDS):
continue
if source_is_local:
source_node_id = await self.binding_manager.get_local_id(self.node_type, created_node.node_id)
else:
source_node_id = await self.binding_manager.get_remote_id(self.node_type, created_node.node_id)
if not source_node_id:
continue
source_node = source_collection.get(source_node_id)
if source_node is None:
continue
source_data = source_node.get_data() or {}
project_id = source_data.get("project_id")
if not project_id:
continue
project_node = source_collection.get_by_data_id("project", str(project_id))
if project_node is None:
continue
project_data = project_node.get_data() or {}
updated_data: dict[str, Any] = dict(created_data)
changed = False
for field in self._PRODUCTION_PROJECT_FIELDS:
if updated_data.get(field) in (None, "") and project_data.get(field) not in (None, ""):
updated_data[field] = project_data.get(field)
changed = True
if changed:
created_node.set_data(updated_data)
tests/unit/test_material_detail_api_handler.py
+67
View File
@@ -12,9 +12,12 @@ from sync_state_machine.datasource.task_result import TaskStatus
class _MaterialDetailSchema(BaseModel):
id: str
project_id: str | None = None
parent_id: str
date: str | None = None
quantity: float
remark: str | None = None
is_complete: bool | None = None
class _MaterialDetailNode(SyncNode[_MaterialDetailSchema]):
@@ -31,6 +34,12 @@ class _MockApiClient:
return {"code": 200, "message": "OK", "data": {"biz_id": kwargs.get("data", {}).get("biz_id")}}
class _FailingApiClient(_MockApiClient):
async def request(self, method: str, endpoint: str, **kwargs: Any) -> dict[str, Any]:
self.requests.append({"method": method, "endpoint": endpoint, **kwargs})
raise Exception("API error: failed request")
@pytest.mark.asyncio
async def test_material_detail_update_returns_in_progress_with_shared_push_id() -> None:
handler = MaterialDetailApiHandler()
@@ -60,3 +69,61 @@ async def test_material_detail_update_returns_in_progress_with_shared_push_id()
assert results[0].status == TaskStatus.IN_PROGRESS
assert results[0].task_id
assert handler.api_client.requests[0]["endpoint"] == "/material/detail/update"
@pytest.mark.asyncio
async def test_material_detail_create_failure_preserves_push_id_for_trace_lookup() -> None:
handler = MaterialDetailApiHandler()
handler.api_client = _FailingApiClient()
node = _MaterialDetailNode(
node_id="detail-node-create-1",
data_id="",
data={
"id": "runtime-detail-1",
"project_id": "project-1",
"parent_id": "material-1",
"date": "2025-03-01",
"quantity": 4.0,
"remark": None,
"is_complete": False,
},
)
results = await handler.create_all([node])
assert len(results) == 1
assert results[0].status == TaskStatus.FAILED
assert results[0].push_id
assert handler.api_client.requests[0]["endpoint"] == "/material/detail/create"
@pytest.mark.asyncio
async def test_material_detail_update_failure_preserves_push_id_for_trace_lookup() -> None:
handler = MaterialDetailApiHandler()
handler.api_client = _FailingApiClient()
node = _MaterialDetailNode(
node_id="detail-node-update-fail-1",
data_id="detail-1",
data={
"id": "detail-1",
"parent_id": "material-1",
"quantity": 10.0,
"remark": "new",
},
origin_data={
"id": "detail-1",
"parent_id": "material-1",
"quantity": 5.0,
"remark": None,
},
context={"material_id": "material-1"},
)
results = await handler.update_all([node])
assert len(results) == 1
assert results[0].status == TaskStatus.FAILED
assert results[0].push_id
assert handler.api_client.requests[0]["endpoint"] == "/material/detail/update"
tests/unit/test_project_detail_api_handler.py
+128
View File
@@ -57,6 +57,23 @@ class _FailingApiClient:
return {"code": 500, "message": "boom"}
class _RecordingApiClient:
def __init__(self) -> None:
self.calls = []
async def request(self, method, endpoint, params=None, data=None, **kwargs):
self.calls.append(
{
"method": method,
"endpoint": endpoint,
"params": params,
"data": data,
"kwargs": kwargs,
}
)
return {"code": 200, "message": "OK"}
@pytest.mark.asyncio
async def test_project_detail_create_failure_preserves_push_id_and_biz_id() -> None:
collection = DataCollection("remote")
@@ -96,6 +113,117 @@ async def test_project_detail_create_failure_preserves_push_id_and_biz_id() -> N
assert result.metadata["key"] == "production"
@pytest.mark.asyncio
async def test_project_detail_production_payload_excludes_ic_plan_dates() -> None:
collection = DataCollection("remote")
project = _ProjectNode(
node_id="project-node-1",
data_id="project-1",
data={"id": "project-1"},
context={"all_info": {"project_detail": []}},
)
await collection.add(project)
node = ProjectDetailSyncNode(
node_id="detail-node-1",
data_id="local-detail-1",
data={
"id": "local-detail-1",
"project_id": "project-1",
"key": "production",
"value": [{"date": "2026-01-01", "capacity": 1.0}],
"plan_production_date": "2026-01-01",
"plan_full_production_date": "2026-06-01",
"ic_plan_first_production_date": "2026-02-01",
"ic_plan_full_production_date": "2026-07-01",
"actual_production_date": "2026-01-15",
"actual_full_production_date": "2026-03-15",
},
)
handler = ProjectDetailApiHandler()
handler.set_collection(collection)
handler.api_client = _RecordingApiClient()
results = await handler.create_all([node])
assert len(results) == 1
assert results[0].status == TaskStatus.IN_PROGRESS
assert len(handler.api_client.calls) == 1
request_payload = handler.api_client.calls[0]["data"]["data"]
assert request_payload["plan_production_date"] == "2026-01-01"
assert request_payload["plan_full_production_date"] == "2026-06-01"
assert request_payload["actual_production_date"] == "2026-01-15"
assert request_payload["actual_full_production_date"] == "2026-03-15"
assert "ic_plan_first_production_date" not in request_payload
assert "ic_plan_full_production_date" not in request_payload
@pytest.mark.asyncio
async def test_project_detail_create_all_keeps_partial_results_when_schema_check_errors() -> None:
collection = DataCollection("remote")
project = _ProjectNode(
node_id="project-node-1",
data_id="project-1",
data={"id": "project-1"},
context={"all_info": {"project_detail": []}},
)
await collection.add(project)
valid_node = ProjectDetailSyncNode(
node_id="detail-node-1",
data_id="local-detail-1",
data={
"id": "local-detail-1",
"project_id": "project-1",
"key": "plan_construction",
"value": [{"date": "2026-01-01", "capacity": 1.0}],
},
)
invalid_node = ProjectDetailSyncNode(
node_id="detail-node-2",
data_id="local-detail-2",
data={
"id": "local-detail-2",
"project_id": "project-1",
"key": "production",
"value": [{"date": "2026-01-02", "capacity": 2.0}],
},
)
handler = ProjectDetailApiHandler()
handler.set_collection(collection)
handler.api_client = _RecordingApiClient()
class _ExplodingSchema:
@classmethod
def model_validate(cls, data):
raise RuntimeError("schema exploded")
original_get_update_config = handler._get_update_config
def _patched_get_update_config(key):
config = original_get_update_config(key)
if key == "production":
assert config is not None
return _ExplodingSchema, config[1]
return config
handler._get_update_config = _patched_get_update_config
results = await handler.create_all([valid_node, invalid_node])
assert len(results) == 2
assert results[0].status == TaskStatus.IN_PROGRESS
assert results[1].status == TaskStatus.FAILED
assert results[1].error == "schema exploded"
assert len(handler.api_client.calls) == 1
assert handler.api_client.calls[0]["endpoint"] == "/project/plan_construction"
def test_project_detail_load_allows_same_project_key_with_different_data_id() -> None:
collection = DataCollection("remote")
tests/unit/test_project_detail_sync_strategy.py
+76 -1
View File
@@ -15,6 +15,7 @@ from sync_state_machine.domain.project_detail.sync_strategy import ProjectDetail
from sync_state_machine.domain.project_detail.sync_node import ProjectDetailSyncNode
from sync_state_machine.sync_system.config import UpdateDirection
from sync_state_machine.sync_system.strategy_ops import BoundNodePair
from pydantic import BaseModel
def _fake_node(*, node_id: str, key: str):
@@ -44,6 +45,19 @@ def _build_project_detail_node(*, node_id: str, data_id: str, key: str) -> Proje
)
class _ProjectSchema(BaseModel):
id: str
plan_production_date: str | None = None
plan_full_production_date: str | None = None
actual_production_date: str | None = None
actual_full_production_date: str | None = None
class _ProjectNode(SyncNode[_ProjectSchema]):
node_type = "project"
schema = _ProjectSchema
@pytest_asyncio.fixture
async def project_detail_env(tmp_path):
persistence = SQLitePersistenceBackend(str(tmp_path / "project-detail.db"))
@@ -144,4 +158,65 @@ async def test_project_detail_bind_does_not_push_group_production_plan_from_loca
assert created_nodes == []
assert remote.filter(node_type="project_detail_data") == []
assert await binding_manager.get_remote_id("project_detail_data", local_node.node_id) is None
assert await binding_manager.get_remote_id("project_detail_data", local_node.node_id) is None
@pytest.mark.asyncio
async def test_project_detail_create_remote_production_carries_project_plan_dates(project_detail_env) -> None:
local, remote, binding_manager, _ = project_detail_env
strategy = ProjectDetailSyncStrategy("project_detail_data", local, remote, binding_manager)
local_project = _ProjectNode(
node_id="local-project-node",
data_id="local-project-id",
data={
"id": "local-project-id",
"plan_production_date": "2023-12-30",
"plan_full_production_date": "2024-12-30",
"actual_production_date": "2023-12-30",
"actual_full_production_date": "2024-12-30",
},
)
remote_project = _ProjectNode(
node_id="remote-project-node",
data_id="remote-project-id",
data={"id": "remote-project-id"},
)
await local.add(local_project)
await remote.add(remote_project)
await binding_manager.bind("project", local_project.node_id, remote_project.node_id)
local_detail = ProjectDetailSyncNode(
node_id="local-detail-node",
data_id="local-detail-id",
data={
"id": "local-detail-id",
"project_id": "local-project-id",
"key": "production",
"value": [{"date": "2023-12-30", "capacity": 3.0}],
},
)
await local.add(local_detail)
created_remote = ProjectDetailSyncNode(
node_id="remote-detail-node",
data_id="",
data={
"id": "",
"project_id": "remote-project-id",
"key": "production",
"value": [{"date": "2023-12-30", "capacity": 3.0}],
},
)
await remote.add(created_remote)
await binding_manager.bind("project_detail_data", local_detail.node_id, created_remote.node_id)
await strategy._enrich_created_production_nodes([created_remote], source_is_local=True)
created_data = created_remote.get_data()
assert created_data is not None
assert created_data["project_id"] == "remote-project-id"
assert created_data["plan_production_date"] == "2023-12-30"
assert created_data["plan_full_production_date"] == "2024-12-30"
assert created_data["actual_production_date"] == "2023-12-30"
assert created_data["actual_full_production_date"] == "2024-12-30"