strepsiades/ecm_sync_system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

解决了部分detail load效率问题和load失败终止pipeline问题

Browse Source
This commit is contained in:
strepsiades
2026-04-02 14:35:28 +08:00
parent bb8ea931b5
commit 74df53889c
14 changed files with 626 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sync_state_machine/domain/construction_task_detail/api_handler.py
+41 -9
View File
@@ -17,39 +17,71 @@ from schemas.construction.construction_detail import (
ConstructionDetailCreate,
ConstructionDetailUpdate
)
from ..project.api_handler import ProjectApiHandler
class ConstructionTaskDetailApiHandler(BaseApiHandler):
"""施工明细 API Handler"""
def __init__(self):
super().__init__()
def __init__(self, **handler_config: Any):
super().__init__(**handler_config)
self._node_type = "construction_task_detail"
self._node_class = ConstructionTaskDetailSyncNode
self._schema = ConstructionDetailResponse
self.update_schemas = [ConstructionDetailCreate, ConstructionDetailUpdate]
async def load(self) -> List[SyncNode]:
"""加载施工明细列表（依赖 ConstructionTask）"""
def _use_aggregate_load(self) -> bool:
return str(self.handler_config.get("load_method", "legacy")).lower() == "aggregate"
async def _load_via_project_aggregate(self) -> List[SyncNode]:
projects = self._collection.filter(node_type="project")
project_ids = [project.data_id for project in projects if project.data_id]
if not project_ids:
return []
all_details: List[SyncNode] = []
aggregate_limit = int(self.handler_config.get("aggregate_limit", 10000))
for project_id in project_ids:
aggregate = await ProjectApiHandler.get_aggregate(
self.api_client,
project_id,
domains=["construction_details"],
limit=aggregate_limit,
)
truncated = set(aggregate.get("meta", {}).get("truncated_domains", []))
if "construction_details" in truncated:
raise RuntimeError(
f"project aggregate truncated domain construction_details for project {project_id}"
)
for detail_data in aggregate.get("data", {}).get("construction_details", []):
all_details.append(self._create_node(dict(detail_data)))
return all_details
async def _load_via_legacy_list(self) -> List[SyncNode]:
tasks = self._collection.filter(node_type="construction_task")
task_ids = [t.data_id for t in tasks if t.data_id]
if not task_ids:
return []
all_details = []
for task_id in task_ids:
try:
details = await api_get_construction_task_detail_list(self.api_client, task_id)
for detail_model in details:
# 已验证的 Pydantic model，转换为 dict
detail_data = detail_model.model_dump()
all_details.append(self._create_node(detail_data))
except Exception as e:
print(f"❌ Failed to load construction details for task {task_id}: {e}")
return all_details
async def load(self) -> List[SyncNode]:
"""加载施工明细列表（依赖 ConstructionTask）"""
if self._use_aggregate_load():
return await self._load_via_project_aggregate()
return await self._load_via_legacy_list()
async def create_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
"""批量创建施工明细"""
from pydantic import ValidationError
sync_state_machine/domain/contract_settlement_detail/api_handler.py
+41 -8
View File
@@ -17,26 +17,53 @@ from schemas.contract_settlement.contract_settlement_detail import (
ContractSettlementDetailCreate,
ContractSettlementDetailUpdate
)
from ..project.api_handler import ProjectApiHandler
class ContractSettlementDetailApiHandler(BaseApiHandler):
"""合同结算明细 API Handler"""
def __init__(self):
super().__init__()
def __init__(self, **handler_config: Any):
super().__init__(**handler_config)
self._node_type = "contract_settlement_detail"
self._node_class = ContractSettlementDetailSyncNode
self._schema = ContractSettlementDetailResponse
self.update_schemas = [ContractSettlementDetailCreate, ContractSettlementDetailUpdate]
async def load(self) -> List[SyncNode]:
"""加载合同结算明细列表（依赖 ContractSettlement）"""
def _use_aggregate_load(self) -> bool:
return str(self.handler_config.get("load_method", "legacy")).lower() == "aggregate"
async def _load_via_project_aggregate(self) -> List[SyncNode]:
projects = self._collection.filter(node_type="project")
project_ids = [project.data_id for project in projects if project.data_id]
if not project_ids:
return []
all_details: List[SyncNode] = []
aggregate_limit = int(self.handler_config.get("aggregate_limit", 10000))
for project_id in project_ids:
aggregate = await ProjectApiHandler.get_aggregate(
self.api_client,
project_id,
domains=["contract_settlement_details"],
limit=aggregate_limit,
)
truncated = set(aggregate.get("meta", {}).get("truncated_domains", []))
if "contract_settlement_details" in truncated:
raise RuntimeError(
f"project aggregate truncated domain contract_settlement_details for project {project_id}"
)
for detail_data in aggregate.get("data", {}).get("contract_settlement_details", []):
all_details.append(self._create_node(dict(detail_data)))
return all_details
async def _load_via_legacy_list(self) -> List[SyncNode]:
settlements = self._collection.filter(node_type="contract_settlement")
settlement_ids = [s.data_id for s in settlements if s.data_id]
if not settlement_ids:
return []
all_details = []
for settlement_id in settlement_ids:
try:
@@ -46,9 +73,15 @@ class ContractSettlementDetailApiHandler(BaseApiHandler):
all_details.append(self._create_node(detail_data))
except Exception as e:
print(f"❌ Failed to load settlement details for {settlement_id}: {e}")
return all_details
async def load(self) -> List[SyncNode]:
"""加载合同结算明细列表（依赖 ContractSettlement）"""
if self._use_aggregate_load():
return await self._load_via_project_aggregate()
return await self._load_via_legacy_list()
async def create_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
"""批量创建合同结算明细"""
from pydantic import ValidationError
sync_state_machine/domain/material_detail/api_handler.py
+54 -26
View File
@@ -32,44 +32,72 @@ from schemas.material.material_detail import (
MaterialDetailCreate,
MaterialDetailUpdate
)
from ..project.api_handler import ProjectApiHandler
class MaterialDetailApiHandler(BaseApiHandler):
"""物资明细 API Handler"""
def __init__(self):
super().__init__()
def __init__(self, **handler_config: Any):
super().__init__(**handler_config)
self._node_type = "material_detail"
self._node_class = MaterialDetailSyncNode
self._schema = MaterialDetailResponse
self.update_schemas = [MaterialDetailCreate, MaterialDetailUpdate]
def _use_aggregate_load(self) -> bool:
return str(self.handler_config.get("load_method", "legacy")).lower() == "aggregate"
async def _load_via_project_aggregate(self) -> List[SyncNode]:
projects = self._collection.filter(node_type="project")
project_ids = [project.data_id for project in projects if project.data_id]
if not project_ids:
return []
all_nodes: List[SyncNode] = []
aggregate_limit = int(self.handler_config.get("aggregate_limit", 10000))
for project_id in project_ids:
aggregate = await ProjectApiHandler.get_aggregate(
self.api_client,
project_id,
domains=["material_details"],
limit=aggregate_limit,
)
truncated = set(aggregate.get("meta", {}).get("truncated_domains", []))
if "material_details" in truncated:
raise RuntimeError(
f"project aggregate truncated domain material_details for project {project_id}"
)
for detail_data in aggregate.get("data", {}).get("material_details", []):
node = self._create_node(dict(detail_data))
all_nodes.append(node)
return all_nodes
async def _load_via_legacy_list(self) -> List[SyncNode]:
materials = self._collection.filter(node_type="material")
material_ids = [m.data_id for m in materials if m.data_id]
if not material_ids:
return []
all_nodes = []
for material_id in material_ids:
try:
details = await api_get_material_detail_list(self.api_client, material_id)
for detail_model in details:
detail_data = detail_model.model_dump()
node = self._create_node(detail_data)
all_nodes.append(node)
except Exception as e:
print(f"❌ Failed to load material details for {material_id}: {e}")
return all_nodes
async def load(self) -> List[SyncNode]:
"""加载物资明细列表（依赖 Material）"""
try:
materials = self._collection.filter(node_type="material")
material_ids = [m.data_id for m in materials if m.data_id]
if not material_ids:
return []
all_nodes = []
for material_id in material_ids:
try:
details = await api_get_material_detail_list(self.api_client, material_id)
for detail_model in details:
# 已验证的 Pydantic model，转换为 dict
detail_data = detail_model.model_dump()
node = self._create_node(detail_data)
all_nodes.append(node)
except Exception as e:
print(f"❌ Failed to load material details for {material_id}: {e}")
return all_nodes
except Exception as e:
print(f"❌ Failed to load material details: {e}")
return []
if self._use_aggregate_load():
return await self._load_via_project_aggregate()
return await self._load_via_legacy_list()
async def create_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
"""批量创建物资明细"""
sync_state_machine/domain/personnel_detail/api_handler.py
+42 -8
View File
@@ -13,26 +13,54 @@ from ...common.sync_node import SyncNode
from ...datasource.api.errors import ERROR_NODE_DATA_NONE, ERROR_ID_NOT_FOUND
from .sync_node import PersonnelDetailSyncNode
from schemas.power.power_detail import PowerDetail, PowerDetailCreate, PowerDetailUpdate
from ..project.api_handler import ProjectApiHandler
class PersonnelDetailApiHandler(BaseApiHandler):
"""人员明细 API Handler"""
def __init__(self):
super().__init__()
def __init__(self, **handler_config: Any):
super().__init__(**handler_config)
self._node_type = "personnel_detail"
self._node_class = PersonnelDetailSyncNode
self._schema = PowerDetail
self.update_schemas = [PowerDetailCreate, PowerDetailUpdate]
async def load(self) -> List[SyncNode]:
"""加载人员明细列表（依赖 Personnel）"""
def _use_aggregate_load(self) -> bool:
return str(self.handler_config.get("load_method", "legacy")).lower() == "aggregate"
async def _load_via_project_aggregate(self) -> List[SyncNode]:
projects = self._collection.filter(node_type="project")
project_ids = [project.data_id for project in projects if project.data_id]
if not project_ids:
return []
all_details: List[SyncNode] = []
aggregate_limit = int(self.handler_config.get("aggregate_limit", 10000))
for project_id in project_ids:
aggregate = await ProjectApiHandler.get_aggregate(
self.api_client,
project_id,
domains=["power_details"],
limit=aggregate_limit,
)
truncated = set(aggregate.get("meta", {}).get("truncated_domains", []))
if "power_details" in truncated:
raise RuntimeError(
f"project aggregate truncated domain power_details for project {project_id}"
)
for detail_data in aggregate.get("data", {}).get("power_details", []):
validated = PowerDetail.model_validate(detail_data)
all_details.append(self._create_node(validated.model_dump()))
return all_details
async def _load_via_legacy_list(self) -> List[SyncNode]:
personnel_list = self._collection.filter(node_type="personnel")
personnel_ids = [p.data_id for p in personnel_list if p.data_id]
if not personnel_ids:
return []
all_details = []
for personnel_id in personnel_ids:
try:
@@ -42,9 +70,15 @@ class PersonnelDetailApiHandler(BaseApiHandler):
all_details.append(self._create_node(validated.model_dump()))
except Exception as e:
print(f"❌ Failed to load personnel details for {personnel_id}: {e}")
return all_details
async def load(self) -> List[SyncNode]:
"""加载人员明细列表（依赖 Personnel）"""
if self._use_aggregate_load():
return await self._load_via_project_aggregate()
return await self._load_via_legacy_list()
async def create_all(self, nodes: List[SyncNode]) -> List[TaskResult]:
"""批量创建人员明细"""
results = []
sync_state_machine/domain/project/api_handler.py
+39
View File
@@ -34,6 +34,7 @@ from schemas.project.project_base import (
ProjectSettlementInvestment
)
from schemas.project.project import ProjectInfoResponse
from schemas.project.project_aggregate import ProjectAggregateResponse
_PROJECT_EXTENSION_FIELD_NAMES = {
@@ -94,6 +95,23 @@ class ProjectApiHandler(BaseApiHandler):
all_info = await api_get_project_all_info(client, project_id)
cls._all_info_cache[project_id] = all_info
return all_info
@classmethod
async def get_aggregate(
cls,
client,
project_id: str,
*,
domains: List[str],
limit: int = 10000,
) -> Dict[str, Any]:
normalized_domains = tuple(dict.fromkeys(domains))
return await api_get_project_aggregate(
client,
project_id,
domains=list(normalized_domains),
limit=limit,
)
@classmethod
def clear_cache(cls):
@@ -407,6 +425,27 @@ async def api_get_project_all_info(client, project_id: str) -> Dict[str, Any]:
return validated.model_dump()
async def api_get_project_aggregate(
client,
project_id: str,
*,
domains: List[str],
limit: int = 10000,
) -> Dict[str, Any]:
"""POST /project/aggregate - 获取项目聚合数据（校验为 ProjectAggregateResponse）"""
response = await client.request(
method="POST",
endpoint="/project/aggregate",
data={"project_id": project_id, "domains": domains, "limit": limit},
)
code = response.get("code")
if code != 200:
raise Exception(f"API error {code}: {response.get('message', 'Unknown error')}")
validated = ProjectAggregateResponse.model_validate(response.get("data", {}))
return validated.model_dump()
async def api_update_project_base_info(client, data: Dict[str, Any], push_id: str, biz_id: str, *, steps: List[Dict[str, Any]] | None = None) -> None:
"""PUT /project - 更新项目基本信息"""
ProjectBaseInfoUpdate.model_validate(data)