strepsiades/ecm_sync_system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

优化性能

Browse Source
This commit is contained in:
strepsiades
2026-04-01 18:13:18 +08:00
parent 9a3a34c58b
commit f93a6c5c68
8 changed files with 198 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sync_state_machine/common/binding.py
+1 -1
View File
@@ -14,7 +14,7 @@ class BindingManager:
BindingManager 负责维护不同 DataCollection 间节点的绑定关系。
所有查询均在内存中进行，持久化仅作为备份。
"""
def __init__(self, persistence: PersistenceBackend, auto_persist: bool = True, scope: str = "global"):
def __init__(self, persistence: PersistenceBackend, auto_persist: bool = False, scope: str = "global"):
self.persistence = persistence
self.auto_persist = auto_persist
self.scope = scope
sync_state_machine/common/collection.py
+29 -20
View File
@@ -36,6 +36,8 @@ class DataCollection:
self._nodes: Dict[str, SyncNode] = {} # 内存缓存
# data_id 全局唯一（按 collection 维度），用于 O(1) 反查 node_id
self._data_id_to_node_id: Dict[str, str] = {}
# node_id -> data_id 反向索引，用于 O(1) 维护 data_id 更新
self._node_id_to_data_id: Dict[str, str] = {}
# auto_persist=False 时，delete() 不会立即落库；这里记录删除集合，persist() 时统一写回
self._deleted_node_ids: set[str] = set()
@@ -51,6 +53,20 @@ class DataCollection:
"""设置 Collection 默认状态机 runtime。"""
self._sm_runtime = runtime
def _remove_data_id_index(self, node_id: str, data_id: Optional[str] = None) -> None:
target_data_id = data_id
if target_data_id is None:
target_data_id = self._node_id_to_data_id.pop(node_id, None)
else:
self._node_id_to_data_id.pop(node_id, None)
if target_data_id and self._data_id_to_node_id.get(target_data_id) == node_id:
self._data_id_to_node_id.pop(target_data_id, None)
def _set_data_id_index(self, node_id: str, data_id: str) -> None:
self._data_id_to_node_id[data_id] = node_id
self._node_id_to_data_id[node_id] = data_id
async def add(self, node: SyncNode):
# 检查 node_id 唯一性
if node.node_id in self._nodes:
@@ -65,7 +81,7 @@ class DataCollection:
raise ValueError(
f"Duplicate data_id detected: {node.data_id} already bound to node_id={existing_node_id}"
)
self._data_id_to_node_id[node.data_id] = node.node_id
self._set_data_id_index(node.node_id, node.data_id)
self._nodes[node.node_id] = node
self._deleted_node_ids.discard(node.node_id)
@@ -73,18 +89,9 @@ class DataCollection:
await self.persistence.save_node(self.collection_id, self.scope, node)
async def update(self, node: SyncNode):
old_node = self._nodes.get(node.node_id)
if old_node and old_node.data_id and old_node.data_id != "" and old_node.data_id != node.data_id:
# node_id 不变但 data_id 变更，清理旧索引
self._data_id_to_node_id.pop(old_node.data_id, None)
stale_data_ids = [
data_id
for data_id, existing_node_id in self._data_id_to_node_id.items()
if existing_node_id == node.node_id and data_id != node.data_id
]
for stale_data_id in stale_data_ids:
self._data_id_to_node_id.pop(stale_data_id, None)
old_data_id = self._node_id_to_data_id.get(node.node_id)
if old_data_id and old_data_id != node.data_id:
self._remove_data_id_index(node.node_id, old_data_id)
if node.data_id and node.data_id != "":
existing_node_id = self._data_id_to_node_id.get(node.data_id)
@@ -92,16 +99,16 @@ class DataCollection:
raise ValueError(
f"Duplicate data_id detected: {node.data_id} already bound to node_id={existing_node_id}"
)
self._data_id_to_node_id[node.data_id] = node.node_id
self._set_data_id_index(node.node_id, node.data_id)
else:
self._node_id_to_data_id.pop(node.node_id, None)
self._nodes[node.node_id] = node
self._deleted_node_ids.discard(node.node_id)
if self.persistence and self.auto_persist:
await self.persistence.save_node(self.collection_id, self.scope, node)
async def delete(self, node_id: str):
old_node = self._nodes.get(node_id)
if old_node and old_node.data_id and old_node.data_id != "":
self._data_id_to_node_id.pop(old_node.data_id, None)
self._remove_data_id_index(node_id)
if node_id in self._nodes:
del self._nodes[node_id]
@@ -379,6 +386,7 @@ class DataCollection:
self._nodes = {}
self._data_id_to_node_id = {}
self._node_id_to_data_id = {}
self._deleted_node_ids.clear()
node_dicts = await self.persistence.load_nodes(self.collection_id, self.scope)
@@ -498,7 +506,7 @@ class DataCollection:
raise ValueError(
f"Duplicate data_id detected while loading: {node.data_id} already bound to node_id={existing_node_id}"
)
self._data_id_to_node_id[node.data_id] = node.node_id
self._set_data_id_index(node.node_id, node.data_id)
node_types = {node.node_type for node in self._nodes.values()}
for node_type in node_types:
@@ -517,6 +525,7 @@ class DataCollection:
self._nodes = {}
self._data_id_to_node_id = {}
self._node_id_to_data_id = {}
self._deleted_node_ids.clear()
node_dicts = await self.persistence.load_nodes(
@@ -622,7 +631,7 @@ class DataCollection:
raise ValueError(
f"Duplicate data_id detected while loading: {node.data_id} already bound to node_id={existing_node_id}"
)
self._data_id_to_node_id[node.data_id] = node.node_id
self._set_data_id_index(node.node_id, node.data_id)
loaded_node_types = {node.node_type for node in self._nodes.values()}
for node_type in loaded_node_types:
@@ -647,7 +656,7 @@ class DataCollection:
raise ValueError(
f"Duplicate data_id detected: {node.data_id} already bound to node_id={existing_node_id}"
)
self._data_id_to_node_id[node.data_id] = node.node_id
self._set_data_id_index(node.node_id, node.data_id)
self._nodes[node.node_id] = node
self._deleted_node_ids.discard(node.node_id)
sync_state_machine/domain/project_detail/api_handler.py
-38
View File
@@ -308,51 +308,13 @@ class ProjectDetailApiHandler(BaseApiHandler):
def _create_node(self, data: Dict[str, Any]) -> SyncNode:
project_id_raw = data.get("project_id")
key_raw = self._normalize_key(data.get("key"))
project_id = str(project_id_raw) if project_id_raw else ""
key = str(key_raw) if key_raw else ""
loaded_data_id = str(data.get("id") or data.get("_id") or "")
existing_by_pair = self._find_existing_by_project_key(project_id=project_id, key=key)
if existing_by_pair is not None:
if loaded_data_id and existing_by_pair.data_id == loaded_data_id:
existing_by_pair.set_data(data)
existing_by_pair.set_origin_data(data)
if project_id:
existing_by_pair.context["project_id"] = project_id
return existing_by_pair
existing_data_id = existing_by_pair.data_id or "<empty>"
loaded_data_id_text = loaded_data_id or "<empty>"
raise RuntimeError(
"project_detail_data load conflict: "
f"project_id={project_id}, key={key}, existing_data_id={existing_data_id}, loaded_data_id={loaded_data_id_text}"
)
node = self._create_basic_node(data, depend_ids=[])
if project_id:
node.context["project_id"] = project_id
return node
def _find_existing_by_project_key(self, *, project_id: str, key: str) -> Optional[SyncNode]:
if not project_id or not key or self._collection is None:
return None
candidates = self._collection.filter(
node_type=self.node_type,
node_filter=lambda n: (
((n.get_data() or {}).get("project_id") == project_id)
and (self._normalize_key((n.get_data() or {}).get("key")) == key)
),
)
if not candidates:
return None
if len(candidates) > 1:
raise RuntimeError(
f"Duplicate project_detail_data detected: project_id={project_id}, key={key}, count={len(candidates)}"
)
return candidates[0]
async def _get_project_all_info(self, project_id: str, *, force_refresh: bool = False) -> Dict[str, Any]:
from ..project.api_handler import ProjectApiHandler
sync_state_machine/pipeline/full_sync_pipeline.py
+80 -18
View File
@@ -6,6 +6,7 @@ from __future__ import annotations
import asyncio
import logging
from dataclasses import dataclass
from typing import Any, Dict, List, Optional, TYPE_CHECKING
from pathlib import Path
@@ -30,6 +31,16 @@ from .summary_report import print_pipeline_summary
logger = get_logger(__name__)
@dataclass(frozen=True)
class WriteOutcome:
local_written: bool = False
remote_written: bool = False
@property
def any_written(self) -> bool:
return self.local_written or self.remote_written
class FullSyncPipeline:
"""
Full pipeline phases:
@@ -163,10 +174,15 @@ class FullSyncPipeline:
created_remote_node_ids = {
node.node_id for node in created if self.remote_collection.get_by_node_id(node.node_id) is not None
}
create_written = await self.commit_creates(node_type)
create_outcome = await self.commit_creates(node_type)
await self.normalize_create_success_to_update_entry(node_type)
if create_written:
await self._reload_node_type(node_type, reason="create wrote data")
if create_outcome.any_written:
await self._reload_node_type_after_write(
node_type,
reason="create wrote data",
reload_local=create_outcome.local_written,
reload_remote=create_outcome.remote_written,
)
await refresh_bind_data_id(
node_type=node_type,
local_collection=self.local_collection,
@@ -178,9 +194,14 @@ class FullSyncPipeline:
updated = await strategy.update()
self._logger.info(f"✅ Strategy update: {node_type} (updated={len(updated)})")
update_written = await self.commit_updates(node_type)
if update_written:
await self._reload_node_type(node_type, reason="update wrote data")
update_outcome = await self.commit_updates(node_type)
if update_outcome.any_written:
await self._reload_node_type_after_write(
node_type,
reason="update wrote data",
reload_local=update_outcome.local_written,
reload_remote=update_outcome.remote_written,
)
self._logger.info(f"✅ Strategy done: {node_type}")
except Exception as exc:
@@ -233,10 +254,15 @@ class FullSyncPipeline:
created_remote_node_ids = {
node.node_id for node in created if self.remote_collection.get_by_node_id(node.node_id) is not None
}
create_written = await self.commit_creates(node_type)
create_outcome = await self.commit_creates(node_type)
await self.normalize_create_success_to_update_entry(node_type)
if create_written:
await self._reload_node_type(node_type, reason="create wrote data")
if create_outcome.any_written:
await self._reload_node_type_after_write(
node_type,
reason="create wrote data",
reload_local=create_outcome.local_written,
reload_remote=create_outcome.remote_written,
)
await refresh_bind_data_id(
node_type=node_type,
local_collection=self.local_collection,
@@ -263,9 +289,14 @@ class FullSyncPipeline:
try:
updated = await strategy.update()
self._logger.info(f"✅ Strategy update: {node_type} (updated={len(updated)})")
update_written = await self.commit_updates(node_type)
if update_written:
await self._reload_node_type(node_type, reason="update wrote data")
update_outcome = await self.commit_updates(node_type)
if update_outcome.any_written:
await self._reload_node_type_after_write(
node_type,
reason="update wrote data",
reload_local=update_outcome.local_written,
reload_remote=update_outcome.remote_written,
)
await self._evaluate_consistency_for_node_type(node_type)
except Exception as exc:
self.stats["failed"] += 1
@@ -410,6 +441,33 @@ class FullSyncPipeline:
return
await self._load_node_type(node_type, reason=reason)
async def _reload_node_type_after_write(
self,
node_type: str,
*,
reason: str,
reload_local: bool,
reload_remote: bool,
) -> None:
if not reload_local and not reload_remote:
return
if self._should_skip_reload():
self._logger.info("⏭️ Reload skipped for JSONL pipeline: node_type=%s reason=%s", node_type, reason)
return
self._logger.info(
"🔄 Reload node_type=%s after write, reason=%s, local=%s, remote=%s",
node_type,
reason,
reload_local,
reload_remote,
)
if reload_local:
await self.local_datasource.load_all(order=[node_type])
if reload_remote:
await self.remote_datasource.load_all(order=[node_type])
async def _evaluate_consistency_for_node_type(self, node_type: str) -> Dict[str, Any]:
strategy = next((item for item in self.strategies if item.node_type == node_type), None)
if strategy is None:
@@ -439,7 +497,7 @@ class FullSyncPipeline:
self._consistency_by_type[node_type] = result
return result
async def commit_creates(self, node_type: str) -> bool:
async def commit_creates(self, node_type: str) -> WriteOutcome:
"""提交 CREATE 操作（包含异步轮询）"""
# 执行同步（sync_all 内部会根据 action 筛选节点）
await self.local_datasource.sync_all(
@@ -464,8 +522,10 @@ class FullSyncPipeline:
local_success = self._get_action_success_count(self.local_datasource, node_type, "create")
remote_success = self._get_action_success_count(self.remote_datasource, node_type, "create")
wrote = (local_success + remote_success) > 0
return wrote
return WriteOutcome(
local_written=local_success > 0,
remote_written=remote_success > 0,
)
async def _finalize_peer_creating_sources(self, node_type: str) -> None:
finalized_success, finalized_failed = await finalize_peer_creating_sources(
@@ -502,7 +562,7 @@ class FullSyncPipeline:
if normalized > 0:
self._logger.info(f"🔁 Post-create normalization: {normalized} nodes moved S11C->S01 for {node_type}")
async def commit_updates(self, node_type: str) -> bool:
async def commit_updates(self, node_type: str) -> WriteOutcome:
"""提交 UPDATE 操作（包含异步轮询）"""
# 执行同步（sync_all 内部会根据 action 筛选节点）
await self.local_datasource.sync_all(
@@ -525,8 +585,10 @@ class FullSyncPipeline:
local_success = self._get_action_success_count(self.local_datasource, node_type, "update")
remote_success = self._get_action_success_count(self.remote_datasource, node_type, "update")
wrote = (local_success + remote_success) > 0
return wrote
return WriteOutcome(
local_written=local_success > 0,
remote_written=remote_success > 0,
)
async def phase_post_check(self) -> bool:
"""同步结果报告。