strepsiades/ecm_sync_system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

更新了部分测试

Browse Source
This commit is contained in:
strepsiades
2026-03-24 09:25:37 +08:00
parent b51ed0175b
commit 7d92b3f1fe
19 changed files with 115 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/runtime_config_guide.md
+7 -2
View File
@@ -173,7 +173,12 @@ logging:
- `local_orphan_action: none`
- `remote_orphan_action: none`
- `update_direction: none`
- 当三者同时为 `none/none/none` 时，pipeline 会跳过该 node_type 的 bind/create/update。
- `skip_sync=true` 的当前语义是：
- 仍会执行该 node_type 的 load / bind
- 只跳过 create / update 写入阶段
- 适合“保留绑定与依赖可见性，但临时禁止写入”的场景。
- `none/none/none` 是更推荐的输入风格，因为它表达的是“该类型不做创建、不做补本地、不做更新”。
- 这组配置不会绕过前置 load / bind / 依赖检查；它只会把该类型变成无写入策略。
### 4.6 Handler 级参数（`handler_configs`）
@@ -227,7 +232,7 @@ project_detail_data:
- 若发现差异，会在日志中打印差异样本，并在汇总表新增 `Consistent` 列展示 `Y` / `N(差异数)`。
- 全局跳过
- `local_orphan_action=none` + `remote_orphan_action=none` + `update_direction=none`
可直接跳过该类型的 bind/create/update。
可将该类型收敛为“只做 load/bind，不做 create/update 写入”的只读同步策略。
## 6. 覆盖配置建议
docs/testing_guide.md
+6
View File
@@ -16,6 +16,11 @@
- 校验 datasource/handler 工具链、配置严格性、导入面稳定性。
- 对具体 domain 的 update/create 行为做快速回归。
单元测试取舍原则：
- 优先测文档里承诺的行为语义，不优先测单纯转发、setter、空包装器。
- 如果一个测试只是在断言“函数 A 调了函数 B”，但不验证最终行为，通常应该删掉或改写成行为测试。
- 如果需要 test double，优先让它承载真实行为断言，不保留仅为凑接口而存在的空壳类。
常用命令：
- 全量单元测试：`pytest tests/unit -q`
- 指定文件：`pytest tests/unit/test_update_ops.py -q`
@@ -165,6 +170,7 @@ python tools/remote_env.py \
1. 先跑 `pytest tests/unit -q`。
2. 如果改动影响 pipeline，再跑目标集成测试。
3. 如果改动影响真实接口流程，再使用 `tools/remote_env.py` 管理远程环境。
4. 如果修改了配置语义或 pipeline 控制流，优先补一条直接覆盖设计语义的 pytest，而不是只补装配/转发测试。
### 3.2 真实接口联调
sync_state_machine/datasource/api/datasource.py
+4 -4
View File
@@ -65,9 +65,9 @@ class ApiDataSource(BaseDataSource):
"""关闭数据源（关闭 HTTP 客户端）"""
await self.client.close()
def register_handler(self, handler: NodeHandler) -> None:
def add_handler(self, handler: NodeHandler) -> None:
"""
注册业务 Handler
添加业务 Handler
Args:
handler: API Handler 实例
@@ -75,8 +75,8 @@ class ApiDataSource(BaseDataSource):
handler.set_api_client(self.client)
handler.set_poll_mode(self.poll_mode)
# 调用基类注册方法
super().register_handler(handler)
# 调用基类添加方法
super().add_handler(handler)
# 如果 collection 已设置，确保新 handler 拿到引用
if self._collection is not None:
sync_state_machine/datasource/datasource.py
+6 -6
View File
@@ -57,9 +57,9 @@ class BaseDataSource(ABC):
# 1. 初始化 DataSource
datasource = MyDataSource()
# 2. 注册 Handler
datasource.register_handler(project_handler)
datasource.register_handler(contract_handler)
# 2. 添加 Handler
datasource.add_handler(project_handler)
datasource.add_handler(contract_handler)
# 3. 设置 Collection
datasource.set_collection(collection)
@@ -360,15 +360,15 @@ class BaseDataSource(ABC):
# ========== Handler 管理 ==========
def register_handler(self, handler: "NodeHandler") -> None:
def add_handler(self, handler: "NodeHandler") -> None:
"""
注册 Handler
添加 Handler 到当前 datasource
Args:
handler: 节点处理器实例
示例：
datasource.register_handler(ContractNodeHandler(api_client))
datasource.add_handler(ContractNodeHandler(api_client))
"""
self._handlers[handler.node_type] = handler
sync_state_machine/datasource/handler.py
-53
View File
@@ -495,59 +495,6 @@ class BaseNodeHandler(ABC, Generic[T]):
return self.create_node(data)
class CompatibleNodeHandler(BaseNodeHandler[T]):
def __init__(self, handler: BaseNodeHandler[T]):
super().__init__(handler.node_type, handler.node_class, handler.schema)
self._handler = handler
def set_collection(self, collection: "DataCollection") -> None:
super().set_collection(collection)
self._handler.set_collection(collection)
def set_handler_config(self, config: Dict[str, Any]) -> None:
self._handler.set_handler_config(config)
self.handler_config = dict(self._handler.handler_config)
def set_api_client(self, client: "ApiClient") -> None:
self._handler.set_api_client(client)
def set_poll_mode(self, mode: str) -> None:
self._handler.set_poll_mode(mode)
def get_update_fields(self, *, exclude: frozenset[str] = frozenset({"id"})) -> List[str]:
return self._handler.get_update_fields(exclude=exclude)
async def load(self) -> List['SyncNode']:
return await self._handler.load()
async def create_all(
self,
nodes: List["SyncNode"]
) -> List[TaskResult]:
return await self._handler.create_all(nodes)
async def update_all(
self,
nodes: List["SyncNode"]
) -> List[TaskResult]:
return await self._handler.update_all(nodes)
async def delete_all(
self,
nodes: List["SyncNode"]
) -> List[TaskResult]:
return await self._handler.delete_all(nodes)
async def poll_tasks(self, task_ids: List[str]) -> Dict[str, TaskResult]:
return await self._handler.poll_tasks(task_ids)
async def validate(self, data: Dict[str, Any]) -> ValidationResult:
return await self._handler.validate(data)
def extract_id(self, data: Dict[str, Any]) -> str:
return self._handler.extract_id(data)
class NodeHandlerRegistry:
"""节点处理器注册表"""
sync_state_machine/datasource/jsonl/datasource.py
+1 -1
View File
@@ -32,7 +32,7 @@ class JsonlDataSource(BaseDataSource):
示例：
datasource = JsonlDataSource(Path("filtered_datasource/datasource/ecm-2025-12-02"))
datasource.register_handler(ContractJsonlHandler(datasource))
datasource.add_handler(ContractJsonlHandler(datasource))
collection = DataCollection()
datasource.set_collection(collection)
sync_state_machine/domain/registry.py
+4 -4
View File
@@ -4,13 +4,13 @@ Domain Handler Registry - 集中管理所有领域的 API 和 JSONL Handlers
使用方式：
from sync_state_machine.domain.registry import API_HANDLERS, JSONL_HANDLERS
# 注册所有 API handlers
# 添加所有 API handlers
for handler_class in API_HANDLERS:
api_ds.register_handler(handler_class())
api_ds.add_handler(handler_class())
# 注册所有 JSONL handlers
# 添加所有 JSONL handlers
for handler_class in JSONL_HANDLERS:
jsonl_ds.register_handler(handler_class(jsonl_ds))
jsonl_ds.add_handler(handler_class(jsonl_ds))
"""
# ============================================
sync_state_machine/pipeline/copy_data_pipeline.py
+3 -3
View File
@@ -67,7 +67,7 @@ class CopyDataPipeline:
supported_types = []
for handler_class in JSONL_HANDLERS:
handler = handler_class(self.source_datasource)
self.source_datasource.register_handler(handler)
self.source_datasource.add_handler(handler)
supported_types.append(handler.node_type)
self._owns_source = True
self._supported_types = supported_types
@@ -82,7 +82,7 @@ class CopyDataPipeline:
self.target_datasource = JsonlDataSource(dir_path=Path(target_dir), read_only=False)
# 自动注册所有 JSONL handlers
for handler_class in JSONL_HANDLERS:
self.target_datasource.register_handler(handler_class(self.target_datasource))
self.target_datasource.add_handler(handler_class(self.target_datasource))
self._owns_target = True
else:
raise ValueError("Must provide either target_datasource or target_dir")
@@ -228,7 +228,7 @@ class CopyDataPipeline:
handler_class = DomainRegistry.get_jsonl_handler(node_type)
if handler_class:
handler = handler_class(datasource)
datasource.register_handler(handler)
datasource.add_handler(handler)
def _print_summary(self):
"""打印执行摘要"""
sync_state_machine/pipeline/factory.py
+6 -6
View File
@@ -151,14 +151,14 @@ async def _initialize_datasource(datasource, *, endpoint_name: str) -> None:
ensure_builtin_datasource_types_registered()
def _register_handlers_for_endpoint(
def _add_handlers_for_endpoint(
*,
datasource,
ds_config: DataSourceConfig,
node_types: List[str],
) -> None:
for node_type in node_types:
datasource.register_handler(
datasource.add_handler(
_create_handler(
node_type=node_type,
ds_config=ds_config,
@@ -167,18 +167,18 @@ def _register_handlers_for_endpoint(
)
def _register_handlers(
def _add_handlers(
config: PipelineRunConfig,
local_ds,
remote_ds,
all_types: List[str],
) -> None:
_register_handlers_for_endpoint(
_add_handlers_for_endpoint(
datasource=local_ds,
ds_config=config.local_datasource,
node_types=all_types,
)
_register_handlers_for_endpoint(
_add_handlers_for_endpoint(
datasource=remote_ds,
ds_config=config.remote_datasource,
node_types=all_types,
@@ -320,7 +320,7 @@ async def create_pipeline_from_config(
project_root=_project_root(),
)
await _initialize_datasource(remote_ds, endpoint_name="remote")
_register_handlers(config, local_ds, remote_ds, all_types)
_add_handlers(config, local_ds, remote_ds, all_types)
local_collection, remote_collection, binding_manager = _build_collections_and_bindings(
persistence=persistence,
sync_state_machine/pipeline/full_sync_pipeline.py
-9
View File
@@ -172,15 +172,6 @@ class FullSyncPipeline:
self._logger.error(f"❌ Strategy failed but pipeline continues: {node_type} | {type(exc).__name__}: {exc}")
continue
@staticmethod
def _is_noop_strategy(strategy: BaseSyncStrategy[Any]) -> bool:
cfg = strategy.config
return (
cfg.local_orphan_action == OrphanAction.NONE
and cfg.remote_orphan_action == OrphanAction.NONE
and cfg.update_direction == UpdateDirection.NONE
)
async def phase_bootstrap(self) -> None:
await self._load_persistence_state()
self._logger.info("✅ Bootstrap: persistence loaded")
tests/README.md
+1
View File
@@ -17,6 +17,7 @@
## 设计原则
- 不绕过 schema/状态机校验；配置错误必须显式失败。
- 单元测试聚焦纯逻辑，集成测试覆盖流水线与边界行为。
- 优先保留行为测试，避免只验证转发器、setter 或仅供测试存在的空壳包装层。
- 所有测试均从仓库根路径解析配置，避免目录迁移导致路径脆弱。
- 保留高价值回归测试，移除仅重复路径或无断言价值的历史脚本型测试（脚本保留在 `scripts/`，不纳入 pytest）。
tests/integration/test_full_pipeline_multinode_mock.py
+6 -6
View File
@@ -112,7 +112,7 @@ async def pipeline_bundle():
local_ds = InMemoryDataSource(poll_max_retries=2, poll_interval=0)
remote_ds = InMemoryDataSource(poll_max_retries=2, poll_interval=0)
local_ds.register_handler(
local_ds.add_handler(
InMemoryNodeHandler(
node_type="test_project",
node_class=TestProjectNode,
@@ -120,7 +120,7 @@ async def pipeline_bundle():
load_nodes=local_project_nodes,
)
)
local_ds.register_handler(
local_ds.add_handler(
InMemoryNodeHandler(
node_type="test_contract",
node_class=TestContractNode,
@@ -129,7 +129,7 @@ async def pipeline_bundle():
)
)
remote_ds.register_handler(
remote_ds.add_handler(
InMemoryNodeHandler(
node_type="test_project",
node_class=TestProjectNode,
@@ -143,7 +143,7 @@ async def pipeline_bundle():
update_mode_by_code={"UPD-1": "success"},
)
)
remote_ds.register_handler(
remote_ds.add_handler(
InMemoryNodeHandler(
node_type="test_contract",
node_class=TestContractNode,
@@ -453,7 +453,7 @@ async def test_pipeline_create_then_update_completes_in_single_run_with_reload()
local_ds = InMemoryDataSource(poll_max_retries=2, poll_interval=0)
remote_ds = InMemoryDataSource(poll_max_retries=2, poll_interval=0)
local_ds.register_handler(
local_ds.add_handler(
InMemoryNodeHandler(
node_type="test_project",
node_class=TestProjectNode,
@@ -464,7 +464,7 @@ async def test_pipeline_create_then_update_completes_in_single_run_with_reload()
)
)
remote_handler = _MutableProjectRemoteHandler()
remote_ds.register_handler(remote_handler)
remote_ds.add_handler(remote_handler)
project_strategy = TestProjectStrategy("test_project", local_collection, remote_collection, binding_manager)
project_strategy.update_config(
tests/integration/test_full_pipeline_multinode_persisted_db.py
+4 -4
View File
@@ -73,7 +73,7 @@ async def test_full_pipeline_multinode_mock_persisted_db_snapshot() -> None:
local_ds = InMemoryDataSource(poll_max_retries=2, poll_interval=0)
remote_ds = InMemoryDataSource(poll_max_retries=2, poll_interval=0)
local_ds.register_handler(
local_ds.add_handler(
InMemoryNodeHandler(
node_type="test_project",
node_class=TestProjectNode,
@@ -81,7 +81,7 @@ async def test_full_pipeline_multinode_mock_persisted_db_snapshot() -> None:
load_nodes=local_project_nodes,
)
)
local_ds.register_handler(
local_ds.add_handler(
InMemoryNodeHandler(
node_type="test_contract",
node_class=TestContractNode,
@@ -90,7 +90,7 @@ async def test_full_pipeline_multinode_mock_persisted_db_snapshot() -> None:
)
)
remote_ds.register_handler(
remote_ds.add_handler(
InMemoryNodeHandler(
node_type="test_project",
node_class=TestProjectNode,
@@ -104,7 +104,7 @@ async def test_full_pipeline_multinode_mock_persisted_db_snapshot() -> None:
update_mode_by_code={"UPD-1": "success"},
)
)
remote_ds.register_handler(
remote_ds.add_handler(
InMemoryNodeHandler(
node_type="test_contract",
node_class=TestContractNode,
tests/unit/test_datasource_load_continue_on_error.py
+2 -2
View File
@@ -97,8 +97,8 @@ class _DS(BaseDataSource):
@pytest.mark.asyncio
async def test_load_all_continue_when_one_handler_fails(capsys) -> None:
ds = _DS()
ds.register_handler(_FailHandler())
ds.register_handler(_OkHandler())
ds.add_handler(_FailHandler())
ds.add_handler(_OkHandler())
ds.set_collection(DataCollection("local"))
await ds.load_all(order=["fail", "ok"])
tests/unit/test_factory_db_handler_registration.py
+10 -9
View File
@@ -2,6 +2,7 @@ from __future__ import annotations
from pathlib import Path
import pytest
from pydantic import BaseModel
from sync_state_machine.common.registry import DomainRegistry
@@ -17,7 +18,7 @@ from sync_state_machine.config import (
from sync_state_machine.datasource import BaseDataSource
from sync_state_machine.datasource.handler import BaseNodeHandler
from sync_state_machine.datasource.jsonl import BaseJsonlHandler, JsonlDataSource
from sync_state_machine.pipeline.factory import _register_handlers
from sync_state_machine.pipeline.factory import create_pipeline_from_config
from sync_state_machine.sync_system.strategy import DefaultSyncStrategy
@@ -79,7 +80,8 @@ def _register_test_db_datasource_type() -> None:
)
def test_register_handlers_uses_db_handler_for_db_datasource(tmp_path: Path) -> None:
@pytest.mark.asyncio
async def test_register_handlers_uses_db_handler_for_db_datasource(tmp_path: Path) -> None:
_register_test_db_datasource_type()
if not DomainRegistry.is_registered("test_db_handler"):
@@ -104,10 +106,9 @@ def test_register_handlers_uses_db_handler_for_db_datasource(tmp_path: Path) ->
logging=LoggingConfig(initialize=False),
)
local_ds = BaseDataSource()
remote_ds = JsonlDataSource(tmp_path)
_register_handlers(config, local_ds, remote_ds, ["test_db_handler"])
assert isinstance(local_ds.get_handler("test_db_handler"), TestDbHandler)
assert isinstance(remote_ds.get_handler("test_db_handler"), TestJsonlHandler)
pipeline = await create_pipeline_from_config(config)
try:
assert isinstance(pipeline.local_datasource.get_handler("test_db_handler"), TestDbHandler)
assert isinstance(pipeline.remote_datasource.get_handler("test_db_handler"), TestJsonlHandler)
finally:
await pipeline.close()
tests/unit/test_handler_shared_config.py
-86
View File
@@ -1,86 +0,0 @@
from __future__ import annotations
from typing import Any
from pydantic import BaseModel
from sync_state_machine.common.sync_node import SyncNode
from sync_state_machine.datasource.api.handler import BaseApiHandler
from sync_state_machine.datasource.handler import BaseNodeHandler, CompatibleNodeHandler
from sync_state_machine.datasource.task_result import TaskResult
class SharedConfigSchema(BaseModel):
__test__ = False
id: str
class SharedConfigNode(SyncNode[SharedConfigSchema]):
__test__ = False
node_type = "shared_config_demo"
schema = SharedConfigSchema
class SharedConfigBaseHandler(BaseNodeHandler[SharedConfigSchema]):
__test__ = False
def __init__(self) -> None:
super().__init__("shared_config_demo", SharedConfigNode, SharedConfigSchema)
async def load(self):
return []
async def create_all(self, nodes):
return []
async def update_all(self, nodes):
return []
def extract_id(self, data: dict[str, Any]) -> str:
return str(data.get("id", ""))
class SharedConfigApiHandler(BaseApiHandler[SharedConfigSchema]):
__test__ = False
def __init__(self) -> None:
super().__init__()
self._node_type = "shared_api_demo"
self._node_class = SharedConfigNode
self._schema = SharedConfigSchema
async def load(self):
return []
async def create_all(self, nodes):
return [TaskResult.success(node_id=node.node_id) for node in nodes]
async def update_all(self, nodes):
return [TaskResult.success(node_id=node.node_id) for node in nodes]
async def delete_all(self, nodes):
return [TaskResult.success(node_id=node.node_id) for node in nodes]
def test_base_handler_supports_shared_handler_config() -> None:
handler = SharedConfigBaseHandler()
handler.set_handler_config({"mode": "local", "batch_size": 10})
assert handler.handler_config == {"mode": "local", "batch_size": 10}
def test_compatible_handler_forwards_shared_handler_config() -> None:
handler = CompatibleNodeHandler(SharedConfigBaseHandler())
handler.set_handler_config({"mode": "remote"})
assert handler.handler_config == {"mode": "remote"}
def test_api_handler_supports_shared_handler_config() -> None:
handler = SharedConfigApiHandler()
handler.set_handler_config({"filter_project_id": ["p1", "p2"]})
assert handler.handler_config == {"filter_project_id": ["p1", "p2"]}
tests/unit/test_pipeline_strategy_continue_on_error.py
+50
View File
@@ -35,14 +35,18 @@ class _OkStrategy(BaseSyncStrategy):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.bound = False
self.create_calls = 0
self.update_calls = 0
async def bind(self, **kwargs):
self.bound = True
async def create(self):
self.create_calls += 1
return []
async def update(self):
self.update_calls += 1
return []
@@ -87,6 +91,52 @@ async def test_phase_sync_by_node_type_continues_after_strategy_error(tmp_path:
await pipeline.phase_sync_by_node_type()
assert ok_strategy.bound is True
assert ok_strategy.create_calls == 0
assert ok_strategy.update_calls == 0
assert pipeline.stats["failed"] >= 1
await pipeline.close()
@pytest.mark.asyncio
async def test_phase_sync_by_node_type_skip_sync_keeps_bind_but_skips_writes(tmp_path: Path) -> None:
db_path = tmp_path / "state.db"
persistence = PersistenceBackend(str(db_path))
await persistence.initialize()
local_collection = DataCollection("local", persistence=persistence, auto_persist=False)
remote_collection = DataCollection("remote", persistence=persistence, auto_persist=False)
binding_manager = BindingManager(persistence, auto_persist=False)
strategy = _OkStrategy("ok", local_collection, remote_collection, binding_manager)
strategy.skip_sync = True
pipeline = FullSyncPipeline(
local_datasource=_DS(),
remote_datasource=_DS(),
strategies=[strategy],
persistence=persistence,
local_collection=local_collection,
remote_collection=remote_collection,
binding_manager=binding_manager,
node_types=["ok"],
load_order=["ok"],
sync_order=["ok"],
enable_persistence=False,
)
load_calls: list[tuple[str, str]] = []
async def _record_load(node_type: str, reason: str = "") -> None:
load_calls.append((node_type, reason))
pipeline._load_node_type = _record_load # type: ignore[method-assign]
await pipeline.phase_sync_by_node_type()
assert load_calls == [("ok", "pre-strategy refresh")]
assert strategy.bound is True
assert strategy.create_calls == 0
assert strategy.update_calls == 0
await pipeline.close()
tests/unit/test_target_project_preload_filter.py
+4 -4
View File
@@ -45,7 +45,7 @@ async def test_jsonl_project_load_applies_target_project_filter(tmp_path):
datasource = JsonlDataSource(tmp_path)
handler = BaseJsonlHandler(datasource, "project", _ProjectNode, _ProjectSchema)
handler.set_handler_config({"data_id_filter": ["p1"]})
datasource.register_handler(handler)
datasource.add_handler(handler)
datasource.set_collection(DataCollection("local"))
await datasource.load_all(order=["project"])
@@ -70,7 +70,7 @@ async def test_jsonl_non_project_load_filters_rows_by_project_id(tmp_path):
datasource = JsonlDataSource(tmp_path)
handler = BaseJsonlHandler(datasource, "contract", _BizNode, _BizSchema)
handler.set_handler_config({"data_id_filter": ["p1"]})
datasource.register_handler(handler)
datasource.add_handler(handler)
datasource.set_collection(DataCollection("local"))
await datasource.load_all(order=["contract"])
@@ -104,8 +104,8 @@ async def test_jsonl_non_project_load_inherits_project_filter_from_loaded_projec
project_handler = BaseJsonlHandler(datasource, "project", _ProjectNode, _ProjectSchema)
project_handler.set_handler_config({"data_id_filter": ["p1"]})
contract_handler = BaseJsonlHandler(datasource, "contract", _BizNode, _BizSchema)
datasource.register_handler(project_handler)
datasource.register_handler(contract_handler)
datasource.add_handler(project_handler)
datasource.add_handler(contract_handler)
datasource.set_collection(DataCollection("local"))
await datasource.load_all(order=["project", "contract"])
tests/unit/test_units_jsonl_handler_alias_files.py
+1 -1
View File
@@ -27,7 +27,7 @@ async def test_units_jsonl_handler_loads_units_filename(tmp_path):
)
datasource = JsonlDataSource(tmp_path)
datasource.register_handler(UnitsJsonlHandler(datasource))
datasource.add_handler(UnitsJsonlHandler(datasource))
datasource.set_collection(DataCollection("local"))
await datasource.load_all(order=["units"])