整理配置和代码入口,增加部分测试。

This commit is contained in:
strepsiades
2026-03-10 12:22:54 +08:00
parent 515c3d9388
commit a638e4203b
35 changed files with 1314 additions and 199 deletions
@@ -355,3 +355,55 @@ async def test_poll_tasks_queries_all_info_once_per_project_for_multiple_keys(mo
assert polled[task_ids[1]].status == TaskStatus.SUCCESS
assert polled[task_ids[1]].data_id == "remote-detail-2"
assert client.all_info_calls == ["project-1"]
@pytest.mark.asyncio
async def test_poll_tasks_retries_until_project_detail_appears(monkeypatch: pytest.MonkeyPatch) -> None:
collection = DataCollection("remote")
project = _build_project_node(node_id="p-node", project_id="project-1", all_info={"project_detail": []})
await collection.add(project)
handler = ProjectDetailApiHandler()
handler.set_collection(collection)
handler.api_client = _MockApiClient()
call_count = {"value": 0}
async def _fake_get_all_info(cls, _client, project_id: str):
call_count["value"] += 1
if call_count["value"] >= 2:
return {
"project_detail": [
{
"id": "remote-detail-1",
"project_id": project_id,
"key": "ensure_production",
"value": [{"year": 2026, "capacity": 21.0}],
}
]
}
return {"project_detail": []}
monkeypatch.setattr(ProjectApiHandler, "get_all_info", classmethod(_fake_get_all_info))
node = ProjectDetailSyncNode(
node_id="detail-node-1",
data_id="local-detail-1",
data={
"id": "local-detail-1",
"project_id": "project-1",
"key": "ensure_production",
"value": [{"year": 2026, "capacity": 21.0}],
},
)
create_result = (await handler.create_all([node]))[0]
assert create_result.task_id
handler.api_client.push_logs[create_result.task_id] = {"code": 200, "data": {"biz_id": "project-1"}}
polled = await handler.poll_tasks([create_result.task_id])
final = polled[create_result.task_id]
assert final.status == TaskStatus.SUCCESS
assert final.data_id == "remote-detail-1"
assert call_count["value"] >= 2
@@ -0,0 +1,65 @@
from __future__ import annotations
import importlib.util
from pathlib import Path
import sys
import yaml
def _load_base_module():
scripts_dir = Path(__file__).resolve().parents[2] / "scripts"
if str(scripts_dir) not in sys.path:
sys.path.insert(0, str(scripts_dir))
file_path = Path(__file__).resolve().parents[2] / "scripts" / "auto_test_domains" / "base.py"
spec = importlib.util.spec_from_file_location("auto_test_domain_base_for_test", file_path)
if spec is None or spec.loader is None:
raise RuntimeError(f"Unable to load module from {file_path}")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_build_temp_run_profile_uses_explicit_project_prebind(tmp_path: Path) -> None:
module = _load_base_module()
source_profile = tmp_path / "source.yaml"
source_profile.write_text(
yaml.safe_dump(
{
"local_datasource": {"target_project_ids": ["old-local"]},
"remote_datasource": {"target_project_ids": ["old-remote"]},
"strategies": {"project": {"config": {}}},
},
allow_unicode=True,
sort_keys=False,
),
encoding="utf-8",
)
config = {
"pipeline": {
"run_profile": str(source_profile),
"command": ["python", "run.py", "--config_path", "placeholder.yaml"],
}
}
profile_path, command = module.build_temp_run_profile_for_project_auto_bind(
config=config,
report_root=tmp_path,
profile_name="demo",
local_project_id="local-project",
remote_project_id="remote-project",
)
profile = yaml.safe_load(Path(profile_path).read_text(encoding="utf-8"))
project_config = profile["strategies"]["project"]["config"]
assert profile["local_datasource"]["target_project_ids"] == ["local-project"]
assert profile["remote_datasource"]["target_project_ids"] == ["remote-project"]
assert project_config["auto_bind"] is False
assert project_config["auto_bind_fields"] == []
assert project_config["pre_bind_data_id"] == [
{"local_data_id": "local-project", "remote_data_id": "remote-project"}
]
assert command[-1] == profile_path
@@ -7,6 +7,7 @@ from sync_state_machine.config import (
PipelineRunConfig,
JsonlDataSourceConfig,
ApiDataSourceConfig,
DbDataSourceConfig,
PersistConfig,
LoggingConfig,
)
@@ -101,3 +102,11 @@ def test_api_datasource_rate_limit_config_fields() -> None:
)
assert overridden.api_rate_limit_max_requests == 5
assert overridden.api_rate_limit_window_seconds == 2.5
def test_db_datasource_accepts_minimal_config() -> None:
cfg = DbDataSourceConfig(type="db")
assert cfg.type == "db"
assert cfg.target_project_ids == []
assert cfg.handler_configs == {}
@@ -0,0 +1,89 @@
from __future__ import annotations
from pathlib import Path
from pydantic import BaseModel
from sync_state_machine.common.registry import DomainRegistry
from sync_state_machine.common.sync_node import SyncNode
from sync_state_machine.config import DbDataSourceConfig, JsonlDataSourceConfig, LoggingConfig, PersistConfig, PipelineRunConfig
from sync_state_machine.datasource.db import BaseDbHandler, DbDataSource
from sync_state_machine.datasource.jsonl import BaseJsonlHandler, JsonlDataSource
from sync_state_machine.pipeline.factory import _register_handlers
from sync_state_machine.sync_system.strategy import DefaultSyncStrategy
class TestDbSchema(BaseModel):
__test__ = False
id: str
name: str
class TestDbNode(SyncNode[TestDbSchema]):
__test__ = False
node_type = "test_db_handler"
schema = TestDbSchema
class TestDbStrategy(DefaultSyncStrategy[TestDbSchema]):
__test__ = False
schema = TestDbSchema
class TestDbHandler(BaseDbHandler[TestDbSchema]):
__test__ = False
def __init__(self, datasource: DbDataSource):
super().__init__(datasource, "test_db_handler", TestDbNode, TestDbSchema)
def extract_id(self, data: dict[str, str]) -> str:
return str(data.get("id", ""))
async def load(self):
return []
async def create_all(self, nodes):
return []
async def update_all(self, nodes):
return []
class TestJsonlHandler(BaseJsonlHandler):
__test__ = False
def __init__(self, datasource: JsonlDataSource):
super().__init__(datasource, "test_db_handler", TestDbNode, TestDbSchema)
def test_register_handlers_uses_db_handler_for_db_datasource(tmp_path: Path) -> None:
if not DomainRegistry.is_registered("test_db_handler"):
DomainRegistry.register(
node_type="test_db_handler",
schema=TestDbSchema,
node_class=TestDbNode,
strategy_class=TestDbStrategy,
jsonl_handler_class=TestJsonlHandler,
)
DomainRegistry.register_db_handler("test_db_handler", TestDbHandler)
config = PipelineRunConfig(
node_types=["test_db_handler"],
local_datasource=DbDataSourceConfig(type="db"),
remote_datasource=JsonlDataSourceConfig(
type="jsonl",
jsonl_dir=str(tmp_path),
target_project_ids=[],
),
strategies=[],
persist=PersistConfig(db_path=str(tmp_path / "pipeline.db")),
logging=LoggingConfig(initialize=False),
)
local_ds = DbDataSource()
remote_ds = JsonlDataSource(tmp_path)
_register_handlers(config, local_ds, remote_ds, ["test_db_handler"])
assert isinstance(local_ds.get_handler("test_db_handler"), TestDbHandler)
assert isinstance(remote_ds.get_handler("test_db_handler"), TestJsonlHandler)
@@ -0,0 +1,105 @@
from __future__ import annotations
from pathlib import Path
import pytest
from pydantic import BaseModel
from sync_state_machine.common.registry import DomainRegistry
from sync_state_machine.common.sync_node import SyncNode
from sync_state_machine.config import DbDataSourceConfig, LoggingConfig, PersistConfig, PipelineRunConfig
from sync_state_machine.datasource.db import BaseDbHandler, DbDataSource
from sync_state_machine.pipeline.factory import create_pipeline_from_config
from sync_state_machine.sync_system.strategy import DefaultSyncStrategy
class EndpointInitSchema(BaseModel):
__test__ = False
id: str
name: str | None = None
class EndpointInitNode(SyncNode[EndpointInitSchema]):
__test__ = False
node_type = "factory_endpoint_demo"
schema = EndpointInitSchema
class EndpointInitStrategy(DefaultSyncStrategy[EndpointInitSchema]):
__test__ = False
schema = EndpointInitSchema
class EndpointInitDbHandler(BaseDbHandler[EndpointInitSchema]):
__test__ = False
def __init__(self, datasource: DbDataSource):
super().__init__(datasource, "factory_endpoint_demo", EndpointInitNode, EndpointInitSchema)
def extract_id(self, data: dict[str, str]) -> str:
return str(data.get("id", ""))
async def load(self):
return []
async def create_all(self, nodes):
return []
async def update_all(self, nodes):
return []
def _ensure_registered() -> None:
node_type = "factory_endpoint_demo"
if DomainRegistry.is_registered(node_type):
reg = DomainRegistry.get_registration(node_type)
reg.schema = EndpointInitSchema
reg.node_class = EndpointInitNode
reg.strategy_class = EndpointInitStrategy
reg.db_handler_class = EndpointInitDbHandler
return
DomainRegistry.register(
node_type=node_type,
schema=EndpointInitSchema,
node_class=EndpointInitNode,
strategy_class=EndpointInitStrategy,
db_handler_class=EndpointInitDbHandler,
)
@pytest.mark.asyncio
async def test_factory_initializes_same_datasource_type_with_distinct_handler_configs(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
_ensure_registered()
config = PipelineRunConfig(
node_types=["factory_endpoint_demo"],
local_datasource=DbDataSourceConfig(
type="db",
handler_configs={"factory_endpoint_demo": {"side": "local", "batch_size": 10}},
target_project_ids=["project_local"],
),
remote_datasource=DbDataSourceConfig(
type="db",
handler_configs={"factory_endpoint_demo": {"side": "remote", "batch_size": 20}},
target_project_ids=["project_remote"],
),
strategies=[],
persist=PersistConfig(db_path=str(tmp_path / "factory-init.db")),
logging=LoggingConfig(initialize=False),
)
pipeline = await create_pipeline_from_config(config)
try:
local_handler = pipeline.local_datasource.get_handler("factory_endpoint_demo")
remote_handler = pipeline.remote_datasource.get_handler("factory_endpoint_demo")
assert isinstance(local_handler, EndpointInitDbHandler)
assert isinstance(remote_handler, EndpointInitDbHandler)
assert local_handler.handler_config == {"side": "local", "batch_size": 10}
assert remote_handler.handler_config == {"side": "remote", "batch_size": 20}
finally:
await pipeline.close()
+86
View File
@@ -0,0 +1,86 @@
from __future__ import annotations
from typing import Any
from pydantic import BaseModel
from sync_state_machine.common.sync_node import SyncNode
from sync_state_machine.datasource.api.handler import BaseApiHandler
from sync_state_machine.datasource.handler import BaseNodeHandler, CompatibleNodeHandler
from sync_state_machine.datasource.task_result import TaskResult
class SharedConfigSchema(BaseModel):
__test__ = False
id: str
class SharedConfigNode(SyncNode[SharedConfigSchema]):
__test__ = False
node_type = "shared_config_demo"
schema = SharedConfigSchema
class SharedConfigBaseHandler(BaseNodeHandler[SharedConfigSchema]):
__test__ = False
def __init__(self) -> None:
super().__init__("shared_config_demo", SharedConfigNode, SharedConfigSchema)
async def load(self):
return []
async def create_all(self, nodes):
return []
async def update_all(self, nodes):
return []
def extract_id(self, data: dict[str, Any]) -> str:
return str(data.get("id", ""))
class SharedConfigApiHandler(BaseApiHandler[SharedConfigSchema]):
__test__ = False
def __init__(self) -> None:
super().__init__()
self._node_type = "shared_api_demo"
self._node_class = SharedConfigNode
self._schema = SharedConfigSchema
async def load(self):
return []
async def create_all(self, nodes):
return [TaskResult.success(node_id=node.node_id) for node in nodes]
async def update_all(self, nodes):
return [TaskResult.success(node_id=node.node_id) for node in nodes]
async def delete_all(self, nodes):
return [TaskResult.success(node_id=node.node_id) for node in nodes]
def test_base_handler_supports_shared_handler_config() -> None:
handler = SharedConfigBaseHandler()
handler.set_handler_config({"mode": "local", "batch_size": 10})
assert handler.handler_config == {"mode": "local", "batch_size": 10}
def test_compatible_handler_forwards_shared_handler_config() -> None:
handler = CompatibleNodeHandler(SharedConfigBaseHandler())
handler.set_handler_config({"mode": "remote"})
assert handler.handler_config == {"mode": "remote"}
def test_api_handler_supports_shared_handler_config() -> None:
handler = SharedConfigApiHandler()
handler.set_handler_config({"filter_project_id": ["p1", "p2"]})
assert handler.handler_config == {"filter_project_id": ["p1", "p2"]}
@@ -0,0 +1,62 @@
from __future__ import annotations
from typing import Any
import pytest
from pydantic import BaseModel
from sync_state_machine.common.sync_node import SyncNode
from sync_state_machine.domain.material_detail.api_handler import MaterialDetailApiHandler
from sync_state_machine.datasource.task_result import TaskStatus
class _MaterialDetailSchema(BaseModel):
id: str
parent_id: str
quantity: float
remark: str | None = None
class _MaterialDetailNode(SyncNode[_MaterialDetailSchema]):
node_type = "material_detail"
schema = _MaterialDetailSchema
class _MockApiClient:
def __init__(self) -> None:
self.requests: list[dict[str, Any]] = []
async def request(self, method: str, endpoint: str, **kwargs: Any) -> dict[str, Any]:
self.requests.append({"method": method, "endpoint": endpoint, **kwargs})
return {"code": 200, "message": "OK", "data": {"biz_id": kwargs.get("data", {}).get("biz_id")}}
@pytest.mark.asyncio
async def test_material_detail_update_returns_in_progress_with_shared_push_id() -> None:
handler = MaterialDetailApiHandler()
handler.api_client = _MockApiClient()
node = _MaterialDetailNode(
node_id="detail-node-1",
data_id="detail-1",
data={
"id": "detail-1",
"parent_id": "material-1",
"quantity": 10.0,
"remark": "new",
},
origin_data={
"id": "detail-1",
"parent_id": "material-1",
"quantity": 5.0,
"remark": None,
},
context={"material_id": "material-1"},
)
results = await handler.update_all([node])
assert len(results) == 1
assert results[0].status == TaskStatus.IN_PROGRESS
assert results[0].task_id
assert handler.api_client.requests[0]["endpoint"] == "/material/detail/update"
@@ -0,0 +1,84 @@
from __future__ import annotations
from uuid import uuid4
from sync_state_machine.domain.project.api_handler import _merge_project_all_info_fields
from sync_state_machine.domain.project.sync_node import ProjectSyncNode
def test_merge_project_all_info_fields_flattens_extension_investment_fields() -> None:
project_data = {
"id": "project-1",
"name": "Demo Project",
"dynamic_investment": 123.0,
}
all_info = {
"project": {"id": "project-1"},
"hydropower_extension": {
"implementation_estimate": 1000.0,
"static_total_investment": 990.0,
"completed_investment": 120.0,
"total_contract_quantity": 50.0,
},
}
merged = _merge_project_all_info_fields(project_data, all_info)
assert merged["dynamic_investment"] == 123.0
assert merged["implementation_estimate"] == 1000.0
assert merged["static_total_investment"] == 990.0
assert merged["completed_investment"] == 120.0
assert merged["total_contract_quantity"] == 50.0
def test_merge_project_all_info_fields_keeps_existing_project_values() -> None:
project_data = {
"id": "project-1",
"implementation_estimate": 2000.0,
"static_total_investment": 1800.0,
}
all_info = {
"hydropower_extension": {
"implementation_estimate": 1000.0,
"static_total_investment": 990.0,
"completed_investment": 120.0,
},
}
merged = _merge_project_all_info_fields(project_data, all_info)
assert merged["implementation_estimate"] == 2000.0
assert merged["static_total_investment"] == 1800.0
assert merged["completed_investment"] == 120.0
def test_project_sync_node_preserves_extension_backed_fields() -> None:
project_id = str(uuid4())
data = {
"_id": project_id,
"id": project_id,
"project_type": "hydropower",
"region_name": ["中国", "测试省"],
"region_path": ["100000", "110000"],
"province": "110000",
"is_domestic": True,
"company_id": str(uuid4()),
"company_name": "测试公司",
"address": "测试地址",
"code": "P-001",
"name": "测试项目",
"progress_type": "A",
"dynamic_investment": 100.0,
"complete_investment": 80.0,
"implementation_estimate": 120.0,
"static_total_investment": 110.0,
"completed_investment": 70.0,
}
node = ProjectSyncNode(node_id=str(uuid4()), data_id=data["id"], data=data)
normalized = node.get_data()
assert normalized is not None
assert normalized["implementation_estimate"] == 120.0
assert normalized["static_total_investment"] == 110.0
assert normalized["completed_investment"] == 70.0
@@ -0,0 +1,56 @@
from __future__ import annotations
from typing import Any
import pytest
from pydantic import BaseModel
from sync_state_machine.common.sync_node import SyncNode
from sync_state_machine.domain.project.api_handler import ProjectApiHandler
from sync_state_machine.datasource.task_result import TaskStatus
class _ProjectUpdateSchema(BaseModel):
id: str
dynamic_investment: float
class _ProjectUpdateNode(SyncNode[_ProjectUpdateSchema]):
node_type = "project"
schema = _ProjectUpdateSchema
class _MockApiClient:
def __init__(self) -> None:
self.requests: list[dict[str, Any]] = []
self._poll_count = 0
async def request(self, method: str, endpoint: str, **kwargs: Any) -> dict[str, Any]:
self.requests.append({"method": method, "endpoint": endpoint, **kwargs})
return {"code": 200, "message": "OK", "data": {"biz_id": kwargs.get("data", {}).get("biz_id")}}
async def get_push_logs(self, task_ids: list[str]) -> dict[str, dict[str, Any]]:
self._poll_count += 1
if self._poll_count == 1:
return {task_ids[0]: {}}
return {task_ids[0]: {"code": 200, "data": {"biz_id": "project-1"}}}
@pytest.mark.asyncio
async def test_project_update_waits_for_async_push_completion() -> None:
handler = ProjectApiHandler(filter_project_id=["project-1"])
handler.api_client = _MockApiClient()
node = _ProjectUpdateNode(
node_id="project-node-1",
data_id="project-1",
data={"id": "project-1", "dynamic_investment": 200.0},
origin_data={"id": "project-1", "dynamic_investment": 100.0},
)
results = await handler.update_all([node])
assert len(results) == 1
assert results[0].status == TaskStatus.SUCCESS
assert handler.api_client.requests[0]["endpoint"] == "/project/dynamic_investment"
assert handler.api_client._poll_count == 2
+53
View File
@@ -0,0 +1,53 @@
from __future__ import annotations
import importlib.util
from pathlib import Path
import sys
def _load_case_module():
scripts_dir = Path(__file__).resolve().parents[2] / "scripts"
if str(scripts_dir) not in sys.path:
sys.path.insert(0, str(scripts_dir))
file_path = scripts_dir / "auto_test_domains" / "project_case.py"
spec = importlib.util.spec_from_file_location("project_case_for_test", file_path)
if spec is None or spec.loader is None:
raise RuntimeError(f"Unable to load module from {file_path}")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_project_case_round_3_focuses_on_stable_readback_fields(monkeypatch, tmp_path: Path) -> None:
module = _load_case_module()
captured_rounds: dict[str, dict] = {}
def _fake_run_pipeline_round(**kwargs):
captured_rounds[kwargs["round_name"]] = kwargs
return {"name": kwargs["round_name"], "passed": True, "issues": []}
def _fake_build_domain_report(**kwargs):
return kwargs
monkeypatch.setattr(module, "run_pipeline_round", _fake_run_pipeline_round)
monkeypatch.setattr(module, "build_domain_report", _fake_build_domain_report)
module.run_case(config={"project_root": str(tmp_path)}, report_root=tmp_path)
round_3 = captured_rounds["round_3_investment_update"]
expected_fields = round_3["expected_bound_fields"]["remote"]["project"][module.LOCAL_PROJECT_ID]
changed_fields = round_3["expected_changes"]["project"][module.LOCAL_PROJECT_ID]
assert set(changed_fields) == {
"investment_decision_capacity",
"investment_decision_date",
"complete_investment",
"dynamic_investment",
}
assert set(expected_fields.keys()) == {
"investment_decision_capacity",
"investment_decision_date",
"complete_investment",
"dynamic_investment",
}
assert "PUT /project/settlement_investment" in round_3["coverage"]["api"]
@@ -0,0 +1,46 @@
from __future__ import annotations
import importlib.util
from pathlib import Path
import sys
def _load_case_module():
scripts_dir = Path(__file__).resolve().parents[2] / "scripts"
if str(scripts_dir) not in sys.path:
sys.path.insert(0, str(scripts_dir))
file_path = scripts_dir / "auto_test_domains" / "project_detail_data_case.py"
spec = importlib.util.spec_from_file_location("project_detail_data_case_for_test", file_path)
if spec is None or spec.loader is None:
raise RuntimeError(f"Unable to load module from {file_path}")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_project_detail_case_excludes_production_from_unified_rounds(monkeypatch, tmp_path: Path) -> None:
module = _load_case_module()
recorded_rounds: list[str] = []
def _fake_run_pipeline_round(**kwargs):
recorded_rounds.append(kwargs["round_name"])
return {"name": kwargs["round_name"], "passed": True, "issues": []}
def _fake_build_domain_report(**kwargs):
return kwargs
monkeypatch.setattr(module, "run_pipeline_round", _fake_run_pipeline_round)
monkeypatch.setattr(module, "build_domain_report", _fake_build_domain_report)
report = module.run_case(config={"project_root": str(tmp_path)}, report_root=tmp_path)
assert recorded_rounds == [
"round_1_create",
"round_2_update",
"round_3_plan_construction_create",
"round_5_ensure_production_push",
"round_6_climb_production_push",
"round_7_challenge_production_push",
]
assert "production" in report["description"]
assert "不混入本组统一回归" in report["description"]
@@ -0,0 +1,49 @@
from __future__ import annotations
from pydantic import BaseModel
from sync_state_machine.common.registry import DomainRegistry
class RegistryLookupSchema(BaseModel):
__test__ = False
id: str
class RegistryLookupNode:
__test__ = False
class RegistryLookupJsonlHandler:
__test__ = False
class RegistryLookupApiHandler:
__test__ = False
class RegistryLookupDbHandler:
__test__ = False
def test_registry_supports_unified_handler_lookup() -> None:
node_type = "registry_lookup_demo"
if DomainRegistry.is_registered(node_type):
reg = DomainRegistry.get_registration(node_type)
reg.jsonl_handler_class = RegistryLookupJsonlHandler
reg.api_handler_class = RegistryLookupApiHandler
reg.db_handler_class = RegistryLookupDbHandler
else:
DomainRegistry.register(
node_type=node_type,
schema=RegistryLookupSchema,
node_class=RegistryLookupNode,
jsonl_handler_class=RegistryLookupJsonlHandler,
api_handler_class=RegistryLookupApiHandler,
db_handler_class=RegistryLookupDbHandler,
)
assert DomainRegistry.get_handler(node_type, "jsonl") is RegistryLookupJsonlHandler
assert DomainRegistry.get_handler(node_type, "api") is RegistryLookupApiHandler
assert DomainRegistry.get_handler(node_type, "db") is RegistryLookupDbHandler