多pipeline同步支持
This commit is contained in:
+2
-1
@@ -12,6 +12,7 @@
|
||||
- 工具链专项:`pytest tests/integration/test_toolchain_config_and_graph.py -q`
|
||||
- UI 专项:`pytest tests/integration/test_ui_api_smoke.py -q`
|
||||
- 测试总览与远程环境工具:`docs/testing_guide.md`
|
||||
- 最近常用的定向回归:`pytest tests/unit/test_update_ops.py tests/unit/test_post_check_dependency_id_normalization.py -q`
|
||||
|
||||
## 设计原则
|
||||
- 不绕过 schema/状态机校验;配置错误必须显式失败。
|
||||
@@ -26,5 +27,5 @@
|
||||
- 覆盖核心业务链路(bind/create/update 与 full pipeline)。
|
||||
|
||||
## 大规模自动回归
|
||||
- 按轮迭代的大规模自动测试方法见 [auto_test_spec/agent_auto_test_method.md](auto_test_spec/agent_auto_test_method.md)。
|
||||
- 按轮迭代的大规模自动测试方法见 [../docs/auto_test_spec/agent_auto_test_method.md](../docs/auto_test_spec/agent_auto_test_method.md)。
|
||||
- 执行时优先采用“单 domain 复现 -> 修复/分类 -> 全量聚合回归 -> 记录总计变化”的节奏,避免一次携带过多上下文。
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from sync_state_machine.common.persistence import PersistenceBackend
|
||||
from sync_state_machine.pipeline import run_profile_from_file
|
||||
from sync_state_machine.pipeline.full_sync_pipeline import FullSyncPipeline
|
||||
|
||||
|
||||
def _write_yaml(path: Path, content: str) -> None:
|
||||
path.write_text(content.strip() + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multi_project_pipeline_runs_jsonl_to_jsonl_with_shared_persistence(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
pytest.importorskip("yaml")
|
||||
|
||||
async def _skip_post_check(self) -> bool:
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(FullSyncPipeline, "phase_post_check", _skip_post_check)
|
||||
|
||||
project_root = Path(__file__).resolve().parents[2]
|
||||
local_dir = project_root / "working_local_datasource" / "20260302-020003"
|
||||
remote_dir = tmp_path / "remote_jsonl"
|
||||
db_path = tmp_path / "multi_project.db"
|
||||
logs_dir = tmp_path / "logs"
|
||||
remote_dir.mkdir(parents=True, exist_ok=True)
|
||||
logs_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
global_cfg = tmp_path / "global.yaml"
|
||||
project_cfg = tmp_path / "project.yaml"
|
||||
multi_cfg = tmp_path / "multi.yaml"
|
||||
|
||||
_write_yaml(
|
||||
global_cfg,
|
||||
f"""
|
||||
node_types:
|
||||
- project
|
||||
local_datasource:
|
||||
type: jsonl
|
||||
jsonl_dir: {local_dir.as_posix()}
|
||||
read_only: false
|
||||
remote_datasource:
|
||||
type: jsonl
|
||||
jsonl_dir: {remote_dir.as_posix()}
|
||||
read_only: false
|
||||
persist:
|
||||
backend: sqlite
|
||||
db_path: {db_path.as_posix()}
|
||||
enable: true
|
||||
wipe_on_start: true
|
||||
logging:
|
||||
initialize: true
|
||||
file_dir: {logs_dir.as_posix()}
|
||||
file_name_pattern: multi_global_{{timestamp}}.log
|
||||
level: 20
|
||||
strategies:
|
||||
project:
|
||||
skip_sync: false
|
||||
config:
|
||||
auto_bind: true
|
||||
auto_bind_fields:
|
||||
- name
|
||||
depend_fields: {{}}
|
||||
local_orphan_action: create_remote
|
||||
remote_orphan_action: none
|
||||
update_direction: push
|
||||
""",
|
||||
)
|
||||
|
||||
_write_yaml(
|
||||
project_cfg,
|
||||
f"""
|
||||
node_types:
|
||||
- lar
|
||||
local_datasource:
|
||||
type: jsonl
|
||||
jsonl_dir: {local_dir.as_posix()}
|
||||
read_only: false
|
||||
remote_datasource:
|
||||
type: jsonl
|
||||
jsonl_dir: {remote_dir.as_posix()}
|
||||
read_only: false
|
||||
persist:
|
||||
backend: sqlite
|
||||
db_path: {db_path.as_posix()}
|
||||
enable: true
|
||||
wipe_on_start: false
|
||||
logging:
|
||||
initialize: true
|
||||
file_dir: {logs_dir.as_posix()}
|
||||
file_name_pattern: multi_project_{{timestamp}}.log
|
||||
level: 20
|
||||
strategies:
|
||||
lar:
|
||||
skip_sync: false
|
||||
config:
|
||||
auto_bind: true
|
||||
auto_bind_fields:
|
||||
- project_id
|
||||
depend_fields:
|
||||
project_id: project
|
||||
local_orphan_action: create_remote
|
||||
remote_orphan_action: none
|
||||
update_direction: push
|
||||
""",
|
||||
)
|
||||
|
||||
_write_yaml(
|
||||
multi_cfg,
|
||||
f"""
|
||||
global_config: {global_cfg.as_posix()}
|
||||
default_project_config: {project_cfg.as_posix()}
|
||||
projects:
|
||||
- local_project_id: f3e02e84-e81c-4aa6-88d4-f5aa108c8227
|
||||
remote_project_id: f3e02e84-e81c-4aa6-88d4-f5aa108c8227
|
||||
- local_project_id: 0f9c3e22-f4bb-4803-825a-4f238eeb06d5
|
||||
remote_project_id: 0f9c3e22-f4bb-4803-825a-4f238eeb06d5
|
||||
""",
|
||||
)
|
||||
|
||||
results = await run_profile_from_file(project_root=project_root, config_path=multi_cfg, print_summary=False)
|
||||
|
||||
assert "global" in results
|
||||
assert "project_id:f3e02e84-e81c-4aa6-88d4-f5aa108c8227" in results
|
||||
assert "project_id:0f9c3e22-f4bb-4803-825a-4f238eeb06d5" in results
|
||||
|
||||
_, scope_rows = PersistenceBackend.query_records(
|
||||
backend="sqlite",
|
||||
db_path=str(db_path),
|
||||
sql="SELECT DISTINCT scope FROM nodes ORDER BY scope",
|
||||
)
|
||||
scopes = [row["scope"] for row in scope_rows]
|
||||
assert scopes == [
|
||||
"global",
|
||||
"project_id:0f9c3e22-f4bb-4803-825a-4f238eeb06d5",
|
||||
"project_id:f3e02e84-e81c-4aa6-88d4-f5aa108c8227",
|
||||
]
|
||||
|
||||
_, copied_rows = PersistenceBackend.query_records(
|
||||
backend="sqlite",
|
||||
db_path=str(db_path),
|
||||
sql=(
|
||||
"SELECT scope, collection_id, node_type, COUNT(1) AS c "
|
||||
"FROM nodes "
|
||||
"WHERE scope <> 'global' AND node_type = 'project' "
|
||||
"GROUP BY scope, collection_id, node_type "
|
||||
"ORDER BY scope, collection_id, node_type"
|
||||
),
|
||||
)
|
||||
assert copied_rows == []
|
||||
@@ -1,11 +1,13 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from sync_state_machine.common.binding import BindingManager
|
||||
from sync_state_machine.common.collection import DataCollection
|
||||
from sync_state_machine.datasource.jsonl import JsonlDataSource
|
||||
from sync_state_machine.pipeline.full_sync_pipeline import FullSyncPipeline
|
||||
|
||||
|
||||
@@ -93,4 +95,52 @@ async def test_phase_sync_loads_each_node_type_before_strategy(monkeypatch: pyte
|
||||
("project", "create wrote data"),
|
||||
("preparation", "pre-strategy refresh"),
|
||||
("contract", "pre-strategy refresh"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_phase_sync_skips_reload_for_jsonl_datasources(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
project_strategy = _StubStrategy("project")
|
||||
|
||||
pipeline = FullSyncPipeline(
|
||||
local_datasource=JsonlDataSource(tmp_path / "local", read_only=True),
|
||||
remote_datasource=JsonlDataSource(tmp_path / "remote", read_only=True),
|
||||
strategies=[project_strategy],
|
||||
persistence=_FakePersistence(),
|
||||
local_collection=DataCollection("local"),
|
||||
remote_collection=DataCollection("remote"),
|
||||
binding_manager=BindingManager(_FakePersistence(), auto_persist=False),
|
||||
node_types=["project"],
|
||||
load_order=["project"],
|
||||
sync_order=["project"],
|
||||
enable_persistence=False,
|
||||
)
|
||||
|
||||
reload_calls: list[tuple[str, str]] = []
|
||||
|
||||
async def fake_commit_creates(node_type: str) -> bool:
|
||||
return node_type == "project"
|
||||
|
||||
async def fake_commit_updates(node_type: str) -> bool:
|
||||
del node_type
|
||||
return False
|
||||
|
||||
async def fake_load_node_type(node_type: str, *, reason: str) -> None:
|
||||
reload_calls.append((node_type, reason))
|
||||
|
||||
async def fake_normalize(node_type: str) -> None:
|
||||
del node_type
|
||||
|
||||
monkeypatch.setattr(pipeline, "commit_creates", fake_commit_creates)
|
||||
monkeypatch.setattr(pipeline, "commit_updates", fake_commit_updates)
|
||||
monkeypatch.setattr(pipeline, "_load_node_type", fake_load_node_type)
|
||||
monkeypatch.setattr(pipeline, "normalize_create_success_to_update_entry", fake_normalize)
|
||||
|
||||
await pipeline.phase_sync_by_node_type()
|
||||
|
||||
assert reload_calls == [
|
||||
("project", "pre-strategy refresh"),
|
||||
]
|
||||
@@ -0,0 +1,123 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
from pydantic import BaseModel
|
||||
|
||||
from sync_state_machine.config.multi_project_config import build_multi_project_config_from_file
|
||||
from sync_state_machine.common.binding import BindingManager
|
||||
from sync_state_machine.common.collection import DataCollection
|
||||
from sync_state_machine.common.persistence import PersistenceBackend
|
||||
from sync_state_machine.common.sync_node import SyncNode
|
||||
from sync_state_machine.pipeline.multi_project_pipeline import (
|
||||
_project_scope_overrides,
|
||||
_resolve_project_bootstrap_node_types,
|
||||
)
|
||||
|
||||
|
||||
def test_build_multi_project_config_reads_explicit_project_bootstrap_node_types(tmp_path: Path) -> None:
|
||||
config_path = tmp_path / "multi.yaml"
|
||||
config_path.write_text(
|
||||
yaml.safe_dump(
|
||||
{
|
||||
"global_config": "run_profiles/global.yaml",
|
||||
"default_project_config": "run_profiles/project.yaml",
|
||||
"project_bootstrap_node_types": ["company", "supplier"],
|
||||
"projects": [
|
||||
{
|
||||
"local_project_id": "local-project",
|
||||
"remote_project_id": "remote-project",
|
||||
}
|
||||
],
|
||||
},
|
||||
allow_unicode=True,
|
||||
sort_keys=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
config = build_multi_project_config_from_file(project_root=tmp_path, file_path=config_path)
|
||||
|
||||
assert config.project_bootstrap_node_types == ["company", "supplier"]
|
||||
|
||||
|
||||
def test_project_scope_overrides_only_inject_project_handler_filter() -> None:
|
||||
item = type("Item", (), {"local_project_id": "local-project", "remote_project_id": "remote-project"})()
|
||||
|
||||
overrides = _project_scope_overrides(item, node_types=["project", "contract", "user"])
|
||||
|
||||
assert overrides == {
|
||||
"local_datasource": {
|
||||
"handler_configs": {
|
||||
"project": {"data_id_filter": ["local-project"]},
|
||||
}
|
||||
},
|
||||
"remote_datasource": {
|
||||
"handler_configs": {
|
||||
"project": {"data_id_filter": ["remote-project"]},
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_resolve_project_bootstrap_node_types_prefers_explicit_config() -> None:
|
||||
config = type(
|
||||
"Config",
|
||||
(),
|
||||
{"project_bootstrap_node_types": ["company", "supplier", "company"]},
|
||||
)()
|
||||
|
||||
resolved = _resolve_project_bootstrap_node_types(
|
||||
config=config,
|
||||
shared_node_types=["company", "supplier", "project"],
|
||||
)
|
||||
|
||||
assert resolved == ["company", "supplier"]
|
||||
|
||||
|
||||
class _ProjectSchema(BaseModel):
|
||||
id: str
|
||||
|
||||
|
||||
class _ProjectNode(SyncNode[dict]):
|
||||
node_type = "project"
|
||||
schema = _ProjectSchema
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_binding_manager_prunes_stale_bindings_after_project_scope_cleanup() -> None:
|
||||
persistence = PersistenceBackend(":memory:")
|
||||
await persistence.initialize()
|
||||
try:
|
||||
local_collection = DataCollection("local", persistence=persistence, auto_persist=False)
|
||||
remote_collection = DataCollection("remote", persistence=persistence, auto_persist=False)
|
||||
binding_manager = BindingManager(persistence, auto_persist=False, scope="project_id:p1")
|
||||
|
||||
local_node = _ProjectNode(node_id="local-project-1", data={"id": "p1"}, data_id="p1")
|
||||
remote_node = _ProjectNode(node_id="remote-project-1", data={"id": "p1"}, data_id="p1")
|
||||
stale_local_node = _ProjectNode(node_id="local-project-2", data={"id": "p2"}, data_id="p2")
|
||||
stale_remote_node = _ProjectNode(node_id="remote-project-2", data={"id": "p2"}, data_id="p2")
|
||||
|
||||
await local_collection.add(local_node)
|
||||
await local_collection.add(stale_local_node)
|
||||
await remote_collection.add(remote_node)
|
||||
await remote_collection.add(stale_remote_node)
|
||||
await binding_manager.bind("project", "local-project-1", "remote-project-1")
|
||||
await binding_manager.bind("project", "local-project-2", "remote-project-2")
|
||||
|
||||
await local_collection.filter_by_project_ids(["p1"])
|
||||
await remote_collection.filter_by_project_ids(["p1"])
|
||||
|
||||
deleted = await binding_manager.prune_missing_nodes(
|
||||
local_collection=local_collection,
|
||||
remote_collection=remote_collection,
|
||||
node_types=["project"],
|
||||
)
|
||||
|
||||
assert deleted == 1
|
||||
assert await binding_manager.get_remote_id("project", "local-project-1") == "remote-project-1"
|
||||
assert await binding_manager.get_remote_id("project", "local-project-2") is None
|
||||
finally:
|
||||
await persistence.close()
|
||||
@@ -10,7 +10,9 @@ from sync_state_machine.common.collection import DataCollection
|
||||
from sync_state_machine.common.persistence import PersistenceBackend
|
||||
from sync_state_machine.common.types import BindingStatus, SyncAction, SyncStatus
|
||||
from sync_state_machine.domain.construction.sync_node import ConstructionTaskSyncNode
|
||||
from sync_state_machine.domain.construction.sync_strategy import ConstructionTaskSyncStrategy
|
||||
from sync_state_machine.domain.contract.sync_node import ContractSyncNode
|
||||
from sync_state_machine.domain.contract.sync_strategy import ContractSyncStrategy
|
||||
from sync_state_machine.sync_system.strategy_ops.post_check_ops import evaluate_consistency_for_node_type
|
||||
|
||||
|
||||
@@ -138,8 +140,16 @@ async def test_post_check_uses_same_id_resolver_as_create_update_for_depend_ids(
|
||||
await remote_collection.add(remote_task)
|
||||
await binding_manager.bind("construction_task", local_task.node_id, remote_task.node_id)
|
||||
|
||||
strategy = ConstructionTaskSyncStrategy(
|
||||
"construction_task",
|
||||
local_collection,
|
||||
remote_collection,
|
||||
binding_manager,
|
||||
)
|
||||
|
||||
result = await evaluate_consistency_for_node_type(
|
||||
node_type="construction_task",
|
||||
strategy=strategy,
|
||||
local_collection=local_collection,
|
||||
remote_collection=remote_collection,
|
||||
binding_manager=binding_manager,
|
||||
@@ -152,3 +162,92 @@ async def test_post_check_uses_same_id_resolver_as_create_update_for_depend_ids(
|
||||
assert result["mismatch_count"] == 0
|
||||
|
||||
await persistence.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_check_ignores_only_explicit_configured_fields(tmp_path: Path) -> None:
|
||||
db_path = tmp_path / "state.db"
|
||||
persistence = PersistenceBackend(str(db_path))
|
||||
await persistence.initialize()
|
||||
|
||||
local_collection = DataCollection("local", persistence=persistence, auto_persist=False)
|
||||
remote_collection = DataCollection("remote", persistence=persistence, auto_persist=False)
|
||||
binding_manager = BindingManager(persistence, auto_persist=False)
|
||||
|
||||
local_contract = ContractSyncNode(
|
||||
node_id="local-contract-node",
|
||||
data_id="local-contract-1",
|
||||
data={
|
||||
"id": "local-contract-1",
|
||||
"project_id": "",
|
||||
"name": "合同A-LOCAL",
|
||||
"short_name": "合同A",
|
||||
"code": "C-LOCAL",
|
||||
"contract_type": "施工",
|
||||
"currency": "CNY",
|
||||
"manager_name": "张三",
|
||||
"manager_phone": "13800000000",
|
||||
"sign_company": "示例公司",
|
||||
"company_id": "company-1",
|
||||
"sign_date": "2025-01-01",
|
||||
"credit_code": None,
|
||||
"total_price": 1.0,
|
||||
"sub_name": None,
|
||||
},
|
||||
binding_status=BindingStatus.NORMAL,
|
||||
action=SyncAction.NONE,
|
||||
status=SyncStatus.SUCCESS,
|
||||
)
|
||||
remote_contract = ContractSyncNode(
|
||||
node_id="remote-contract-node",
|
||||
data_id="remote-contract-1",
|
||||
data={
|
||||
"id": "remote-contract-1",
|
||||
"project_id": "",
|
||||
"name": "合同A-REMOTE",
|
||||
"short_name": "合同A",
|
||||
"code": "C-REMOTE",
|
||||
"contract_type": "施工",
|
||||
"currency": "CNY",
|
||||
"manager_name": "张三",
|
||||
"manager_phone": "13800000000",
|
||||
"sign_company": "示例公司",
|
||||
"company_id": "company-1",
|
||||
"sign_date": "2025-01-01",
|
||||
"credit_code": None,
|
||||
"total_price": 1.0,
|
||||
"sub_name": None,
|
||||
},
|
||||
binding_status=BindingStatus.NORMAL,
|
||||
action=SyncAction.NONE,
|
||||
status=SyncStatus.SUCCESS,
|
||||
)
|
||||
|
||||
await local_collection.add(local_contract)
|
||||
await remote_collection.add(remote_contract)
|
||||
await binding_manager.bind("contract", local_contract.node_id, remote_contract.node_id)
|
||||
|
||||
strategy = ContractSyncStrategy(
|
||||
"contract",
|
||||
local_collection,
|
||||
remote_collection,
|
||||
binding_manager,
|
||||
)
|
||||
|
||||
result = await evaluate_consistency_for_node_type(
|
||||
node_type="contract",
|
||||
strategy=strategy,
|
||||
local_collection=local_collection,
|
||||
remote_collection=remote_collection,
|
||||
binding_manager=binding_manager,
|
||||
logger=logging.getLogger("sync_state_machine"),
|
||||
compare_to_remote=True,
|
||||
ignore_fields=["code"],
|
||||
)
|
||||
|
||||
field_names = {field["field_name"] for field in result["fields"]}
|
||||
assert "code" not in field_names
|
||||
assert "name" in field_names
|
||||
assert result["ignore_fields"] == ["code"]
|
||||
|
||||
await persistence.close()
|
||||
|
||||
@@ -6,7 +6,7 @@ import pytest
|
||||
|
||||
from pydantic import ConfigDict
|
||||
|
||||
from sync_state_machine.config import BaseDataSourceConfig, build_config_from_file, register_datasource_type
|
||||
from sync_state_machine.config import BaseDataSourceConfig, build_config_from_file, config_snapshot, register_datasource_type
|
||||
from sync_state_machine.config.run_presets import load_overrides_from_file
|
||||
from sync_state_machine.datasource import BaseDataSource
|
||||
|
||||
@@ -123,3 +123,84 @@ logging:
|
||||
|
||||
assert config.local_datasource.type == "file_backed"
|
||||
assert getattr(config.local_datasource, "namespace") == "local-demo"
|
||||
|
||||
|
||||
def test_build_config_from_sparse_profile_resolves_full_strategy_defaults(tmp_path: Path) -> None:
|
||||
profile = tmp_path / "sparse_profile.yaml"
|
||||
profile.write_text(
|
||||
"""
|
||||
node_types:
|
||||
- project
|
||||
- production
|
||||
local_datasource:
|
||||
type: jsonl
|
||||
jsonl_dir: ${PROJECT_ROOT}/filtered_datasource/datasource/filtered
|
||||
remote_datasource:
|
||||
type: jsonl
|
||||
jsonl_dir: ${PROJECT_ROOT}/remote_datasource/datasource
|
||||
persist:
|
||||
db_path: ${PROJECT_ROOT}/runtime/test.db
|
||||
logging:
|
||||
initialize: false
|
||||
strategies:
|
||||
project:
|
||||
config:
|
||||
local_orphan_action: none
|
||||
post_check_ignore_fields:
|
||||
- code
|
||||
production:
|
||||
config:
|
||||
post_check_ignore_fields:
|
||||
- is_exist
|
||||
""".strip(),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
config = build_config_from_file(project_root=tmp_path, file_path=profile)
|
||||
strategies = {item.node_type: item for item in config.strategies}
|
||||
|
||||
project = strategies["project"].config
|
||||
assert project.auto_bind is False
|
||||
assert project.auto_bind_fields == []
|
||||
assert project.depend_fields == {"company_id": "company"}
|
||||
assert project.local_orphan_action == "none"
|
||||
assert project.remote_orphan_action == "none"
|
||||
assert project.update_direction == "push"
|
||||
assert project.post_check_ignore_fields == ["code"]
|
||||
assert project.compare_ignore_fields == []
|
||||
|
||||
production = strategies["production"].config
|
||||
assert production.auto_bind_fields == ["project_id", "code"]
|
||||
assert production.depend_fields == {"project_id": "project"}
|
||||
assert production.post_check_ignore_fields == ["is_exist"]
|
||||
|
||||
snapshot = config_snapshot(config)
|
||||
assert snapshot["strategies"][0]["config"]["compare_ignore_fields"] == []
|
||||
assert snapshot["strategies"][0]["config"]["field_direction_overrides"] == {}
|
||||
|
||||
|
||||
def test_build_config_from_file_rejects_legacy_strategy_keys(tmp_path: Path) -> None:
|
||||
profile = tmp_path / "legacy_profile.yaml"
|
||||
profile.write_text(
|
||||
"""
|
||||
node_types:
|
||||
- supplier
|
||||
local_datasource:
|
||||
type: jsonl
|
||||
jsonl_dir: ${PROJECT_ROOT}/filtered_datasource/datasource/filtered
|
||||
remote_datasource:
|
||||
type: jsonl
|
||||
jsonl_dir: ${PROJECT_ROOT}/remote_datasource/datasource
|
||||
persist:
|
||||
db_path: ${PROJECT_ROOT}/runtime/test.db
|
||||
logging:
|
||||
initialize: false
|
||||
strategies:
|
||||
supplier:
|
||||
load_mode: persisted_only
|
||||
""".strip(),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="unsupported legacy keys: load_mode"):
|
||||
build_config_from_file(project_root=tmp_path, file_path=profile)
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from sync_state_machine.common.binding import BindingManager
|
||||
from sync_state_machine.common.collection import DataCollection
|
||||
from sync_state_machine.common.persistence import PersistenceBackend
|
||||
from tests.fixtures.mock_domain import build_project_node, register_test_domain
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scope_partitioning_and_copy_between_scopes(tmp_path: Path) -> None:
|
||||
register_test_domain()
|
||||
|
||||
db_path = tmp_path / "scope_partitioning.db"
|
||||
persistence = PersistenceBackend(str(db_path))
|
||||
await persistence.initialize()
|
||||
|
||||
global_collection = DataCollection("local", scope="global", persistence=persistence, auto_persist=False)
|
||||
global_node = build_project_node("node-global-1", "project-global-1", name="Global Project", code="GP-1")
|
||||
await global_collection.add(global_node)
|
||||
await global_collection.persist()
|
||||
|
||||
global_bindings = BindingManager(persistence, auto_persist=False, scope="global")
|
||||
await global_bindings.bind("test_project", global_node.node_id, "remote-node-1")
|
||||
await global_bindings.persist()
|
||||
|
||||
target_scope = "project_id:abc"
|
||||
await persistence.copy_nodes_between_scopes(
|
||||
collection_id="local",
|
||||
source_scope="global",
|
||||
target_scope=target_scope,
|
||||
node_types=["test_project"],
|
||||
)
|
||||
await persistence.copy_bindings_between_scopes(
|
||||
source_scope="global",
|
||||
target_scope=target_scope,
|
||||
node_types=["test_project"],
|
||||
)
|
||||
|
||||
scoped_collection = DataCollection("local", scope=target_scope, persistence=persistence, auto_persist=False)
|
||||
await scoped_collection.load_from_persistence()
|
||||
scoped_nodes = scoped_collection.filter(node_type="test_project")
|
||||
assert len(scoped_nodes) == 1
|
||||
assert scoped_nodes[0].data_id == "project-global-1"
|
||||
|
||||
scoped_bindings = BindingManager(persistence, auto_persist=False, scope=target_scope)
|
||||
await scoped_bindings.load_from_persistence("test_project")
|
||||
assert await scoped_bindings.get_remote_id("test_project", "node-global-1") == "remote-node-1"
|
||||
|
||||
reloaded_global = DataCollection("local", scope="global", persistence=persistence, auto_persist=False)
|
||||
await reloaded_global.load_from_persistence()
|
||||
assert len(reloaded_global.filter(node_type="test_project")) == 1
|
||||
|
||||
await persistence.close()
|
||||
@@ -0,0 +1,78 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from io import StringIO
|
||||
|
||||
import pytest
|
||||
|
||||
from sync_state_machine.pipeline.summary_report import print_pipeline_summary
|
||||
|
||||
|
||||
class _FakeBindingManager:
|
||||
async def get_all_records(self, node_type: str) -> list[dict]:
|
||||
return []
|
||||
|
||||
|
||||
class _FakeDataSource:
|
||||
def get_action_summary(self) -> dict:
|
||||
return {}
|
||||
|
||||
|
||||
class _FakeCollection:
|
||||
def filter(self, *, node_type: str) -> list:
|
||||
return []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pipeline_summary_omits_diff_schemas_column() -> None:
|
||||
stream = StringIO()
|
||||
logger = logging.getLogger("tests.summary_report")
|
||||
logger.handlers.clear()
|
||||
logger.setLevel(logging.INFO)
|
||||
logger.propagate = False
|
||||
|
||||
handler = logging.StreamHandler(stream)
|
||||
handler.setFormatter(logging.Formatter("%(message)s"))
|
||||
logger.addHandler(handler)
|
||||
|
||||
try:
|
||||
await print_pipeline_summary(
|
||||
logger=logger,
|
||||
node_types=["project"],
|
||||
binding_manager=_FakeBindingManager(),
|
||||
local_datasource=_FakeDataSource(),
|
||||
remote_datasource=_FakeDataSource(),
|
||||
local_collection=_FakeCollection(),
|
||||
remote_collection=_FakeCollection(),
|
||||
consistency_by_type={
|
||||
"project": {
|
||||
"checked_pairs": 1,
|
||||
"mismatch_count": 1,
|
||||
"fields": [
|
||||
{
|
||||
"field_name": "actual_production_date",
|
||||
"mismatch_count": 1,
|
||||
"total_count": 1,
|
||||
"schema_refs": ["ProjectResponseBase", "ProjectBaseInfoUpdate"],
|
||||
"samples": [{"local": None, "remote": "2025-11-14"}],
|
||||
}
|
||||
],
|
||||
}
|
||||
},
|
||||
stats={
|
||||
"loaded": 1,
|
||||
"synced": 0,
|
||||
"failed": 0,
|
||||
"post_check_passed": True,
|
||||
},
|
||||
)
|
||||
finally:
|
||||
logger.removeHandler(handler)
|
||||
|
||||
output = stream.getvalue()
|
||||
assert "diff_schemas" not in output
|
||||
assert "ProjectResponseBase" not in output
|
||||
assert "ProjectBaseInfoUpdate" not in output
|
||||
assert "field mismatch/total samples" in output
|
||||
assert "actual_production_date" in output
|
||||
assert "None/'2025-11-14'" in output
|
||||
@@ -77,3 +77,38 @@ async def test_jsonl_non_project_load_filters_rows_by_project_id(tmp_path):
|
||||
|
||||
loaded = datasource._collection.filter(node_type="contract") # type: ignore[union-attr]
|
||||
assert {node.data_id for node in loaded} == {"c1", "c3"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_jsonl_non_project_load_inherits_project_filter_from_loaded_projects(tmp_path):
|
||||
project_file = tmp_path / "project_1.jsonl"
|
||||
project_file.write_text(
|
||||
"\n".join([
|
||||
json.dumps({"id": "p1"}, ensure_ascii=False),
|
||||
])
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
contract_file = tmp_path / "contract_1.jsonl"
|
||||
contract_file.write_text(
|
||||
"\n".join([
|
||||
json.dumps({"id": "c1", "project_id": "p1"}, ensure_ascii=False),
|
||||
json.dumps({"id": "c2", "project_id": "p2"}, ensure_ascii=False),
|
||||
json.dumps({"id": "c3"}, ensure_ascii=False),
|
||||
])
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
datasource = JsonlDataSource(tmp_path)
|
||||
project_handler = BaseJsonlHandler(datasource, "project", _ProjectNode, _ProjectSchema)
|
||||
project_handler.set_handler_config({"data_id_filter": ["p1"]})
|
||||
contract_handler = BaseJsonlHandler(datasource, "contract", _BizNode, _BizSchema)
|
||||
datasource.register_handler(project_handler)
|
||||
datasource.register_handler(contract_handler)
|
||||
datasource.set_collection(DataCollection("local"))
|
||||
|
||||
await datasource.load_all(order=["project", "contract"])
|
||||
|
||||
loaded = datasource._collection.filter(node_type="contract") # type: ignore[union-attr]
|
||||
assert {node.data_id for node in loaded} == {"c1", "c3"}
|
||||
|
||||
@@ -2,6 +2,7 @@ import pytest
|
||||
from unittest.mock import AsyncMock, patch, MagicMock
|
||||
from sync_state_machine.sync_system.config import UpdateDirection
|
||||
from sync_state_machine.sync_system.strategy_ops.update_ops import run_update
|
||||
from sync_state_machine.domain.construction.sync_strategy import ConstructionTaskSyncStrategy
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_update_with_direction_overrides():
|
||||
@@ -65,3 +66,23 @@ async def test_run_update_with_direction_overrides():
|
||||
node_empty = FakeNode(None)
|
||||
assert push_filter(node_empty) is True
|
||||
assert pull_filter(node_empty) is False
|
||||
|
||||
|
||||
def test_construction_task_compare_payload_ignores_plan_node_is_audit():
|
||||
strategy = ConstructionTaskSyncStrategy(
|
||||
"construction_task",
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
MagicMock(),
|
||||
)
|
||||
|
||||
local_payload = {
|
||||
"plan_node": [{"name": "A", "is_audit": True}],
|
||||
"task_type": "demo",
|
||||
}
|
||||
remote_payload = {
|
||||
"plan_node": [{"name": "A", "is_audit": False}],
|
||||
"task_type": "demo",
|
||||
}
|
||||
|
||||
assert strategy.normalize_compare_payload(local_payload) == strategy.normalize_compare_payload(remote_payload)
|
||||
|
||||
Reference in New Issue
Block a user