strepsiades
417 lines
16 KiB
Python
417 lines
16 KiB
Python
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
import yaml
|
|
from pydantic import BaseModel
|
|
|
|
from sync_state_machine.config.multi_project_config import build_multi_project_config_from_file
|
|
from sync_state_machine.config import PipelineRunConfig, JsonlDataSourceConfig, PersistConfig, LoggingConfig, StrategyRuntimeConfig, StrategyConfig
|
|
from sync_state_machine.common.binding import BindingManager
|
|
from sync_state_machine.common.collection import DataCollection
|
|
from sync_state_machine.common.sqlite_backend import SQLitePersistenceBackend
|
|
from sync_state_machine.common.persistence_service import PersistenceService
|
|
from sync_state_machine.common.sync_node import SyncNode
|
|
from sync_state_machine.domain.project.sync_node import ProjectSyncNode
|
|
from sync_state_machine.pipeline.multi_project_pipeline import (
|
|
_extract_implicit_project_items,
|
|
_split_implicit_project_node_types,
|
|
is_implicit_project_batch_config,
|
|
_project_scope_overrides,
|
|
_resolve_project_bootstrap_node_types,
|
|
)
|
|
from sync_state_machine.pipeline.factory import run_pipeline_from_config
|
|
|
|
|
|
def test_build_multi_project_config_reads_explicit_project_bootstrap_node_types(tmp_path: Path) -> None:
|
|
config_path = tmp_path / "multi.yaml"
|
|
config_path.write_text(
|
|
yaml.safe_dump(
|
|
{
|
|
"global_config": "run_profiles/global.yaml",
|
|
"default_project_config": "run_profiles/project.yaml",
|
|
"project_bootstrap_node_types": ["company", "supplier"],
|
|
"projects": [
|
|
{
|
|
"local_project_id": "local-project",
|
|
"remote_project_id": "remote-project",
|
|
}
|
|
],
|
|
},
|
|
allow_unicode=True,
|
|
sort_keys=False,
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
config = build_multi_project_config_from_file(project_root=tmp_path, file_path=config_path)
|
|
|
|
assert config.project_bootstrap_node_types == ["company", "supplier"]
|
|
|
|
|
|
def test_build_multi_project_config_reads_max_concurrency(tmp_path: Path) -> None:
|
|
config_path = tmp_path / "multi.yaml"
|
|
config_path.write_text(
|
|
yaml.safe_dump(
|
|
{
|
|
"global_config": "run_profiles/global.yaml",
|
|
"default_project_config": "run_profiles/project.yaml",
|
|
"max_concurrency": 8,
|
|
"projects": [
|
|
{
|
|
"local_project_id": "local-project",
|
|
"remote_project_id": "remote-project",
|
|
}
|
|
],
|
|
},
|
|
allow_unicode=True,
|
|
sort_keys=False,
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
config = build_multi_project_config_from_file(project_root=tmp_path, file_path=config_path)
|
|
|
|
assert config.max_concurrency == 8
|
|
|
|
|
|
def test_project_scope_overrides_only_inject_project_handler_filter() -> None:
|
|
item = type("Item", (), {"local_project_id": "local-project", "remote_project_id": "remote-project"})()
|
|
|
|
overrides = _project_scope_overrides(item, node_types=["project", "contract", "user"])
|
|
|
|
assert overrides == {
|
|
"local_datasource": {
|
|
"handler_configs": {
|
|
"project": {"data_id_filter": ["local-project"]},
|
|
}
|
|
},
|
|
"remote_datasource": {
|
|
"handler_configs": {
|
|
"project": {"data_id_filter": ["remote-project"]},
|
|
}
|
|
},
|
|
}
|
|
|
|
|
|
def test_resolve_project_bootstrap_node_types_prefers_explicit_config() -> None:
|
|
config = type(
|
|
"Config",
|
|
(),
|
|
{"project_bootstrap_node_types": ["company", "supplier", "company"]},
|
|
)()
|
|
|
|
resolved = _resolve_project_bootstrap_node_types(
|
|
config=config,
|
|
shared_node_types=["company", "supplier", "project"],
|
|
)
|
|
|
|
assert resolved == ["company", "supplier"]
|
|
|
|
|
|
class _ProjectSchema(BaseModel):
|
|
id: str
|
|
|
|
|
|
class _ProjectNode(SyncNode[dict]):
|
|
node_type = "project"
|
|
schema = _ProjectSchema
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_binding_manager_prunes_stale_bindings_after_project_scope_cleanup() -> None:
|
|
persistence = SQLitePersistenceBackend(":memory:")
|
|
await persistence.initialize()
|
|
try:
|
|
local_collection = DataCollection("local", persistence=persistence, auto_persist=False)
|
|
remote_collection = DataCollection("remote", persistence=persistence, auto_persist=False)
|
|
binding_manager = BindingManager(persistence, auto_persist=False, scope="project_id:p1")
|
|
|
|
local_node = _ProjectNode(node_id="local-project-1", data={"id": "p1"}, data_id="p1")
|
|
remote_node = _ProjectNode(node_id="remote-project-1", data={"id": "p1"}, data_id="p1")
|
|
stale_local_node = _ProjectNode(node_id="local-project-2", data={"id": "p2"}, data_id="p2")
|
|
stale_remote_node = _ProjectNode(node_id="remote-project-2", data={"id": "p2"}, data_id="p2")
|
|
|
|
await local_collection.add(local_node)
|
|
await local_collection.add(stale_local_node)
|
|
await remote_collection.add(remote_node)
|
|
await remote_collection.add(stale_remote_node)
|
|
await binding_manager.bind("project", "local-project-1", "remote-project-1")
|
|
await binding_manager.bind("project", "local-project-2", "remote-project-2")
|
|
|
|
await local_collection.filter_by_project_ids(["p1"])
|
|
await remote_collection.filter_by_project_ids(["p1"])
|
|
|
|
deleted = await binding_manager.prune_missing_nodes(
|
|
local_collection=local_collection,
|
|
remote_collection=remote_collection,
|
|
node_types=["project"],
|
|
)
|
|
|
|
assert deleted == 1
|
|
assert await binding_manager.get_remote_id("project", "local-project-1") == "remote-project-1"
|
|
assert await binding_manager.get_remote_id("project", "local-project-2") is None
|
|
finally:
|
|
await persistence.close()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_collection_uses_scoped_persistence_view_and_global_fallback() -> None:
|
|
backend = SQLitePersistenceBackend(":memory:")
|
|
service = PersistenceService(backend)
|
|
await service.initialize()
|
|
try:
|
|
global_view = service.view("global")
|
|
project_view = service.view("project_id:p1")
|
|
|
|
global_collection = DataCollection("local", scope="global", persistence=global_view, auto_persist=False)
|
|
project_collection = DataCollection(
|
|
"local",
|
|
scope="project_id:p1",
|
|
persistence=project_view,
|
|
auto_persist=False,
|
|
global_fallback_collection=global_collection,
|
|
)
|
|
|
|
global_node = ProjectSyncNode(node_id="global-project-1", data={"id": "p1", "name": "Project P1"}, data_id="p1")
|
|
await global_collection.add(global_node)
|
|
await global_collection.persist()
|
|
await global_collection.unload()
|
|
await global_collection.load_from_persistence()
|
|
|
|
assert project_collection.get_by_data_id("project", "p1").node_id == "global-project-1"
|
|
|
|
project_node = ProjectSyncNode(node_id="project-local-1", data={"id": "p1", "name": "Project P1 Local"}, data_id="p1")
|
|
await project_collection.add(project_node)
|
|
|
|
assert project_collection.get_by_data_id("project", "p1").node_id == "project-local-1"
|
|
|
|
await project_collection.unload()
|
|
assert project_collection.filter() == []
|
|
assert project_collection.get_by_data_id("project", "p1").node_id == "global-project-1"
|
|
finally:
|
|
await service.close()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_binding_manager_uses_scoped_persistence_view_and_global_fallback() -> None:
|
|
backend = SQLitePersistenceBackend(":memory:")
|
|
service = PersistenceService(backend)
|
|
await service.initialize()
|
|
try:
|
|
global_view = service.view("global")
|
|
project_view = service.view("project_id:p1")
|
|
|
|
global_binding_manager = BindingManager(global_view, auto_persist=False, scope="global")
|
|
project_binding_manager = BindingManager(
|
|
project_view,
|
|
auto_persist=False,
|
|
scope="project_id:p1",
|
|
global_fallback_manager=global_binding_manager,
|
|
)
|
|
|
|
await global_binding_manager.bind("project", "local-global-1", "remote-global-1")
|
|
await global_binding_manager.persist()
|
|
await global_binding_manager.unload()
|
|
await global_binding_manager.load_from_persistence("project")
|
|
|
|
assert await project_binding_manager.get_remote_id("project", "local-global-1") == "remote-global-1"
|
|
assert await project_binding_manager.get_local_id("project", "remote-global-1") == "local-global-1"
|
|
|
|
await project_binding_manager.bind("project", "local-global-1", "remote-project-1")
|
|
assert await project_binding_manager.get_remote_id("project", "local-global-1") == "remote-project-1"
|
|
|
|
await project_binding_manager.unload()
|
|
assert await project_binding_manager.get_remote_id("project", "local-global-1") == "remote-global-1"
|
|
finally:
|
|
await service.close()
|
|
|
|
|
|
def test_regular_pipeline_config_can_be_detected_as_implicit_multi_project() -> None:
|
|
config = PipelineRunConfig(
|
|
node_types=["supplier", "company", "project", "contract"],
|
|
local_datasource=JsonlDataSourceConfig(
|
|
type="jsonl",
|
|
jsonl_dir="./local",
|
|
handler_configs={
|
|
"project": {
|
|
"data_id_filter": [
|
|
"019a62a5-2323-7be2-99c8-82534f8befa4",
|
|
"019c27e4-8b34-7779-9ddb-b8b84913e9bb",
|
|
]
|
|
}
|
|
},
|
|
),
|
|
remote_datasource=JsonlDataSourceConfig(
|
|
type="jsonl",
|
|
jsonl_dir="./remote",
|
|
handler_configs={
|
|
"project": {
|
|
"data_id_filter": [
|
|
"7119be34-57f2-4023-a251-dab30a0e6d7b",
|
|
"403a2a00-d983-4327-a52f-997bf129b833",
|
|
]
|
|
}
|
|
},
|
|
),
|
|
persist=PersistConfig(backend="sqlite", db_path=":memory:"),
|
|
logging=LoggingConfig(),
|
|
strategies=[
|
|
StrategyRuntimeConfig(
|
|
node_type="project",
|
|
config=StrategyConfig(
|
|
auto_bind=False,
|
|
pre_bind_data_id=[
|
|
{
|
|
"local_data_id": "019a62a5-2323-7be2-99c8-82534f8befa4",
|
|
"remote_data_id": "7119be34-57f2-4023-a251-dab30a0e6d7b",
|
|
},
|
|
{
|
|
"local_data_id": "019c27e4-8b34-7779-9ddb-b8b84913e9bb",
|
|
"remote_data_id": "403a2a00-d983-4327-a52f-997bf129b833",
|
|
},
|
|
],
|
|
),
|
|
)
|
|
],
|
|
)
|
|
|
|
assert is_implicit_project_batch_config(config) is True
|
|
assert _split_implicit_project_node_types(list(config.node_types)) == (
|
|
["supplier", "company"],
|
|
["project", "contract"],
|
|
)
|
|
assert _extract_implicit_project_items(config) == [
|
|
type(_extract_implicit_project_items(config)[0])(
|
|
local_project_id="019a62a5-2323-7be2-99c8-82534f8befa4",
|
|
remote_project_id="7119be34-57f2-4023-a251-dab30a0e6d7b",
|
|
),
|
|
type(_extract_implicit_project_items(config)[0])(
|
|
local_project_id="019c27e4-8b34-7779-9ddb-b8b84913e9bb",
|
|
remote_project_id="403a2a00-d983-4327-a52f-997bf129b833",
|
|
),
|
|
]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_run_pipeline_from_config_can_force_full_pipeline(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
config = PipelineRunConfig(
|
|
node_types=["project"],
|
|
local_datasource=JsonlDataSourceConfig(
|
|
type="jsonl",
|
|
jsonl_dir="./local",
|
|
handler_configs={"project": {"data_id_filter": ["p1", "p2"]}},
|
|
),
|
|
remote_datasource=JsonlDataSourceConfig(
|
|
type="jsonl",
|
|
jsonl_dir="./remote",
|
|
handler_configs={"project": {"data_id_filter": ["r1", "r2"]}},
|
|
),
|
|
persist=PersistConfig(backend="sqlite", db_path=":memory:"),
|
|
logging=LoggingConfig(),
|
|
strategies=[
|
|
StrategyRuntimeConfig(
|
|
node_type="project",
|
|
config=StrategyConfig(
|
|
auto_bind=False,
|
|
pre_bind_data_id=[
|
|
{"local_data_id": "p1", "remote_data_id": "r1"},
|
|
{"local_data_id": "p2", "remote_data_id": "r2"},
|
|
],
|
|
),
|
|
)
|
|
],
|
|
)
|
|
|
|
captured: dict[str, object] = {}
|
|
|
|
class _StubPipeline:
|
|
async def run(self):
|
|
captured["ran_full"] = True
|
|
return {"mode": "full"}
|
|
|
|
async def close(self):
|
|
captured["closed_full"] = True
|
|
|
|
async def fake_create_pipeline_from_config(*args, **kwargs):
|
|
captured["scope"] = kwargs.get("scope")
|
|
return _StubPipeline()
|
|
|
|
async def fake_run_implicit_project_batch_from_config(*args, **kwargs):
|
|
captured["ran_multi"] = True
|
|
return {"mode": "multi-project"}
|
|
|
|
monkeypatch.setattr("sync_state_machine.pipeline.factory.create_pipeline_from_config", fake_create_pipeline_from_config)
|
|
monkeypatch.setattr(
|
|
"sync_state_machine.pipeline.multi_project_pipeline.run_implicit_project_batch_from_config",
|
|
fake_run_implicit_project_batch_from_config,
|
|
)
|
|
|
|
result = await run_pipeline_from_config(config, print_summary=False, pipeline_type="full")
|
|
|
|
assert result == {"mode": "full"}
|
|
assert captured["ran_full"] is True
|
|
assert "ran_multi" not in captured
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_run_pipeline_from_config_can_require_multi_project(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
config = PipelineRunConfig(
|
|
node_types=["project"],
|
|
local_datasource=JsonlDataSourceConfig(type="jsonl", jsonl_dir="./local"),
|
|
remote_datasource=JsonlDataSourceConfig(type="jsonl", jsonl_dir="./remote"),
|
|
persist=PersistConfig(backend="sqlite", db_path=":memory:"),
|
|
logging=LoggingConfig(),
|
|
strategies=[],
|
|
)
|
|
|
|
with pytest.raises(ValueError, match="pre_bind_data_id"):
|
|
await run_pipeline_from_config(config, print_summary=False, pipeline_type="multi-project")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_run_pipeline_from_config_can_force_multi_project_for_single_project(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
config = PipelineRunConfig(
|
|
node_types=["supplier", "company", "project", "contract"],
|
|
local_datasource=JsonlDataSourceConfig(
|
|
type="jsonl",
|
|
jsonl_dir="./local",
|
|
handler_configs={"project": {"data_id_filter": ["p1"]}},
|
|
),
|
|
remote_datasource=JsonlDataSourceConfig(
|
|
type="jsonl",
|
|
jsonl_dir="./remote",
|
|
handler_configs={"project": {"data_id_filter": ["r1"]}},
|
|
),
|
|
persist=PersistConfig(backend="sqlite", db_path=":memory:"),
|
|
logging=LoggingConfig(),
|
|
strategies=[
|
|
StrategyRuntimeConfig(
|
|
node_type="project",
|
|
config=StrategyConfig(
|
|
auto_bind=False,
|
|
pre_bind_data_id=[
|
|
{"local_data_id": "p1", "remote_data_id": "r1"},
|
|
],
|
|
),
|
|
)
|
|
],
|
|
)
|
|
|
|
captured: dict[str, object] = {}
|
|
|
|
async def fake_run_implicit_project_batch_from_config(*args, **kwargs):
|
|
captured["title"] = kwargs.get("title")
|
|
captured["print_summary"] = kwargs.get("print_summary")
|
|
return {"mode": "multi-project", "projects": 1}
|
|
|
|
monkeypatch.setattr(
|
|
"sync_state_machine.pipeline.multi_project_pipeline.run_implicit_project_batch_from_config",
|
|
fake_run_implicit_project_batch_from_config,
|
|
)
|
|
|
|
result = await run_pipeline_from_config(config, print_summary=False, pipeline_type="multi-project")
|
|
|
|
assert result == {"mode": "multi-project", "projects": 1}
|
|
assert captured["print_summary"] is False |