from __future__ import annotations from pathlib import Path import pytest import yaml from pydantic import BaseModel from sync_state_machine.config.multi_project_config import build_multi_project_config_from_file from sync_state_machine.config import PipelineRunConfig, JsonlDataSourceConfig, PersistConfig, LoggingConfig, StrategyRuntimeConfig, StrategyConfig from sync_state_machine.common.binding import BindingManager from sync_state_machine.common.collection import DataCollection from sync_state_machine.common.sqlite_backend import SQLitePersistenceBackend from sync_state_machine.common.persistence_service import PersistenceService from sync_state_machine.common.sync_node import SyncNode from sync_state_machine.domain.project.sync_node import ProjectSyncNode from sync_state_machine.pipeline.multi_project_pipeline import ( _extract_implicit_project_items, _split_implicit_project_node_types, is_implicit_project_batch_config, _project_scope_overrides, _resolve_project_bootstrap_node_types, ) from sync_state_machine.pipeline.factory import run_pipeline_from_config def test_build_multi_project_config_reads_explicit_project_bootstrap_node_types(tmp_path: Path) -> None: config_path = tmp_path / "multi.yaml" config_path.write_text( yaml.safe_dump( { "global_config": "run_profiles/global.yaml", "default_project_config": "run_profiles/project.yaml", "project_bootstrap_node_types": ["company", "supplier"], "projects": [ { "local_project_id": "local-project", "remote_project_id": "remote-project", } ], }, allow_unicode=True, sort_keys=False, ), encoding="utf-8", ) config = build_multi_project_config_from_file(project_root=tmp_path, file_path=config_path) assert config.project_bootstrap_node_types == ["company", "supplier"] def test_build_multi_project_config_reads_max_concurrency(tmp_path: Path) -> None: config_path = tmp_path / "multi.yaml" config_path.write_text( yaml.safe_dump( { "global_config": "run_profiles/global.yaml", "default_project_config": "run_profiles/project.yaml", "max_concurrency": 8, "projects": [ { "local_project_id": "local-project", "remote_project_id": "remote-project", } ], }, allow_unicode=True, sort_keys=False, ), encoding="utf-8", ) config = build_multi_project_config_from_file(project_root=tmp_path, file_path=config_path) assert config.max_concurrency == 8 def test_project_scope_overrides_only_inject_project_handler_filter() -> None: item = type("Item", (), {"local_project_id": "local-project", "remote_project_id": "remote-project"})() overrides = _project_scope_overrides(item, node_types=["project", "contract", "user"]) assert overrides == { "local_datasource": { "handler_configs": { "project": {"data_id_filter": ["local-project"]}, } }, "remote_datasource": { "handler_configs": { "project": {"data_id_filter": ["remote-project"]}, } }, } def test_resolve_project_bootstrap_node_types_prefers_explicit_config() -> None: config = type( "Config", (), {"project_bootstrap_node_types": ["company", "supplier", "company"]}, )() resolved = _resolve_project_bootstrap_node_types( config=config, shared_node_types=["company", "supplier", "project"], ) assert resolved == ["company", "supplier"] class _ProjectSchema(BaseModel): id: str class _ProjectNode(SyncNode[dict]): node_type = "project" schema = _ProjectSchema @pytest.mark.asyncio async def test_binding_manager_prunes_stale_bindings_after_project_scope_cleanup() -> None: persistence = SQLitePersistenceBackend(":memory:") await persistence.initialize() try: local_collection = DataCollection("local", persistence=persistence, auto_persist=False) remote_collection = DataCollection("remote", persistence=persistence, auto_persist=False) binding_manager = BindingManager(persistence, auto_persist=False, scope="project_id:p1") local_node = _ProjectNode(node_id="local-project-1", data={"id": "p1"}, data_id="p1") remote_node = _ProjectNode(node_id="remote-project-1", data={"id": "p1"}, data_id="p1") stale_local_node = _ProjectNode(node_id="local-project-2", data={"id": "p2"}, data_id="p2") stale_remote_node = _ProjectNode(node_id="remote-project-2", data={"id": "p2"}, data_id="p2") await local_collection.add(local_node) await local_collection.add(stale_local_node) await remote_collection.add(remote_node) await remote_collection.add(stale_remote_node) await binding_manager.bind("project", "local-project-1", "remote-project-1") await binding_manager.bind("project", "local-project-2", "remote-project-2") await local_collection.filter_by_project_ids(["p1"]) await remote_collection.filter_by_project_ids(["p1"]) deleted = await binding_manager.prune_missing_nodes( local_collection=local_collection, remote_collection=remote_collection, node_types=["project"], ) assert deleted == 1 assert await binding_manager.get_remote_id("project", "local-project-1") == "remote-project-1" assert await binding_manager.get_remote_id("project", "local-project-2") is None finally: await persistence.close() @pytest.mark.asyncio async def test_collection_uses_scoped_persistence_view_and_global_fallback() -> None: backend = SQLitePersistenceBackend(":memory:") service = PersistenceService(backend) await service.initialize() try: global_view = service.view("global") project_view = service.view("project_id:p1") global_collection = DataCollection("local", scope="global", persistence=global_view, auto_persist=False) project_collection = DataCollection( "local", scope="project_id:p1", persistence=project_view, auto_persist=False, global_fallback_collection=global_collection, ) global_node = ProjectSyncNode(node_id="global-project-1", data={"id": "p1", "name": "Project P1"}, data_id="p1") await global_collection.add(global_node) await global_collection.persist() await global_collection.unload() await global_collection.load_from_persistence() assert project_collection.get_by_data_id("project", "p1").node_id == "global-project-1" project_node = ProjectSyncNode(node_id="project-local-1", data={"id": "p1", "name": "Project P1 Local"}, data_id="p1") await project_collection.add(project_node) assert project_collection.get_by_data_id("project", "p1").node_id == "project-local-1" await project_collection.unload() assert project_collection.filter() == [] assert project_collection.get_by_data_id("project", "p1").node_id == "global-project-1" finally: await service.close() @pytest.mark.asyncio async def test_binding_manager_uses_scoped_persistence_view_and_global_fallback() -> None: backend = SQLitePersistenceBackend(":memory:") service = PersistenceService(backend) await service.initialize() try: global_view = service.view("global") project_view = service.view("project_id:p1") global_binding_manager = BindingManager(global_view, auto_persist=False, scope="global") project_binding_manager = BindingManager( project_view, auto_persist=False, scope="project_id:p1", global_fallback_manager=global_binding_manager, ) await global_binding_manager.bind("project", "local-global-1", "remote-global-1") await global_binding_manager.persist() await global_binding_manager.unload() await global_binding_manager.load_from_persistence("project") assert await project_binding_manager.get_remote_id("project", "local-global-1") == "remote-global-1" assert await project_binding_manager.get_local_id("project", "remote-global-1") == "local-global-1" await project_binding_manager.bind("project", "local-global-1", "remote-project-1") assert await project_binding_manager.get_remote_id("project", "local-global-1") == "remote-project-1" await project_binding_manager.unload() assert await project_binding_manager.get_remote_id("project", "local-global-1") == "remote-global-1" finally: await service.close() def test_regular_pipeline_config_can_be_detected_as_implicit_multi_project() -> None: config = PipelineRunConfig( node_types=["supplier", "company", "project", "contract"], local_datasource=JsonlDataSourceConfig( type="jsonl", jsonl_dir="./local", handler_configs={ "project": { "data_id_filter": [ "019a62a5-2323-7be2-99c8-82534f8befa4", "019c27e4-8b34-7779-9ddb-b8b84913e9bb", ] } }, ), remote_datasource=JsonlDataSourceConfig( type="jsonl", jsonl_dir="./remote", handler_configs={ "project": { "data_id_filter": [ "7119be34-57f2-4023-a251-dab30a0e6d7b", "403a2a00-d983-4327-a52f-997bf129b833", ] } }, ), persist=PersistConfig(backend="sqlite", db_path=":memory:"), logging=LoggingConfig(), strategies=[ StrategyRuntimeConfig( node_type="project", config=StrategyConfig( auto_bind=False, pre_bind_data_id=[ { "local_data_id": "019a62a5-2323-7be2-99c8-82534f8befa4", "remote_data_id": "7119be34-57f2-4023-a251-dab30a0e6d7b", }, { "local_data_id": "019c27e4-8b34-7779-9ddb-b8b84913e9bb", "remote_data_id": "403a2a00-d983-4327-a52f-997bf129b833", }, ], ), ) ], ) assert is_implicit_project_batch_config(config) is True assert _split_implicit_project_node_types(list(config.node_types)) == ( ["supplier", "company"], ["project", "contract"], ) assert _extract_implicit_project_items(config) == [ type(_extract_implicit_project_items(config)[0])( local_project_id="019a62a5-2323-7be2-99c8-82534f8befa4", remote_project_id="7119be34-57f2-4023-a251-dab30a0e6d7b", ), type(_extract_implicit_project_items(config)[0])( local_project_id="019c27e4-8b34-7779-9ddb-b8b84913e9bb", remote_project_id="403a2a00-d983-4327-a52f-997bf129b833", ), ] @pytest.mark.asyncio async def test_run_pipeline_from_config_can_force_full_pipeline(monkeypatch: pytest.MonkeyPatch) -> None: config = PipelineRunConfig( node_types=["project"], local_datasource=JsonlDataSourceConfig( type="jsonl", jsonl_dir="./local", handler_configs={"project": {"data_id_filter": ["p1", "p2"]}}, ), remote_datasource=JsonlDataSourceConfig( type="jsonl", jsonl_dir="./remote", handler_configs={"project": {"data_id_filter": ["r1", "r2"]}}, ), persist=PersistConfig(backend="sqlite", db_path=":memory:"), logging=LoggingConfig(), strategies=[ StrategyRuntimeConfig( node_type="project", config=StrategyConfig( auto_bind=False, pre_bind_data_id=[ {"local_data_id": "p1", "remote_data_id": "r1"}, {"local_data_id": "p2", "remote_data_id": "r2"}, ], ), ) ], ) captured: dict[str, object] = {} class _StubPipeline: async def run(self): captured["ran_full"] = True return {"mode": "full"} async def close(self): captured["closed_full"] = True async def fake_create_pipeline_from_config(*args, **kwargs): captured["scope"] = kwargs.get("scope") return _StubPipeline() async def fake_run_implicit_project_batch_from_config(*args, **kwargs): captured["ran_multi"] = True return {"mode": "multi-project"} monkeypatch.setattr("sync_state_machine.pipeline.factory.create_pipeline_from_config", fake_create_pipeline_from_config) monkeypatch.setattr( "sync_state_machine.pipeline.multi_project_pipeline.run_implicit_project_batch_from_config", fake_run_implicit_project_batch_from_config, ) result = await run_pipeline_from_config(config, print_summary=False, pipeline_type="full") assert result == {"mode": "full"} assert captured["ran_full"] is True assert "ran_multi" not in captured @pytest.mark.asyncio async def test_run_pipeline_from_config_can_require_multi_project(monkeypatch: pytest.MonkeyPatch) -> None: config = PipelineRunConfig( node_types=["project"], local_datasource=JsonlDataSourceConfig(type="jsonl", jsonl_dir="./local"), remote_datasource=JsonlDataSourceConfig(type="jsonl", jsonl_dir="./remote"), persist=PersistConfig(backend="sqlite", db_path=":memory:"), logging=LoggingConfig(), strategies=[], ) with pytest.raises(ValueError, match="pre_bind_data_id"): await run_pipeline_from_config(config, print_summary=False, pipeline_type="multi-project") @pytest.mark.asyncio async def test_run_pipeline_from_config_can_force_multi_project_for_single_project(monkeypatch: pytest.MonkeyPatch) -> None: config = PipelineRunConfig( node_types=["supplier", "company", "project", "contract"], local_datasource=JsonlDataSourceConfig( type="jsonl", jsonl_dir="./local", handler_configs={"project": {"data_id_filter": ["p1"]}}, ), remote_datasource=JsonlDataSourceConfig( type="jsonl", jsonl_dir="./remote", handler_configs={"project": {"data_id_filter": ["r1"]}}, ), persist=PersistConfig(backend="sqlite", db_path=":memory:"), logging=LoggingConfig(), strategies=[ StrategyRuntimeConfig( node_type="project", config=StrategyConfig( auto_bind=False, pre_bind_data_id=[ {"local_data_id": "p1", "remote_data_id": "r1"}, ], ), ) ], ) captured: dict[str, object] = {} async def fake_run_implicit_project_batch_from_config(*args, **kwargs): captured["title"] = kwargs.get("title") captured["print_summary"] = kwargs.get("print_summary") return {"mode": "multi-project", "projects": 1} monkeypatch.setattr( "sync_state_machine.pipeline.multi_project_pipeline.run_implicit_project_batch_from_config", fake_run_implicit_project_batch_from_config, ) result = await run_pipeline_from_config(config, print_summary=False, pipeline_type="multi-project") assert result == {"mode": "multi-project", "projects": 1} assert captured["print_summary"] is False