strepsiades/ecm_sync_system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

初步实现多project pipeline

Browse Source
This commit is contained in:
strepsiades
2026-04-07 14:45:19 +08:00
parent e3eab5ade1
commit 966f2e1270
11 changed files with 1083 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/test_multi_project_pipeline_config.py
+248 -1
View File
@@ -7,14 +7,21 @@ import yaml
from pydantic import BaseModel
from sync_state_machine.config.multi_project_config import build_multi_project_config_from_file
from sync_state_machine.config import PipelineRunConfig, JsonlDataSourceConfig, PersistConfig, LoggingConfig, StrategyRuntimeConfig, StrategyConfig
from sync_state_machine.common.binding import BindingManager
from sync_state_machine.common.collection import DataCollection
from sync_state_machine.common.sqlite_backend import SQLitePersistenceBackend
from sync_state_machine.common.persistence_service import PersistenceService
from sync_state_machine.common.sync_node import SyncNode
from sync_state_machine.domain.project.sync_node import ProjectSyncNode
from sync_state_machine.pipeline.multi_project_pipeline import (
_extract_implicit_project_items,
_split_implicit_project_node_types,
is_implicit_project_batch_config,
_project_scope_overrides,
_resolve_project_bootstrap_node_types,
)
from sync_state_machine.pipeline.factory import run_pipeline_from_config
def test_build_multi_project_config_reads_explicit_project_bootstrap_node_types(tmp_path: Path) -> None:
@@ -43,6 +50,32 @@ def test_build_multi_project_config_reads_explicit_project_bootstrap_node_types(
assert config.project_bootstrap_node_types == ["company", "supplier"]
def test_build_multi_project_config_reads_max_concurrency(tmp_path: Path) -> None:
config_path = tmp_path / "multi.yaml"
config_path.write_text(
yaml.safe_dump(
{
"global_config": "run_profiles/global.yaml",
"default_project_config": "run_profiles/project.yaml",
"max_concurrency": 8,
"projects": [
{
"local_project_id": "local-project",
"remote_project_id": "remote-project",
}
],
},
allow_unicode=True,
sort_keys=False,
),
encoding="utf-8",
)
config = build_multi_project_config_from_file(project_root=tmp_path, file_path=config_path)
assert config.max_concurrency == 8
def test_project_scope_overrides_only_inject_project_handler_filter() -> None:
item = type("Item", (), {"local_project_id": "local-project", "remote_project_id": "remote-project"})()
@@ -120,4 +153,218 @@ async def test_binding_manager_prunes_stale_bindings_after_project_scope_cleanup
assert await binding_manager.get_remote_id("project", "local-project-1") == "remote-project-1"
assert await binding_manager.get_remote_id("project", "local-project-2") is None
finally:
await persistence.close()
await persistence.close()
@pytest.mark.asyncio
async def test_collection_uses_scoped_persistence_view_and_global_fallback() -> None:
backend = SQLitePersistenceBackend(":memory:")
service = PersistenceService(backend)
await service.initialize()
try:
global_view = service.view("global")
project_view = service.view("project_id:p1")
global_collection = DataCollection("local", scope="global", persistence=global_view, auto_persist=False)
project_collection = DataCollection(
"local",
scope="project_id:p1",
persistence=project_view,
auto_persist=False,
global_fallback_collection=global_collection,
)
global_node = ProjectSyncNode(node_id="global-project-1", data={"id": "p1", "name": "Project P1"}, data_id="p1")
await global_collection.add(global_node)
await global_collection.persist()
await global_collection.unload()
await global_collection.load_from_persistence()
assert project_collection.get_by_data_id("project", "p1").node_id == "global-project-1"
project_node = ProjectSyncNode(node_id="project-local-1", data={"id": "p1", "name": "Project P1 Local"}, data_id="p1")
await project_collection.add(project_node)
assert project_collection.get_by_data_id("project", "p1").node_id == "project-local-1"
await project_collection.unload()
assert project_collection.filter() == []
assert project_collection.get_by_data_id("project", "p1").node_id == "global-project-1"
finally:
await service.close()
@pytest.mark.asyncio
async def test_binding_manager_uses_scoped_persistence_view_and_global_fallback() -> None:
backend = SQLitePersistenceBackend(":memory:")
service = PersistenceService(backend)
await service.initialize()
try:
global_view = service.view("global")
project_view = service.view("project_id:p1")
global_binding_manager = BindingManager(global_view, auto_persist=False, scope="global")
project_binding_manager = BindingManager(
project_view,
auto_persist=False,
scope="project_id:p1",
global_fallback_manager=global_binding_manager,
)
await global_binding_manager.bind("project", "local-global-1", "remote-global-1")
await global_binding_manager.persist()
await global_binding_manager.unload()
await global_binding_manager.load_from_persistence("project")
assert await project_binding_manager.get_remote_id("project", "local-global-1") == "remote-global-1"
assert await project_binding_manager.get_local_id("project", "remote-global-1") == "local-global-1"
await project_binding_manager.bind("project", "local-global-1", "remote-project-1")
assert await project_binding_manager.get_remote_id("project", "local-global-1") == "remote-project-1"
await project_binding_manager.unload()
assert await project_binding_manager.get_remote_id("project", "local-global-1") == "remote-global-1"
finally:
await service.close()
def test_regular_pipeline_config_can_be_detected_as_implicit_multi_project() -> None:
config = PipelineRunConfig(
node_types=["supplier", "company", "project", "contract"],
local_datasource=JsonlDataSourceConfig(
type="jsonl",
jsonl_dir="./local",
handler_configs={
"project": {
"data_id_filter": [
"019a62a5-2323-7be2-99c8-82534f8befa4",
"019c27e4-8b34-7779-9ddb-b8b84913e9bb",
]
}
},
),
remote_datasource=JsonlDataSourceConfig(
type="jsonl",
jsonl_dir="./remote",
handler_configs={
"project": {
"data_id_filter": [
"7119be34-57f2-4023-a251-dab30a0e6d7b",
"403a2a00-d983-4327-a52f-997bf129b833",
]
}
},
),
persist=PersistConfig(backend="sqlite", db_path=":memory:"),
logging=LoggingConfig(),
strategies=[
StrategyRuntimeConfig(
node_type="project",
config=StrategyConfig(
auto_bind=False,
pre_bind_data_id=[
{
"local_data_id": "019a62a5-2323-7be2-99c8-82534f8befa4",
"remote_data_id": "7119be34-57f2-4023-a251-dab30a0e6d7b",
},
{
"local_data_id": "019c27e4-8b34-7779-9ddb-b8b84913e9bb",
"remote_data_id": "403a2a00-d983-4327-a52f-997bf129b833",
},
],
),
)
],
)
assert is_implicit_project_batch_config(config) is True
assert _split_implicit_project_node_types(list(config.node_types)) == (
["supplier", "company"],
["project", "contract"],
)
assert _extract_implicit_project_items(config) == [
type(_extract_implicit_project_items(config)[0])(
local_project_id="019a62a5-2323-7be2-99c8-82534f8befa4",
remote_project_id="7119be34-57f2-4023-a251-dab30a0e6d7b",
),
type(_extract_implicit_project_items(config)[0])(
local_project_id="019c27e4-8b34-7779-9ddb-b8b84913e9bb",
remote_project_id="403a2a00-d983-4327-a52f-997bf129b833",
),
]
@pytest.mark.asyncio
async def test_run_pipeline_from_config_can_force_full_pipeline(monkeypatch: pytest.MonkeyPatch) -> None:
config = PipelineRunConfig(
node_types=["project"],
local_datasource=JsonlDataSourceConfig(
type="jsonl",
jsonl_dir="./local",
handler_configs={"project": {"data_id_filter": ["p1", "p2"]}},
),
remote_datasource=JsonlDataSourceConfig(
type="jsonl",
jsonl_dir="./remote",
handler_configs={"project": {"data_id_filter": ["r1", "r2"]}},
),
persist=PersistConfig(backend="sqlite", db_path=":memory:"),
logging=LoggingConfig(),
strategies=[
StrategyRuntimeConfig(
node_type="project",
config=StrategyConfig(
auto_bind=False,
pre_bind_data_id=[
{"local_data_id": "p1", "remote_data_id": "r1"},
{"local_data_id": "p2", "remote_data_id": "r2"},
],
),
)
],
)
captured: dict[str, object] = {}
class _StubPipeline:
async def run(self):
captured["ran_full"] = True
return {"mode": "full"}
async def close(self):
captured["closed_full"] = True
async def fake_create_pipeline_from_config(*args, **kwargs):
captured["scope"] = kwargs.get("scope")
return _StubPipeline()
async def fake_run_implicit_project_batch_from_config(*args, **kwargs):
captured["ran_multi"] = True
return {"mode": "multi-project"}
monkeypatch.setattr("sync_state_machine.pipeline.factory.create_pipeline_from_config", fake_create_pipeline_from_config)
monkeypatch.setattr(
"sync_state_machine.pipeline.multi_project_pipeline.run_implicit_project_batch_from_config",
fake_run_implicit_project_batch_from_config,
)
result = await run_pipeline_from_config(config, print_summary=False, pipeline_type="full")
assert result == {"mode": "full"}
assert captured["ran_full"] is True
assert "ran_multi" not in captured
@pytest.mark.asyncio
async def test_run_pipeline_from_config_can_require_multi_project(monkeypatch: pytest.MonkeyPatch) -> None:
config = PipelineRunConfig(
node_types=["project"],
local_datasource=JsonlDataSourceConfig(type="jsonl", jsonl_dir="./local"),
remote_datasource=JsonlDataSourceConfig(type="jsonl", jsonl_dir="./remote"),
persist=PersistConfig(backend="sqlite", db_path=":memory:"),
logging=LoggingConfig(),
strategies=[],
)
with pytest.raises(ValueError, match="pipeline_type=multi-project"):
await run_pipeline_from_config(config, print_summary=False, pipeline_type="multi-project")
tests/unit/test_summary_report.py
+22 -1
View File
@@ -6,6 +6,7 @@ from io import StringIO
import pytest
from sync_state_machine.pipeline.summary_report import print_pipeline_summary
from sync_state_machine.logging import get_scope_logger
class _FakeBindingManager:
@@ -75,4 +76,24 @@ async def test_pipeline_summary_omits_diff_schemas_column() -> None:
assert "ProjectBaseInfoUpdate" not in output
assert "field mismatch/total samples" in output
assert "actual_production_date" in output
assert "None/'2025-11-14'" in output
assert "None/'2025-11-14'" in output
def test_scope_logger_prefixes_messages() -> None:
stream = StringIO()
base_logger = logging.getLogger("sync_state_machine.tests.scope_logger")
base_logger.handlers.clear()
base_logger.setLevel(logging.INFO)
base_logger.propagate = False
handler = logging.StreamHandler(stream)
handler.setFormatter(logging.Formatter("%(message)s"))
base_logger.addHandler(handler)
try:
scoped_logger = get_scope_logger("project_id:p1", "tests.scope_logger")
scoped_logger.info("Strategy start: project")
finally:
base_logger.removeHandler(handler)
assert stream.getvalue().strip() == "[project_id:p1] Strategy start: project"