实现了多project pipeline

This commit is contained in:
strepsiades
2026-04-07 16:52:42 +08:00
parent 4d6f499508
commit c377264610
5 changed files with 70 additions and 16 deletions
@@ -106,8 +106,8 @@ async def test_load_all_continue_when_one_handler_fails(caplog) -> None:
with caplog.at_level(logging.INFO):
await ds.load_all(order=["fail", "ok"])
assert "[fail] 加载失败" in caplog.text
assert "Loaded nodes for ok: 0" in caplog.text
assert "[fail] 加载失败: boom" in caplog.text
assert "Loaded nodes for ok: 0" not in caplog.text
@pytest.mark.asyncio
@@ -366,5 +366,52 @@ async def test_run_pipeline_from_config_can_require_multi_project(monkeypatch: p
strategies=[],
)
with pytest.raises(ValueError, match="pipeline_type=multi-project"):
await run_pipeline_from_config(config, print_summary=False, pipeline_type="multi-project")
with pytest.raises(ValueError, match="pre_bind_data_id"):
await run_pipeline_from_config(config, print_summary=False, pipeline_type="multi-project")
@pytest.mark.asyncio
async def test_run_pipeline_from_config_can_force_multi_project_for_single_project(monkeypatch: pytest.MonkeyPatch) -> None:
config = PipelineRunConfig(
node_types=["supplier", "company", "project", "contract"],
local_datasource=JsonlDataSourceConfig(
type="jsonl",
jsonl_dir="./local",
handler_configs={"project": {"data_id_filter": ["p1"]}},
),
remote_datasource=JsonlDataSourceConfig(
type="jsonl",
jsonl_dir="./remote",
handler_configs={"project": {"data_id_filter": ["r1"]}},
),
persist=PersistConfig(backend="sqlite", db_path=":memory:"),
logging=LoggingConfig(),
strategies=[
StrategyRuntimeConfig(
node_type="project",
config=StrategyConfig(
auto_bind=False,
pre_bind_data_id=[
{"local_data_id": "p1", "remote_data_id": "r1"},
],
),
)
],
)
captured: dict[str, object] = {}
async def fake_run_implicit_project_batch_from_config(*args, **kwargs):
captured["title"] = kwargs.get("title")
captured["print_summary"] = kwargs.get("print_summary")
return {"mode": "multi-project", "projects": 1}
monkeypatch.setattr(
"sync_state_machine.pipeline.multi_project_pipeline.run_implicit_project_batch_from_config",
fake_run_implicit_project_batch_from_config,
)
result = await run_pipeline_from_config(config, print_summary=False, pipeline_type="multi-project")
assert result == {"mode": "multi-project", "projects": 1}
assert captured["print_summary"] is False