实现了多project pipeline

This commit is contained in:
strepsiades
2026-04-07 16:52:42 +08:00
parent 4d6f499508
commit c377264610
5 changed files with 70 additions and 16 deletions
+6 -6
View File
@@ -482,7 +482,7 @@ class BaseDataSource(ABC):
# Handler 加载数据并创建节点
nodes = await handler.load()
logger.info("%s Loaded nodes for %s: %d", self._datasource_log_prefix(), node_type, len(nodes))
logger.debug("%s Loaded nodes for %s: %d", self._datasource_log_prefix(), node_type, len(nodes))
# 将节点写入 Collection:优先按 data_id 命中已有节点并更新
await self._upsert_loaded_nodes(handler, node_type, nodes)
@@ -641,7 +641,7 @@ class BaseDataSource(ABC):
# 批量创建(传入节点列表)
if create_nodes:
logger.info(f"{self._node_log_prefix(handler.node_type)} create提交开始: nodes={len(create_nodes)}")
logger.debug(f"{self._node_log_prefix(handler.node_type)} create提交开始: nodes={len(create_nodes)}")
try:
create_results = await handler.create_all(create_nodes)
create_success = 0
@@ -658,7 +658,7 @@ class BaseDataSource(ABC):
elif result.status == _TaskStatus.SKIPPED:
create_skipped += 1
await self._handle_task_result(handler, result, async_tasks)
logger.info(
logger.debug(
f"{self._node_log_prefix(handler.node_type)} create提交完成: success={create_success}, "
f"in_progress={create_in_progress}, failed={create_failed}, skipped={create_skipped}"
)
@@ -694,7 +694,7 @@ class BaseDataSource(ABC):
# 轮询异步任务
if async_tasks and poll_async_tasks:
logger.info(
logger.debug(
f"{self._node_log_prefix(handler.node_type)} create已提交,push_id检查即将开始: pending={len(async_tasks)}, "
f"first_wait={self._poll_interval:.3f}s"
)
@@ -845,13 +845,13 @@ class BaseDataSource(ABC):
return
if async_tasks and poll_interval > 0:
logger.info(
logger.debug(
f"{self._node_log_prefix(handler.node_type)} push_id首次检查前等待 {poll_interval:.3f}s, pending={len(async_tasks)}"
)
await asyncio.sleep(poll_interval)
while async_tasks and retry_count < max_retries:
logger.info(
logger.debug(
f"{self._node_log_prefix(handler.node_type)} push_id检查 {retry_count + 1}/{max_retries}, pending={len(async_tasks)}"
)
# 批量查询
+9 -2
View File
@@ -500,11 +500,18 @@ async def run_pipeline_from_config(
from .multi_project_pipeline import is_implicit_project_batch_config, run_implicit_project_batch_from_config
implicit_multi_project = is_implicit_project_batch_config(config)
if pipeline_type == "multi-project" and not implicit_multi_project:
raise ValueError("pipeline_type=multi-project requires an implicit multi-project pipeline config")
if pipeline_type == "full" and implicit_multi_project:
logger.info("pipeline_type=full specified, skipping implicit multi-project routing")
if pipeline_type == "multi-project":
_log_resolved_config(config)
return await run_implicit_project_batch_from_config(
config,
title=title,
print_summary=print_summary,
max_concurrency=getattr(config, "max_concurrency", None),
)
if pipeline_type != "full" and implicit_multi_project:
_log_resolved_config(config)
return await run_implicit_project_batch_from_config(
@@ -511,7 +511,7 @@ class FullSyncPipeline:
)
async def _load_node_type(self, node_type: str, *, reason: str) -> None:
self._logger.info(f"🔄 Load node_type={node_type}, reason={reason}")
self._logger.debug(f"🔄 Load node_type={node_type}, reason={reason}")
await self._load_node_type_from_datasource(
self.local_datasource,
datasource_role="local",
@@ -576,7 +576,7 @@ class FullSyncPipeline:
self._log_reload_skipped(node_type, reason)
return
self._logger.info(
self._logger.debug(
"🔄 Reload node_type=%s after write, reason=%s, local=%s, remote=%s",
node_type,
reason,
@@ -667,7 +667,7 @@ class FullSyncPipeline:
)
if finalized_success or finalized_failed:
self._logger.info(
self._logger.debug(
f"🔁 Peer-creating finalize: node_type={node_type}, success={finalized_success}, failed={finalized_failed}"
)
@@ -690,7 +690,7 @@ class FullSyncPipeline:
normalized += 1
if normalized > 0:
self._logger.info(f"🔁 Post-create normalization: {normalized} nodes moved S11C->S01 for {node_type}")
self._logger.debug(f"🔁 Post-create normalization: {normalized} nodes moved S11C->S01 for {node_type}")
async def commit_updates(self, node_type: str) -> WriteOutcome:
"""提交 UPDATE 操作(包含异步轮询)"""
@@ -106,8 +106,8 @@ async def test_load_all_continue_when_one_handler_fails(caplog) -> None:
with caplog.at_level(logging.INFO):
await ds.load_all(order=["fail", "ok"])
assert "[fail] 加载失败" in caplog.text
assert "Loaded nodes for ok: 0" in caplog.text
assert "[fail] 加载失败: boom" in caplog.text
assert "Loaded nodes for ok: 0" not in caplog.text
@pytest.mark.asyncio
@@ -366,5 +366,52 @@ async def test_run_pipeline_from_config_can_require_multi_project(monkeypatch: p
strategies=[],
)
with pytest.raises(ValueError, match="pipeline_type=multi-project"):
await run_pipeline_from_config(config, print_summary=False, pipeline_type="multi-project")
with pytest.raises(ValueError, match="pre_bind_data_id"):
await run_pipeline_from_config(config, print_summary=False, pipeline_type="multi-project")
@pytest.mark.asyncio
async def test_run_pipeline_from_config_can_force_multi_project_for_single_project(monkeypatch: pytest.MonkeyPatch) -> None:
config = PipelineRunConfig(
node_types=["supplier", "company", "project", "contract"],
local_datasource=JsonlDataSourceConfig(
type="jsonl",
jsonl_dir="./local",
handler_configs={"project": {"data_id_filter": ["p1"]}},
),
remote_datasource=JsonlDataSourceConfig(
type="jsonl",
jsonl_dir="./remote",
handler_configs={"project": {"data_id_filter": ["r1"]}},
),
persist=PersistConfig(backend="sqlite", db_path=":memory:"),
logging=LoggingConfig(),
strategies=[
StrategyRuntimeConfig(
node_type="project",
config=StrategyConfig(
auto_bind=False,
pre_bind_data_id=[
{"local_data_id": "p1", "remote_data_id": "r1"},
],
),
)
],
)
captured: dict[str, object] = {}
async def fake_run_implicit_project_batch_from_config(*args, **kwargs):
captured["title"] = kwargs.get("title")
captured["print_summary"] = kwargs.get("print_summary")
return {"mode": "multi-project", "projects": 1}
monkeypatch.setattr(
"sync_state_machine.pipeline.multi_project_pipeline.run_implicit_project_batch_from_config",
fake_run_implicit_project_batch_from_config,
)
result = await run_pipeline_from_config(config, print_summary=False, pipeline_type="multi-project")
assert result == {"mode": "multi-project", "projects": 1}
assert captured["print_summary"] is False