实现了多project pipeline

This commit is contained in:
strepsiades
2026-04-07 16:52:42 +08:00
parent 4d6f499508
commit c377264610
5 changed files with 70 additions and 16 deletions
+6 -6
View File
@@ -482,7 +482,7 @@ class BaseDataSource(ABC):
# Handler 加载数据并创建节点
nodes = await handler.load()
logger.info("%s Loaded nodes for %s: %d", self._datasource_log_prefix(), node_type, len(nodes))
logger.debug("%s Loaded nodes for %s: %d", self._datasource_log_prefix(), node_type, len(nodes))
# 将节点写入 Collection:优先按 data_id 命中已有节点并更新
await self._upsert_loaded_nodes(handler, node_type, nodes)
@@ -641,7 +641,7 @@ class BaseDataSource(ABC):
# 批量创建(传入节点列表)
if create_nodes:
logger.info(f"{self._node_log_prefix(handler.node_type)} create提交开始: nodes={len(create_nodes)}")
logger.debug(f"{self._node_log_prefix(handler.node_type)} create提交开始: nodes={len(create_nodes)}")
try:
create_results = await handler.create_all(create_nodes)
create_success = 0
@@ -658,7 +658,7 @@ class BaseDataSource(ABC):
elif result.status == _TaskStatus.SKIPPED:
create_skipped += 1
await self._handle_task_result(handler, result, async_tasks)
logger.info(
logger.debug(
f"{self._node_log_prefix(handler.node_type)} create提交完成: success={create_success}, "
f"in_progress={create_in_progress}, failed={create_failed}, skipped={create_skipped}"
)
@@ -694,7 +694,7 @@ class BaseDataSource(ABC):
# 轮询异步任务
if async_tasks and poll_async_tasks:
logger.info(
logger.debug(
f"{self._node_log_prefix(handler.node_type)} create已提交,push_id检查即将开始: pending={len(async_tasks)}, "
f"first_wait={self._poll_interval:.3f}s"
)
@@ -845,13 +845,13 @@ class BaseDataSource(ABC):
return
if async_tasks and poll_interval > 0:
logger.info(
logger.debug(
f"{self._node_log_prefix(handler.node_type)} push_id首次检查前等待 {poll_interval:.3f}s, pending={len(async_tasks)}"
)
await asyncio.sleep(poll_interval)
while async_tasks and retry_count < max_retries:
logger.info(
logger.debug(
f"{self._node_log_prefix(handler.node_type)} push_id检查 {retry_count + 1}/{max_retries}, pending={len(async_tasks)}"
)
# 批量查询
+9 -2
View File
@@ -500,11 +500,18 @@ async def run_pipeline_from_config(
from .multi_project_pipeline import is_implicit_project_batch_config, run_implicit_project_batch_from_config
implicit_multi_project = is_implicit_project_batch_config(config)
if pipeline_type == "multi-project" and not implicit_multi_project:
raise ValueError("pipeline_type=multi-project requires an implicit multi-project pipeline config")
if pipeline_type == "full" and implicit_multi_project:
logger.info("pipeline_type=full specified, skipping implicit multi-project routing")
if pipeline_type == "multi-project":
_log_resolved_config(config)
return await run_implicit_project_batch_from_config(
config,
title=title,
print_summary=print_summary,
max_concurrency=getattr(config, "max_concurrency", None),
)
if pipeline_type != "full" and implicit_multi_project:
_log_resolved_config(config)
return await run_implicit_project_batch_from_config(
@@ -511,7 +511,7 @@ class FullSyncPipeline:
)
async def _load_node_type(self, node_type: str, *, reason: str) -> None:
self._logger.info(f"🔄 Load node_type={node_type}, reason={reason}")
self._logger.debug(f"🔄 Load node_type={node_type}, reason={reason}")
await self._load_node_type_from_datasource(
self.local_datasource,
datasource_role="local",
@@ -576,7 +576,7 @@ class FullSyncPipeline:
self._log_reload_skipped(node_type, reason)
return
self._logger.info(
self._logger.debug(
"🔄 Reload node_type=%s after write, reason=%s, local=%s, remote=%s",
node_type,
reason,
@@ -667,7 +667,7 @@ class FullSyncPipeline:
)
if finalized_success or finalized_failed:
self._logger.info(
self._logger.debug(
f"🔁 Peer-creating finalize: node_type={node_type}, success={finalized_success}, failed={finalized_failed}"
)
@@ -690,7 +690,7 @@ class FullSyncPipeline:
normalized += 1
if normalized > 0:
self._logger.info(f"🔁 Post-create normalization: {normalized} nodes moved S11C->S01 for {node_type}")
self._logger.debug(f"🔁 Post-create normalization: {normalized} nodes moved S11C->S01 for {node_type}")
async def commit_updates(self, node_type: str) -> WriteOutcome:
"""提交 UPDATE 操作(包含异步轮询)"""