实现了多project pipeline
This commit is contained in:
@@ -482,7 +482,7 @@ class BaseDataSource(ABC):
|
||||
# Handler 加载数据并创建节点
|
||||
nodes = await handler.load()
|
||||
|
||||
logger.info("%s Loaded nodes for %s: %d", self._datasource_log_prefix(), node_type, len(nodes))
|
||||
logger.debug("%s Loaded nodes for %s: %d", self._datasource_log_prefix(), node_type, len(nodes))
|
||||
|
||||
# 将节点写入 Collection:优先按 data_id 命中已有节点并更新
|
||||
await self._upsert_loaded_nodes(handler, node_type, nodes)
|
||||
@@ -641,7 +641,7 @@ class BaseDataSource(ABC):
|
||||
|
||||
# 批量创建(传入节点列表)
|
||||
if create_nodes:
|
||||
logger.info(f"{self._node_log_prefix(handler.node_type)} create提交开始: nodes={len(create_nodes)}")
|
||||
logger.debug(f"{self._node_log_prefix(handler.node_type)} create提交开始: nodes={len(create_nodes)}")
|
||||
try:
|
||||
create_results = await handler.create_all(create_nodes)
|
||||
create_success = 0
|
||||
@@ -658,7 +658,7 @@ class BaseDataSource(ABC):
|
||||
elif result.status == _TaskStatus.SKIPPED:
|
||||
create_skipped += 1
|
||||
await self._handle_task_result(handler, result, async_tasks)
|
||||
logger.info(
|
||||
logger.debug(
|
||||
f"{self._node_log_prefix(handler.node_type)} create提交完成: success={create_success}, "
|
||||
f"in_progress={create_in_progress}, failed={create_failed}, skipped={create_skipped}"
|
||||
)
|
||||
@@ -694,7 +694,7 @@ class BaseDataSource(ABC):
|
||||
|
||||
# 轮询异步任务
|
||||
if async_tasks and poll_async_tasks:
|
||||
logger.info(
|
||||
logger.debug(
|
||||
f"{self._node_log_prefix(handler.node_type)} create已提交,push_id检查即将开始: pending={len(async_tasks)}, "
|
||||
f"first_wait={self._poll_interval:.3f}s"
|
||||
)
|
||||
@@ -845,13 +845,13 @@ class BaseDataSource(ABC):
|
||||
return
|
||||
|
||||
if async_tasks and poll_interval > 0:
|
||||
logger.info(
|
||||
logger.debug(
|
||||
f"{self._node_log_prefix(handler.node_type)} push_id首次检查前等待 {poll_interval:.3f}s, pending={len(async_tasks)}"
|
||||
)
|
||||
await asyncio.sleep(poll_interval)
|
||||
|
||||
while async_tasks and retry_count < max_retries:
|
||||
logger.info(
|
||||
logger.debug(
|
||||
f"{self._node_log_prefix(handler.node_type)} push_id检查 {retry_count + 1}/{max_retries}, pending={len(async_tasks)}"
|
||||
)
|
||||
# 批量查询
|
||||
|
||||
@@ -500,11 +500,18 @@ async def run_pipeline_from_config(
|
||||
from .multi_project_pipeline import is_implicit_project_batch_config, run_implicit_project_batch_from_config
|
||||
|
||||
implicit_multi_project = is_implicit_project_batch_config(config)
|
||||
if pipeline_type == "multi-project" and not implicit_multi_project:
|
||||
raise ValueError("pipeline_type=multi-project requires an implicit multi-project pipeline config")
|
||||
if pipeline_type == "full" and implicit_multi_project:
|
||||
logger.info("pipeline_type=full specified, skipping implicit multi-project routing")
|
||||
|
||||
if pipeline_type == "multi-project":
|
||||
_log_resolved_config(config)
|
||||
return await run_implicit_project_batch_from_config(
|
||||
config,
|
||||
title=title,
|
||||
print_summary=print_summary,
|
||||
max_concurrency=getattr(config, "max_concurrency", None),
|
||||
)
|
||||
|
||||
if pipeline_type != "full" and implicit_multi_project:
|
||||
_log_resolved_config(config)
|
||||
return await run_implicit_project_batch_from_config(
|
||||
|
||||
@@ -511,7 +511,7 @@ class FullSyncPipeline:
|
||||
)
|
||||
|
||||
async def _load_node_type(self, node_type: str, *, reason: str) -> None:
|
||||
self._logger.info(f"🔄 Load node_type={node_type}, reason={reason}")
|
||||
self._logger.debug(f"🔄 Load node_type={node_type}, reason={reason}")
|
||||
await self._load_node_type_from_datasource(
|
||||
self.local_datasource,
|
||||
datasource_role="local",
|
||||
@@ -576,7 +576,7 @@ class FullSyncPipeline:
|
||||
self._log_reload_skipped(node_type, reason)
|
||||
return
|
||||
|
||||
self._logger.info(
|
||||
self._logger.debug(
|
||||
"🔄 Reload node_type=%s after write, reason=%s, local=%s, remote=%s",
|
||||
node_type,
|
||||
reason,
|
||||
@@ -667,7 +667,7 @@ class FullSyncPipeline:
|
||||
)
|
||||
|
||||
if finalized_success or finalized_failed:
|
||||
self._logger.info(
|
||||
self._logger.debug(
|
||||
f"🔁 Peer-creating finalize: node_type={node_type}, success={finalized_success}, failed={finalized_failed}"
|
||||
)
|
||||
|
||||
@@ -690,7 +690,7 @@ class FullSyncPipeline:
|
||||
normalized += 1
|
||||
|
||||
if normalized > 0:
|
||||
self._logger.info(f"🔁 Post-create normalization: {normalized} nodes moved S11C->S01 for {node_type}")
|
||||
self._logger.debug(f"🔁 Post-create normalization: {normalized} nodes moved S11C->S01 for {node_type}")
|
||||
|
||||
async def commit_updates(self, node_type: str) -> WriteOutcome:
|
||||
"""提交 UPDATE 操作(包含异步轮询)"""
|
||||
|
||||
Reference in New Issue
Block a user