初步实现多project pipeline

2026-04-07 14:45:19 +08:00
parent e3eab5ade1
commit 966f2e1270
11 changed files with 1083 additions and 169 deletions
@@ -1,28 +1,34 @@
 from __future__ import annotations

+import asyncio
 import logging
-import copy
+from dataclasses import dataclass
 from pathlib import Path
-from typing import Dict, Optional, List, TypedDict, Any
+from typing import Any, Dict, Optional, List

 from ..config import (
    MultiProjectItemConfig,
    MultiProjectRunConfig,
    PipelineRunConfig,
+    StrategyRuntimeConfig,
    apply_config_overrides,
    build_config_from_file,
 )
-from ..logging import get_logger
-from .factory import create_pipeline_from_config
+from ..logging import get_scope_logger
+from .factory import create_pipeline_from_config, create_scope_runtime_from_config


-logger = get_logger(__name__)
+logger = get_scope_logger("multi-project", __name__)


-class _SharedStateSnapshot(TypedDict):
-    local_nodes: List[Any]
-    remote_nodes: List[Any]
-    bindings_by_type: Dict[str, Dict[str, Optional[str]]]
+@dataclass(frozen=True)
+class ImplicitProjectItem:
+    local_project_id: str
+    remote_project_id: str
+
+    @property
+    def scope(self) -> str:
+        return f"project_id:{self.local_project_id}"


 def _resolve_embedded_config_path(raw_path: str, *, project_root: Path, profile_path: Path) -> Path:
@@ -65,7 +71,215 @@ def _resolve_project_bootstrap_node_types(
    return list(dict.fromkeys(shared_node_types))


-class MultiProjectPipeline:
+def _normalize_project_id_items(raw_items: Any) -> list[str]:
+    items = raw_items if isinstance(raw_items, list) else []
+    normalized: list[str] = []
+    for item in items:
+        value = str(item).strip()
+        if value and value not in normalized:
+            normalized.append(value)
+    return normalized
+
+
+def _get_project_handler_filters(config: PipelineRunConfig) -> tuple[list[str], list[str]]:
+    local_filters = _normalize_project_id_items(
+        config.local_datasource.handler_configs.get("project", {}).get("data_id_filter", [])
+    )
+    remote_filters = _normalize_project_id_items(
+        config.remote_datasource.handler_configs.get("project", {}).get("data_id_filter", [])
+    )
+    return local_filters, remote_filters
+
+
+def _get_project_strategy_runtime(config: PipelineRunConfig) -> StrategyRuntimeConfig | None:
+    for item in config.strategies:
+        if item.node_type == "project":
+            return item
+    return None
+
+
+def _get_project_prebind_pairs(config: PipelineRunConfig) -> list[tuple[str, str]]:
+    runtime_cfg = _get_project_strategy_runtime(config)
+    if runtime_cfg is None:
+        return []
+    pairs: list[tuple[str, str]] = []
+    for item in runtime_cfg.config.pre_bind_data_id:
+        local_id = str(item.local_data_id).strip()
+        remote_id = str(item.remote_data_id).strip()
+        if not local_id or not remote_id:
+            continue
+        pairs.append((local_id, remote_id))
+    return pairs
+
+
+def is_implicit_project_batch_config(config: PipelineRunConfig) -> bool:
+    if "project" not in config.node_types:
+        return False
+    local_filters, remote_filters = _get_project_handler_filters(config)
+    prebind_pairs = _get_project_prebind_pairs(config)
+    return max(len(local_filters), len(remote_filters), len(prebind_pairs)) > 1
+
+
+def _extract_implicit_project_items(config: PipelineRunConfig) -> list[ImplicitProjectItem]:
+    local_filters, remote_filters = _get_project_handler_filters(config)
+    prebind_pairs = _get_project_prebind_pairs(config)
+    if not prebind_pairs:
+        raise ValueError("implicit multi-project config requires strategies.project.config.pre_bind_data_id")
+
+    pair_local_ids = [local_id for local_id, _ in prebind_pairs]
+    pair_remote_ids = [remote_id for _, remote_id in prebind_pairs]
+
+    effective_local_filters = local_filters or list(dict.fromkeys(pair_local_ids))
+    effective_remote_filters = remote_filters or list(dict.fromkeys(pair_remote_ids))
+
+    if set(effective_local_filters) != set(pair_local_ids):
+        raise ValueError(
+            "implicit multi-project config mismatch: local project data_id_filter must match project pre_bind_data_id local_data_id"
+        )
+    if set(effective_remote_filters) != set(pair_remote_ids):
+        raise ValueError(
+            "implicit multi-project config mismatch: remote project data_id_filter must match project pre_bind_data_id remote_data_id"
+        )
+
+    seen_local_ids: set[str] = set()
+    seen_remote_ids: set[str] = set()
+    items: list[ImplicitProjectItem] = []
+    for local_id, remote_id in prebind_pairs:
+        if local_id in seen_local_ids:
+            raise ValueError(f"duplicate local project id in implicit multi-project config: {local_id}")
+        if remote_id in seen_remote_ids:
+            raise ValueError(f"duplicate remote project id in implicit multi-project config: {remote_id}")
+        seen_local_ids.add(local_id)
+        seen_remote_ids.add(remote_id)
+        items.append(ImplicitProjectItem(local_project_id=local_id, remote_project_id=remote_id))
+    return items
+
+
+def _split_implicit_project_node_types(node_types: list[str]) -> tuple[list[str], list[str]]:
+    if "project" not in node_types:
+        raise ValueError("implicit multi-project config requires node_types to include 'project'")
+    project_index = node_types.index("project")
+    return node_types[:project_index], node_types[project_index:]
+
+
+def _build_implicit_project_config(
+    base_config: PipelineRunConfig,
+    *,
+    item: ImplicitProjectItem,
+    project_node_types: list[str],
+) -> PipelineRunConfig:
+    return apply_config_overrides(
+        base_config,
+        {
+            "node_types": project_node_types,
+            "persist": {"wipe_on_start": False},
+            "local_datasource": {
+                "handler_configs": {
+                    "project": {"data_id_filter": [item.local_project_id]},
+                }
+            },
+            "remote_datasource": {
+                "handler_configs": {
+                    "project": {"data_id_filter": [item.remote_project_id]},
+                }
+            },
+            "strategies": {
+                "project": {
+                    "config": {
+                        "pre_bind_data_id": [
+                            {
+                                "local_data_id": item.local_project_id,
+                                "remote_data_id": item.remote_project_id,
+                            }
+                        ]
+                    }
+                }
+            },
+        },
+    )
+
+
+async def run_implicit_project_batch_from_config(
+    config: PipelineRunConfig,
+    *,
+    title: str,
+    print_summary: bool = True,
+    max_concurrency: int | None = None,
+) -> Dict[str, Dict[str, object]]:
+    from .factory import create_pipeline_from_config, create_scope_runtime_from_config
+
+    project_items = _extract_implicit_project_items(config)
+    shared_node_types, project_node_types = _split_implicit_project_node_types(list(config.node_types))
+    results: Dict[str, Dict[str, object]] = {}
+
+    global_runtime = None
+    global_pipeline = None
+    global_config = None
+
+    if print_summary:
+        logger.info("\n%s", "=" * 80)
+        logger.info("🚀 Creating implicit multi-project pipeline from in-memory config")
+        logger.info("📦 Projects=%d", len(project_items))
+        logger.info("=" * 80)
+
+    try:
+        if shared_node_types:
+            global_config = apply_config_overrides(
+                config,
+                {
+                    "node_types": shared_node_types,
+                },
+            )
+            global_runtime = await create_scope_runtime_from_config(global_config, scope="global")
+            global_pipeline = await create_pipeline_from_config(
+                global_config,
+                scope="global",
+                pipeline_name=f"{title} [global]",
+                scope_runtime=global_runtime,
+            )
+            results["global"] = await global_pipeline.run()
+
+        semaphore = asyncio.Semaphore(max(1, max_concurrency or len(project_items)))
+
+        async def _run_project(item: ImplicitProjectItem) -> None:
+            project_config = _build_implicit_project_config(
+                config,
+                item=item,
+                project_node_types=project_node_types,
+            )
+            async with semaphore:
+                project_runtime = await create_scope_runtime_from_config(
+                    project_config,
+                    scope=item.scope,
+                    persistence_service=global_runtime.persistence_service if global_runtime is not None else None,
+                    local_fallback_collection=global_runtime.local_collection if global_runtime is not None else None,
+                    remote_fallback_collection=global_runtime.remote_collection if global_runtime is not None else None,
+                    binding_fallback_manager=global_runtime.binding_manager if global_runtime is not None else None,
+                    global_runtime=global_runtime,
+                )
+                try:
+                    project_pipeline = await create_pipeline_from_config(
+                        project_config,
+                        scope=item.scope,
+                        pipeline_name=f"{title} [{item.scope}]",
+                        bootstrap_binding_node_types=shared_node_types,
+                        ephemeral_node_types=shared_node_types,
+                        scope_runtime=project_runtime,
+                    )
+                    results[item.scope] = await project_pipeline.run()
+                finally:
+                    await project_runtime.unload()
+                    await project_runtime.close()
+
+        await asyncio.gather(*[_run_project(item) for item in project_items])
+        return results
+    finally:
+        if global_runtime is not None:
+            await global_runtime.unload()
+            await global_runtime.close()
+
+
+class ProjectBatchPipeline:
    def __init__(self, *, project_root: Path, config_path: Path, config: MultiProjectRunConfig):
        self.project_root = project_root
        self.config_path = config_path
@@ -87,27 +301,6 @@ class MultiProjectPipeline:
        cfg = apply_config_overrides(cfg, {"persist": {**shared_persist, "wipe_on_start": False}})
        return apply_config_overrides(cfg, _project_scope_overrides(item, node_types=list(cfg.node_types)))

-    async def _snapshot_shared_state(self, *, pipeline, node_types: List[str]) -> _SharedStateSnapshot:
-        local_nodes = [
-            copy.deepcopy(node)
-            for node_type in node_types
-            for node in pipeline.local_collection.filter(node_type=node_type)
-        ]
-        remote_nodes = [
-            copy.deepcopy(node)
-            for node_type in node_types
-            for node in pipeline.remote_collection.filter(node_type=node_type)
-        ]
-        bindings_by_type: Dict[str, Dict[str, Optional[str]]] = {}
-        for node_type in node_types:
-            bindings = await pipeline.binding_manager.get_all_bindings(node_type)
-            bindings_by_type[node_type] = {local_id: remote_id for local_id, remote_id in bindings}
-        return {
-            "local_nodes": local_nodes,
-            "remote_nodes": remote_nodes,
-            "bindings_by_type": bindings_by_type,
-        }
-
    async def run(self) -> Dict[str, Dict[str, object]]:
        results: Dict[str, Dict[str, object]] = {}

@@ -119,49 +312,67 @@ class MultiProjectPipeline:
            shared_node_types=shared_node_types,
        )
        logger.info("🚀 Multi-project pipeline start: projects=%d", len(self.config.projects))
-        global_pipeline = await create_pipeline_from_config(
+        global_runtime = await create_scope_runtime_from_config(
            global_config,
            scope="global",
-            pipeline_name="multi-project pipeline [global]",
        )
+        global_pipeline = None
        try:
+            global_pipeline = await create_pipeline_from_config(
+                global_config,
+                scope="global",
+                pipeline_name="multi-project pipeline [global]",
+                scope_runtime=global_runtime,
+            )
            results["global"] = await global_pipeline.run()
-            shared_state = await self._snapshot_shared_state(
-                pipeline=global_pipeline,
-                node_types=project_bootstrap_node_types,
-            )
-        finally:
-            await global_pipeline.close()
+            semaphore = asyncio.Semaphore(self.config.max_concurrency)

-        for item in self.config.projects:
-            project_config = self._prepare_project_config(
-                shared_persist=shared_persist,
-                item=item,
-                base_path=item.config_path,
-            )
-            scope = item.scope
-            title = f"multi-project pipeline [{scope}]"
-            logger.info(
-                "🚀 Project pipeline start: scope=%s local_project_id=%s remote_project_id=%s",
-                scope,
-                item.local_project_id,
-                item.remote_project_id,
-            )
-            project_pipeline = await create_pipeline_from_config(
-                project_config,
-                scope=scope,
-                pipeline_name=title,
-                bootstrap_binding_node_types=project_bootstrap_node_types,
-                ephemeral_node_types=project_bootstrap_node_types,
-            )
-            try:
-                project_pipeline.preload_shared_state(
-                    local_nodes=shared_state["local_nodes"],
-                    remote_nodes=shared_state["remote_nodes"],
-                    bindings_by_type=shared_state["bindings_by_type"],
+            async def _run_project(item: MultiProjectItemConfig) -> None:
+                project_config = self._prepare_project_config(
+                    shared_persist=shared_persist,
+                    item=item,
+                    base_path=item.config_path,
                )
-                results[scope] = await project_pipeline.run()
-            finally:
-                await project_pipeline.close()
+                scope = item.scope
+                title = f"multi-project pipeline [{scope}]"
+                logger.info(
+                    "🚀 Project pipeline start: scope=%s local_project_id=%s remote_project_id=%s",
+                    scope,
+                    item.local_project_id,
+                    item.remote_project_id,
+                )
+                async with semaphore:
+                    project_runtime = await create_scope_runtime_from_config(
+                        project_config,
+                        scope=scope,
+                        persistence_service=global_runtime.persistence_service,
+                        local_fallback_collection=global_runtime.local_collection,
+                        remote_fallback_collection=global_runtime.remote_collection,
+                        binding_fallback_manager=global_runtime.binding_manager,
+                        global_runtime=global_runtime,
+                    )
+                    project_pipeline = None
+                    try:
+                        project_pipeline = await create_pipeline_from_config(
+                            project_config,
+                            scope=scope,
+                            pipeline_name=title,
+                            bootstrap_binding_node_types=project_bootstrap_node_types,
+                            ephemeral_node_types=project_bootstrap_node_types,
+                            scope_runtime=project_runtime,
+                        )
+                        results[scope] = await project_pipeline.run()
+                    finally:
+                        await project_runtime.unload()
+                        await project_runtime.close()

-        return results
+            await asyncio.gather(*[_run_project(item) for item in self.config.projects])
+        finally:
+            await global_runtime.unload()
+            await global_runtime.close()
+
+        return results
+
+
+class MultiProjectPipeline(ProjectBatchPipeline):
+    pass