from __future__ import annotations import asyncio import logging from dataclasses import dataclass from pathlib import Path from typing import Any, Dict, Optional, List from ..config import ( MultiProjectItemConfig, MultiProjectRunConfig, PipelineRunConfig, StrategyRuntimeConfig, apply_config_overrides, build_config_from_file, ) from ..logging import get_scope_logger, get_scope_display_name from .factory import create_pipeline_from_config, create_scope_runtime_from_config logger = get_scope_logger("multi-project", __name__) @dataclass(frozen=True) class ImplicitProjectItem: local_project_id: str remote_project_id: str @property def scope(self) -> str: return f"project_id:{self.local_project_id}" def _resolve_embedded_config_path(raw_path: str, *, project_root: Path, profile_path: Path) -> Path: candidate = Path(raw_path) if candidate.is_absolute(): return candidate profile_relative = (profile_path.parent / candidate).resolve() if profile_relative.exists(): return profile_relative return (project_root / candidate).resolve() def _project_scope_overrides(item: MultiProjectItemConfig, *, node_types: list[str]) -> Dict[str, object]: del node_types local_handler_configs = { "project": {"data_id_filter": [item.local_project_id]}, } remote_handler_configs = { "project": {"data_id_filter": [item.remote_project_id]}, } return { "local_datasource": { "handler_configs": local_handler_configs, }, "remote_datasource": { "handler_configs": remote_handler_configs, }, } def _resolve_project_bootstrap_node_types( *, config: MultiProjectRunConfig, shared_node_types: list[str], ) -> list[str]: configured = [node_type for node_type in config.project_bootstrap_node_types if node_type] if configured: return list(dict.fromkeys(configured)) return list(dict.fromkeys(shared_node_types)) def _normalize_project_id_items(raw_items: Any) -> list[str]: items = raw_items if isinstance(raw_items, list) else [] normalized: list[str] = [] for item in items: value = str(item).strip() if value and value not in normalized: normalized.append(value) return normalized def _get_project_handler_filters(config: PipelineRunConfig) -> tuple[list[str], list[str]]: local_filters = _normalize_project_id_items( config.local_datasource.handler_configs.get("project", {}).get("data_id_filter", []) ) remote_filters = _normalize_project_id_items( config.remote_datasource.handler_configs.get("project", {}).get("data_id_filter", []) ) return local_filters, remote_filters def _get_project_strategy_runtime(config: PipelineRunConfig) -> StrategyRuntimeConfig | None: for item in config.strategies: if item.node_type == "project": return item return None def _get_project_prebind_pairs(config: PipelineRunConfig) -> list[tuple[str, str]]: runtime_cfg = _get_project_strategy_runtime(config) if runtime_cfg is None: return [] pairs: list[tuple[str, str]] = [] for item in runtime_cfg.config.pre_bind_data_id: local_id = str(item.local_data_id).strip() remote_id = str(item.remote_data_id).strip() if not local_id or not remote_id: continue pairs.append((local_id, remote_id)) return pairs def is_implicit_project_batch_config(config: PipelineRunConfig) -> bool: if "project" not in config.node_types: return False local_filters, remote_filters = _get_project_handler_filters(config) prebind_pairs = _get_project_prebind_pairs(config) return max(len(local_filters), len(remote_filters), len(prebind_pairs)) > 1 def _extract_implicit_project_items(config: PipelineRunConfig) -> list[ImplicitProjectItem]: local_filters, remote_filters = _get_project_handler_filters(config) prebind_pairs = _get_project_prebind_pairs(config) if not prebind_pairs: raise ValueError("implicit multi-project config requires strategies.project.config.pre_bind_data_id") pair_local_ids = [local_id for local_id, _ in prebind_pairs] pair_remote_ids = [remote_id for _, remote_id in prebind_pairs] effective_local_filters = local_filters or list(dict.fromkeys(pair_local_ids)) effective_remote_filters = remote_filters or list(dict.fromkeys(pair_remote_ids)) if set(effective_local_filters) != set(pair_local_ids): raise ValueError( "implicit multi-project config mismatch: local project data_id_filter must match project pre_bind_data_id local_data_id" ) if set(effective_remote_filters) != set(pair_remote_ids): raise ValueError( "implicit multi-project config mismatch: remote project data_id_filter must match project pre_bind_data_id remote_data_id" ) seen_local_ids: set[str] = set() seen_remote_ids: set[str] = set() items: list[ImplicitProjectItem] = [] for local_id, remote_id in prebind_pairs: if local_id in seen_local_ids: raise ValueError(f"duplicate local project id in implicit multi-project config: {local_id}") if remote_id in seen_remote_ids: raise ValueError(f"duplicate remote project id in implicit multi-project config: {remote_id}") seen_local_ids.add(local_id) seen_remote_ids.add(remote_id) items.append(ImplicitProjectItem(local_project_id=local_id, remote_project_id=remote_id)) return items def _split_implicit_project_node_types(node_types: list[str]) -> tuple[list[str], list[str]]: if "project" not in node_types: raise ValueError("implicit multi-project config requires node_types to include 'project'") project_index = node_types.index("project") return node_types[:project_index], node_types[project_index:] def _build_implicit_project_config( base_config: PipelineRunConfig, *, item: ImplicitProjectItem, project_node_types: list[str], ) -> PipelineRunConfig: return apply_config_overrides( base_config, { "node_types": project_node_types, "persist": {"wipe_on_start": False}, "local_datasource": { "handler_configs": { "project": {"data_id_filter": [item.local_project_id]}, } }, "remote_datasource": { "handler_configs": { "project": {"data_id_filter": [item.remote_project_id]}, } }, "strategies": { "project": { "config": { "pre_bind_data_id": [ { "local_data_id": item.local_project_id, "remote_data_id": item.remote_project_id, } ] } } }, }, ) async def run_implicit_project_batch_from_config( config: PipelineRunConfig, *, title: str, print_summary: bool = True, max_concurrency: int | None = None, ) -> Dict[str, Dict[str, object]]: from .factory import create_pipeline_from_config, create_scope_runtime_from_config project_items = _extract_implicit_project_items(config) shared_node_types, project_node_types = _split_implicit_project_node_types(list(config.node_types)) results: Dict[str, Dict[str, object]] = {} global_runtime = None global_pipeline = None global_config = None if print_summary: logger.info("\n%s", "=" * 80) logger.info("🚀 Creating implicit multi-project pipeline from in-memory config") logger.info("📦 Projects=%d", len(project_items)) logger.info("=" * 80) try: if shared_node_types: global_config = apply_config_overrides( config, { "node_types": shared_node_types, }, ) global_runtime = await create_scope_runtime_from_config(global_config, scope="global") global_pipeline = await create_pipeline_from_config( global_config, scope="global", pipeline_name=f"{title} [global]", scope_runtime=global_runtime, ) results["global"] = await global_pipeline.run() semaphore = asyncio.Semaphore(max(1, max_concurrency or len(project_items))) async def _run_project(item: ImplicitProjectItem) -> None: project_config = _build_implicit_project_config( config, item=item, project_node_types=project_node_types, ) async with semaphore: project_runtime = await create_scope_runtime_from_config( project_config, scope=item.scope, persistence_service=global_runtime.persistence_service if global_runtime is not None else None, local_fallback_collection=global_runtime.local_collection if global_runtime is not None else None, remote_fallback_collection=global_runtime.remote_collection if global_runtime is not None else None, binding_fallback_manager=global_runtime.binding_manager if global_runtime is not None else None, global_runtime=global_runtime, ) try: project_pipeline = await create_pipeline_from_config( project_config, scope=item.scope, pipeline_name=f"{title} [{item.scope}]", bootstrap_binding_node_types=shared_node_types, ephemeral_node_types=shared_node_types, scope_runtime=project_runtime, ) results[item.scope] = await project_pipeline.run() finally: await project_runtime.unload() await project_runtime.close() await asyncio.gather(*[_run_project(item) for item in project_items]) return results finally: if global_runtime is not None: await global_runtime.unload() await global_runtime.close() class ProjectBatchPipeline: def __init__(self, *, project_root: Path, config_path: Path, config: MultiProjectRunConfig): self.project_root = project_root self.config_path = config_path self.config = config def _load_pipeline_config(self, raw_path: str) -> PipelineRunConfig: resolved = _resolve_embedded_config_path(raw_path, project_root=self.project_root, profile_path=self.config_path) return build_config_from_file(project_root=self.project_root, file_path=resolved) def _prepare_project_config( self, *, shared_persist: Dict[str, object], item: MultiProjectItemConfig, base_path: Optional[str], ) -> PipelineRunConfig: raw_path = base_path or self.config.default_project_config cfg = self._load_pipeline_config(raw_path) cfg = apply_config_overrides(cfg, {"persist": {**shared_persist, "wipe_on_start": False}}) return apply_config_overrides(cfg, _project_scope_overrides(item, node_types=list(cfg.node_types))) async def run(self) -> Dict[str, Dict[str, object]]: results: Dict[str, Dict[str, object]] = {} global_config = self._load_pipeline_config(self.config.global_config) shared_persist = global_config.persist.model_dump(mode="json") shared_node_types = list(global_config.node_types) project_bootstrap_node_types = _resolve_project_bootstrap_node_types( config=self.config, shared_node_types=shared_node_types, ) logger.info("🚀 Multi-project pipeline start: projects=%d", len(self.config.projects)) global_runtime = await create_scope_runtime_from_config( global_config, scope="global", ) global_pipeline = None try: global_pipeline = await create_pipeline_from_config( global_config, scope="global", pipeline_name="multi-project pipeline [global]", scope_runtime=global_runtime, ) results["global"] = await global_pipeline.run() semaphore = asyncio.Semaphore(self.config.max_concurrency) async def _run_project(item: MultiProjectItemConfig) -> None: project_config = self._prepare_project_config( shared_persist=shared_persist, item=item, base_path=item.config_path, ) scope = item.scope title = f"multi-project pipeline [{scope}]" logger.info( "🚀 Project pipeline start: scope=%s local_project_id=%s remote_project_id=%s", get_scope_display_name(scope), item.local_project_id, item.remote_project_id, ) async with semaphore: project_runtime = await create_scope_runtime_from_config( project_config, scope=scope, persistence_service=global_runtime.persistence_service, local_fallback_collection=global_runtime.local_collection, remote_fallback_collection=global_runtime.remote_collection, binding_fallback_manager=global_runtime.binding_manager, global_runtime=global_runtime, ) project_pipeline = None try: project_pipeline = await create_pipeline_from_config( project_config, scope=scope, pipeline_name=title, bootstrap_binding_node_types=project_bootstrap_node_types, ephemeral_node_types=project_bootstrap_node_types, scope_runtime=project_runtime, ) results[scope] = await project_pipeline.run() finally: await project_runtime.unload() await project_runtime.close() await asyncio.gather(*[_run_project(item) for item in self.config.projects]) finally: await global_runtime.unload() await global_runtime.close() return results class MultiProjectPipeline(ProjectBatchPipeline): pass