strepsiades
378 lines
15 KiB
Python
378 lines
15 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any, Dict, Optional, List
|
|
|
|
from ..config import (
|
|
MultiProjectItemConfig,
|
|
MultiProjectRunConfig,
|
|
PipelineRunConfig,
|
|
StrategyRuntimeConfig,
|
|
apply_config_overrides,
|
|
build_config_from_file,
|
|
)
|
|
from ..logging import get_scope_logger, get_scope_display_name
|
|
from .factory import create_pipeline_from_config, create_scope_runtime_from_config
|
|
|
|
|
|
logger = get_scope_logger("multi-project", __name__)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ImplicitProjectItem:
|
|
local_project_id: str
|
|
remote_project_id: str
|
|
|
|
@property
|
|
def scope(self) -> str:
|
|
return f"project_id:{self.local_project_id}"
|
|
|
|
|
|
def _resolve_embedded_config_path(raw_path: str, *, project_root: Path, profile_path: Path) -> Path:
|
|
candidate = Path(raw_path)
|
|
if candidate.is_absolute():
|
|
return candidate
|
|
|
|
profile_relative = (profile_path.parent / candidate).resolve()
|
|
if profile_relative.exists():
|
|
return profile_relative
|
|
return (project_root / candidate).resolve()
|
|
|
|
|
|
def _project_scope_overrides(item: MultiProjectItemConfig, *, node_types: list[str]) -> Dict[str, object]:
|
|
del node_types
|
|
local_handler_configs = {
|
|
"project": {"data_id_filter": [item.local_project_id]},
|
|
}
|
|
remote_handler_configs = {
|
|
"project": {"data_id_filter": [item.remote_project_id]},
|
|
}
|
|
return {
|
|
"local_datasource": {
|
|
"handler_configs": local_handler_configs,
|
|
},
|
|
"remote_datasource": {
|
|
"handler_configs": remote_handler_configs,
|
|
},
|
|
}
|
|
|
|
|
|
def _resolve_project_bootstrap_node_types(
|
|
*,
|
|
config: MultiProjectRunConfig,
|
|
shared_node_types: list[str],
|
|
) -> list[str]:
|
|
configured = [node_type for node_type in config.project_bootstrap_node_types if node_type]
|
|
if configured:
|
|
return list(dict.fromkeys(configured))
|
|
return list(dict.fromkeys(shared_node_types))
|
|
|
|
|
|
def _normalize_project_id_items(raw_items: Any) -> list[str]:
|
|
items = raw_items if isinstance(raw_items, list) else []
|
|
normalized: list[str] = []
|
|
for item in items:
|
|
value = str(item).strip()
|
|
if value and value not in normalized:
|
|
normalized.append(value)
|
|
return normalized
|
|
|
|
|
|
def _get_project_handler_filters(config: PipelineRunConfig) -> tuple[list[str], list[str]]:
|
|
local_filters = _normalize_project_id_items(
|
|
config.local_datasource.handler_configs.get("project", {}).get("data_id_filter", [])
|
|
)
|
|
remote_filters = _normalize_project_id_items(
|
|
config.remote_datasource.handler_configs.get("project", {}).get("data_id_filter", [])
|
|
)
|
|
return local_filters, remote_filters
|
|
|
|
|
|
def _get_project_strategy_runtime(config: PipelineRunConfig) -> StrategyRuntimeConfig | None:
|
|
for item in config.strategies:
|
|
if item.node_type == "project":
|
|
return item
|
|
return None
|
|
|
|
|
|
def _get_project_prebind_pairs(config: PipelineRunConfig) -> list[tuple[str, str]]:
|
|
runtime_cfg = _get_project_strategy_runtime(config)
|
|
if runtime_cfg is None:
|
|
return []
|
|
pairs: list[tuple[str, str]] = []
|
|
for item in runtime_cfg.config.pre_bind_data_id:
|
|
local_id = str(item.local_data_id).strip()
|
|
remote_id = str(item.remote_data_id).strip()
|
|
if not local_id or not remote_id:
|
|
continue
|
|
pairs.append((local_id, remote_id))
|
|
return pairs
|
|
|
|
|
|
def is_implicit_project_batch_config(config: PipelineRunConfig) -> bool:
|
|
if "project" not in config.node_types:
|
|
return False
|
|
local_filters, remote_filters = _get_project_handler_filters(config)
|
|
prebind_pairs = _get_project_prebind_pairs(config)
|
|
return max(len(local_filters), len(remote_filters), len(prebind_pairs)) > 1
|
|
|
|
|
|
def _extract_implicit_project_items(config: PipelineRunConfig) -> list[ImplicitProjectItem]:
|
|
local_filters, remote_filters = _get_project_handler_filters(config)
|
|
prebind_pairs = _get_project_prebind_pairs(config)
|
|
if not prebind_pairs:
|
|
raise ValueError("implicit multi-project config requires strategies.project.config.pre_bind_data_id")
|
|
|
|
pair_local_ids = [local_id for local_id, _ in prebind_pairs]
|
|
pair_remote_ids = [remote_id for _, remote_id in prebind_pairs]
|
|
|
|
effective_local_filters = local_filters or list(dict.fromkeys(pair_local_ids))
|
|
effective_remote_filters = remote_filters or list(dict.fromkeys(pair_remote_ids))
|
|
|
|
if set(effective_local_filters) != set(pair_local_ids):
|
|
raise ValueError(
|
|
"implicit multi-project config mismatch: local project data_id_filter must match project pre_bind_data_id local_data_id"
|
|
)
|
|
if set(effective_remote_filters) != set(pair_remote_ids):
|
|
raise ValueError(
|
|
"implicit multi-project config mismatch: remote project data_id_filter must match project pre_bind_data_id remote_data_id"
|
|
)
|
|
|
|
seen_local_ids: set[str] = set()
|
|
seen_remote_ids: set[str] = set()
|
|
items: list[ImplicitProjectItem] = []
|
|
for local_id, remote_id in prebind_pairs:
|
|
if local_id in seen_local_ids:
|
|
raise ValueError(f"duplicate local project id in implicit multi-project config: {local_id}")
|
|
if remote_id in seen_remote_ids:
|
|
raise ValueError(f"duplicate remote project id in implicit multi-project config: {remote_id}")
|
|
seen_local_ids.add(local_id)
|
|
seen_remote_ids.add(remote_id)
|
|
items.append(ImplicitProjectItem(local_project_id=local_id, remote_project_id=remote_id))
|
|
return items
|
|
|
|
|
|
def _split_implicit_project_node_types(node_types: list[str]) -> tuple[list[str], list[str]]:
|
|
if "project" not in node_types:
|
|
raise ValueError("implicit multi-project config requires node_types to include 'project'")
|
|
project_index = node_types.index("project")
|
|
return node_types[:project_index], node_types[project_index:]
|
|
|
|
|
|
def _build_implicit_project_config(
|
|
base_config: PipelineRunConfig,
|
|
*,
|
|
item: ImplicitProjectItem,
|
|
project_node_types: list[str],
|
|
) -> PipelineRunConfig:
|
|
return apply_config_overrides(
|
|
base_config,
|
|
{
|
|
"node_types": project_node_types,
|
|
"persist": {"wipe_on_start": False},
|
|
"local_datasource": {
|
|
"handler_configs": {
|
|
"project": {"data_id_filter": [item.local_project_id]},
|
|
}
|
|
},
|
|
"remote_datasource": {
|
|
"handler_configs": {
|
|
"project": {"data_id_filter": [item.remote_project_id]},
|
|
}
|
|
},
|
|
"strategies": {
|
|
"project": {
|
|
"config": {
|
|
"pre_bind_data_id": [
|
|
{
|
|
"local_data_id": item.local_project_id,
|
|
"remote_data_id": item.remote_project_id,
|
|
}
|
|
]
|
|
}
|
|
}
|
|
},
|
|
},
|
|
)
|
|
|
|
|
|
async def run_implicit_project_batch_from_config(
|
|
config: PipelineRunConfig,
|
|
*,
|
|
title: str,
|
|
print_summary: bool = True,
|
|
max_concurrency: int | None = None,
|
|
) -> Dict[str, Dict[str, object]]:
|
|
from .factory import create_pipeline_from_config, create_scope_runtime_from_config
|
|
|
|
project_items = _extract_implicit_project_items(config)
|
|
shared_node_types, project_node_types = _split_implicit_project_node_types(list(config.node_types))
|
|
results: Dict[str, Dict[str, object]] = {}
|
|
|
|
global_runtime = None
|
|
global_pipeline = None
|
|
global_config = None
|
|
|
|
if print_summary:
|
|
logger.info("\n%s", "=" * 80)
|
|
logger.info("🚀 Creating implicit multi-project pipeline from in-memory config")
|
|
logger.info("📦 Projects=%d", len(project_items))
|
|
logger.info("=" * 80)
|
|
|
|
try:
|
|
if shared_node_types:
|
|
global_config = apply_config_overrides(
|
|
config,
|
|
{
|
|
"node_types": shared_node_types,
|
|
},
|
|
)
|
|
global_runtime = await create_scope_runtime_from_config(global_config, scope="global")
|
|
global_pipeline = await create_pipeline_from_config(
|
|
global_config,
|
|
scope="global",
|
|
pipeline_name=f"{title} [global]",
|
|
scope_runtime=global_runtime,
|
|
)
|
|
results["global"] = await global_pipeline.run()
|
|
|
|
semaphore = asyncio.Semaphore(max(1, max_concurrency or len(project_items)))
|
|
|
|
async def _run_project(item: ImplicitProjectItem) -> None:
|
|
project_config = _build_implicit_project_config(
|
|
config,
|
|
item=item,
|
|
project_node_types=project_node_types,
|
|
)
|
|
async with semaphore:
|
|
project_runtime = await create_scope_runtime_from_config(
|
|
project_config,
|
|
scope=item.scope,
|
|
persistence_service=global_runtime.persistence_service if global_runtime is not None else None,
|
|
local_fallback_collection=global_runtime.local_collection if global_runtime is not None else None,
|
|
remote_fallback_collection=global_runtime.remote_collection if global_runtime is not None else None,
|
|
binding_fallback_manager=global_runtime.binding_manager if global_runtime is not None else None,
|
|
global_runtime=global_runtime,
|
|
)
|
|
try:
|
|
project_pipeline = await create_pipeline_from_config(
|
|
project_config,
|
|
scope=item.scope,
|
|
pipeline_name=f"{title} [{item.scope}]",
|
|
bootstrap_binding_node_types=shared_node_types,
|
|
ephemeral_node_types=shared_node_types,
|
|
scope_runtime=project_runtime,
|
|
)
|
|
results[item.scope] = await project_pipeline.run()
|
|
finally:
|
|
await project_runtime.unload()
|
|
await project_runtime.close()
|
|
|
|
await asyncio.gather(*[_run_project(item) for item in project_items])
|
|
return results
|
|
finally:
|
|
if global_runtime is not None:
|
|
await global_runtime.unload()
|
|
await global_runtime.close()
|
|
|
|
|
|
class ProjectBatchPipeline:
|
|
def __init__(self, *, project_root: Path, config_path: Path, config: MultiProjectRunConfig):
|
|
self.project_root = project_root
|
|
self.config_path = config_path
|
|
self.config = config
|
|
|
|
def _load_pipeline_config(self, raw_path: str) -> PipelineRunConfig:
|
|
resolved = _resolve_embedded_config_path(raw_path, project_root=self.project_root, profile_path=self.config_path)
|
|
return build_config_from_file(project_root=self.project_root, file_path=resolved)
|
|
|
|
def _prepare_project_config(
|
|
self,
|
|
*,
|
|
shared_persist: Dict[str, object],
|
|
item: MultiProjectItemConfig,
|
|
base_path: Optional[str],
|
|
) -> PipelineRunConfig:
|
|
raw_path = base_path or self.config.default_project_config
|
|
cfg = self._load_pipeline_config(raw_path)
|
|
cfg = apply_config_overrides(cfg, {"persist": {**shared_persist, "wipe_on_start": False}})
|
|
return apply_config_overrides(cfg, _project_scope_overrides(item, node_types=list(cfg.node_types)))
|
|
|
|
async def run(self) -> Dict[str, Dict[str, object]]:
|
|
results: Dict[str, Dict[str, object]] = {}
|
|
|
|
global_config = self._load_pipeline_config(self.config.global_config)
|
|
shared_persist = global_config.persist.model_dump(mode="json")
|
|
shared_node_types = list(global_config.node_types)
|
|
project_bootstrap_node_types = _resolve_project_bootstrap_node_types(
|
|
config=self.config,
|
|
shared_node_types=shared_node_types,
|
|
)
|
|
logger.info("🚀 Multi-project pipeline start: projects=%d", len(self.config.projects))
|
|
global_runtime = await create_scope_runtime_from_config(
|
|
global_config,
|
|
scope="global",
|
|
)
|
|
global_pipeline = None
|
|
try:
|
|
global_pipeline = await create_pipeline_from_config(
|
|
global_config,
|
|
scope="global",
|
|
pipeline_name="multi-project pipeline [global]",
|
|
scope_runtime=global_runtime,
|
|
)
|
|
results["global"] = await global_pipeline.run()
|
|
semaphore = asyncio.Semaphore(self.config.max_concurrency)
|
|
|
|
async def _run_project(item: MultiProjectItemConfig) -> None:
|
|
project_config = self._prepare_project_config(
|
|
shared_persist=shared_persist,
|
|
item=item,
|
|
base_path=item.config_path,
|
|
)
|
|
scope = item.scope
|
|
title = f"multi-project pipeline [{scope}]"
|
|
logger.info(
|
|
"🚀 Project pipeline start: scope=%s local_project_id=%s remote_project_id=%s",
|
|
get_scope_display_name(scope),
|
|
item.local_project_id,
|
|
item.remote_project_id,
|
|
)
|
|
async with semaphore:
|
|
project_runtime = await create_scope_runtime_from_config(
|
|
project_config,
|
|
scope=scope,
|
|
persistence_service=global_runtime.persistence_service,
|
|
local_fallback_collection=global_runtime.local_collection,
|
|
remote_fallback_collection=global_runtime.remote_collection,
|
|
binding_fallback_manager=global_runtime.binding_manager,
|
|
global_runtime=global_runtime,
|
|
)
|
|
project_pipeline = None
|
|
try:
|
|
project_pipeline = await create_pipeline_from_config(
|
|
project_config,
|
|
scope=scope,
|
|
pipeline_name=title,
|
|
bootstrap_binding_node_types=project_bootstrap_node_types,
|
|
ephemeral_node_types=project_bootstrap_node_types,
|
|
scope_runtime=project_runtime,
|
|
)
|
|
results[scope] = await project_pipeline.run()
|
|
finally:
|
|
await project_runtime.unload()
|
|
await project_runtime.close()
|
|
|
|
await asyncio.gather(*[_run_project(item) for item in self.config.projects])
|
|
finally:
|
|
await global_runtime.unload()
|
|
await global_runtime.close()
|
|
|
|
return results
|
|
|
|
|
|
class MultiProjectPipeline(ProjectBatchPipeline):
|
|
pass |