调整了pipeline执行顺序。现在按domain顺序读取数据源数据,防止前面domain的更新影响后面domain.
将target_project_ids替换为更通用的data_id_filter
This commit is contained in:
@@ -277,9 +277,6 @@ async def create_pipeline_from_config(
|
||||
project_root=_project_root(),
|
||||
)
|
||||
await _initialize_datasource(remote_ds, endpoint_name="remote")
|
||||
local_ds.set_target_project_ids([str(pid) for pid in config.local_datasource.target_project_ids if str(pid)])
|
||||
remote_ds.set_target_project_ids([str(pid) for pid in config.remote_datasource.target_project_ids if str(pid)])
|
||||
|
||||
_register_handlers(config, local_ds, remote_ds, all_types)
|
||||
|
||||
local_collection = DataCollection("local", persistence, auto_persist=False)
|
||||
@@ -434,8 +431,6 @@ async def create_jsonl_to_api_pipeline(
|
||||
api_secret: str,
|
||||
# 节点类型
|
||||
node_types: List[str],
|
||||
# 项目过滤
|
||||
target_project_ids: Optional[List[str]] = None,
|
||||
strategy_overrides: Optional[Dict[str, Dict[str, Any]]] = None,
|
||||
# Pipeline 行为
|
||||
enable_persistence: bool = True,
|
||||
@@ -446,8 +441,8 @@ async def create_jsonl_to_api_pipeline(
|
||||
poll_interval: float = 0.5,
|
||||
api_rate_limit_max_requests: int = 30,
|
||||
api_rate_limit_window_seconds: float = 10.0,
|
||||
# Handler 自定义配置(可选,作用于 API 端)
|
||||
handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
|
||||
local_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
|
||||
remote_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
|
||||
# Logger 配置
|
||||
initialize_logger: bool = True,
|
||||
logger_file_path: Optional[str] = None,
|
||||
@@ -459,7 +454,7 @@ async def create_jsonl_to_api_pipeline(
|
||||
type="jsonl",
|
||||
jsonl_dir=local_jsonl_dir,
|
||||
read_only=False,
|
||||
target_project_ids=target_project_ids or [],
|
||||
handler_configs=local_handler_configs or {},
|
||||
),
|
||||
remote_datasource=ApiDataSourceConfig(
|
||||
type="api",
|
||||
@@ -471,8 +466,7 @@ async def create_jsonl_to_api_pipeline(
|
||||
poll_interval=poll_interval,
|
||||
api_rate_limit_max_requests=api_rate_limit_max_requests,
|
||||
api_rate_limit_window_seconds=api_rate_limit_window_seconds,
|
||||
target_project_ids=target_project_ids or [],
|
||||
handler_configs=handler_configs or {},
|
||||
handler_configs=remote_handler_configs or {},
|
||||
),
|
||||
strategies=[],
|
||||
persist=PersistConfig(
|
||||
@@ -502,8 +496,6 @@ async def create_api_to_jsonl_pipeline(
|
||||
persistence_db: str,
|
||||
# 节点类型
|
||||
node_types: List[str],
|
||||
# 项目过滤
|
||||
target_project_ids: Optional[List[str]] = None,
|
||||
strategy_overrides: Optional[Dict[str, Dict[str, Any]]] = None,
|
||||
# Pipeline 行为
|
||||
enable_persistence: bool = True,
|
||||
@@ -514,8 +506,8 @@ async def create_api_to_jsonl_pipeline(
|
||||
poll_interval: float = 0.5,
|
||||
api_rate_limit_max_requests: int = 30,
|
||||
api_rate_limit_window_seconds: float = 10.0,
|
||||
# Handler 自定义配置(可选,作用于 API 端)
|
||||
handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
|
||||
local_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
|
||||
remote_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
|
||||
# Logger 配置
|
||||
initialize_logger: bool = True,
|
||||
logger_file_path: Optional[str] = None,
|
||||
@@ -533,14 +525,13 @@ async def create_api_to_jsonl_pipeline(
|
||||
poll_interval=poll_interval,
|
||||
api_rate_limit_max_requests=api_rate_limit_max_requests,
|
||||
api_rate_limit_window_seconds=api_rate_limit_window_seconds,
|
||||
target_project_ids=target_project_ids or [],
|
||||
handler_configs=handler_configs or {},
|
||||
handler_configs=local_handler_configs or {},
|
||||
),
|
||||
remote_datasource=JsonlDataSourceConfig(
|
||||
type="jsonl",
|
||||
jsonl_dir=remote_jsonl_dir,
|
||||
read_only=False,
|
||||
target_project_ids=target_project_ids or [],
|
||||
handler_configs=remote_handler_configs or {},
|
||||
),
|
||||
strategies=[],
|
||||
persist=PersistConfig(
|
||||
@@ -565,10 +556,11 @@ async def create_jsonl_to_jsonl_pipeline(
|
||||
remote_jsonl_dir: str,
|
||||
persistence_db: str,
|
||||
node_types: List[str],
|
||||
target_project_ids: Optional[List[str]] = None,
|
||||
enable_persistence: bool = True,
|
||||
wipe_persistence_on_start: bool = False,
|
||||
strategy_overrides: Optional[Dict[str, Dict[str, Any]]] = None,
|
||||
local_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
|
||||
remote_handler_configs: Optional[Dict[str, Dict[str, Any]]] = None,
|
||||
initialize_logger: bool = True,
|
||||
logger_file_path: Optional[str] = None,
|
||||
logger_level: int = logging.INFO,
|
||||
@@ -579,13 +571,13 @@ async def create_jsonl_to_jsonl_pipeline(
|
||||
type="jsonl",
|
||||
jsonl_dir=local_jsonl_dir,
|
||||
read_only=False,
|
||||
target_project_ids=target_project_ids or [],
|
||||
handler_configs=local_handler_configs or {},
|
||||
),
|
||||
remote_datasource=JsonlDataSourceConfig(
|
||||
type="jsonl",
|
||||
jsonl_dir=remote_jsonl_dir,
|
||||
read_only=False,
|
||||
target_project_ids=target_project_ids or [],
|
||||
handler_configs=remote_handler_configs or {},
|
||||
),
|
||||
strategies=[],
|
||||
persist=PersistConfig(
|
||||
|
||||
Reference in New Issue
Block a user