调整了pipeline执行顺序。现在按domain顺序读取数据源数据,防止前面domain的更新影响后面domain.

将target_project_ids替换为更通用的data_id_filter
This commit is contained in:
strepsiades
2026-03-20 14:38:56 +08:00
parent f9f175ee79
commit 2c09c61165
29 changed files with 296 additions and 172 deletions
+7 -3
View File
@@ -69,9 +69,13 @@ class BaseApiHandler(NodeHandler[T]):
def set_handler_config(self, config: Dict[str, Any]) -> None:
self.handler_config = dict(config)
def set_target_project_ids(self, target_project_ids: List[str]) -> None:
"""默认忽略项目过滤列表,具体子类按需覆盖。"""
return
def get_data_id_filter(self) -> List[str]:
value = self.handler_config.get("data_id_filter", [])
if isinstance(value, str):
return [value] if value else []
if isinstance(value, list):
return [str(item) for item in value if str(item)]
return []
@property
def node_type(self) -> str:
@@ -74,7 +74,6 @@ class BaseDataSource(ABC):
def __init__(self, poll_max_retries: int = 1, poll_interval: float = 0.5):
self._handlers: Dict[str, "NodeHandler"] = {}
self._collection: Optional["DataCollection"] = None
self._target_project_ids: List[str] = []
# 按节点类型统计信息
# {"company": {"loaded": 10, "synced": 10, "success": 10, "failed": 0}, ...}
@@ -87,13 +86,6 @@ class BaseDataSource(ABC):
self._poll_interval = max(0, poll_interval)
self._sm_runtime: Optional[StateMachineRuntime] = None
def set_target_project_ids(self, target_project_ids: List[str]) -> None:
self._target_project_ids = [str(item) for item in target_project_ids if str(item)]
@property
def target_project_ids(self) -> List[str]:
return list(self._target_project_ids)
def _ensure_sm_runtime(self) -> StateMachineRuntime:
if self._sm_runtime is None:
cfg_path = Path(__file__).resolve().parents[1] / "config" / "node_state_machine.yaml"
@@ -379,17 +371,6 @@ class BaseDataSource(ABC):
datasource.register_handler(ContractNodeHandler(api_client))
"""
self._handlers[handler.node_type] = handler
self._apply_target_project_ids_to_handler(handler)
def set_target_project_ids(self, target_project_ids: Optional[List[str]]) -> None:
"""设置项目过滤列表,并在注册阶段下发给所有 handler。"""
normalized = [str(pid) for pid in (target_project_ids or []) if str(pid)]
self._target_project_ids = normalized
for handler in self._handlers.values():
self._apply_target_project_ids_to_handler(handler)
def _apply_target_project_ids_to_handler(self, handler: "NodeHandler") -> None:
handler.set_target_project_ids(self._target_project_ids)
def get_handler(self, node_type: str) -> "NodeHandler":
"""获取 Handler"""
+61 -8
View File
@@ -74,10 +74,6 @@ class NodeHandler(Protocol[T]):
"""注入 handler 级别配置。默认可忽略。"""
...
def set_target_project_ids(self, target_project_ids: List[str]) -> None:
"""注入项目过滤列表。默认可忽略。"""
...
def set_api_client(self, client: "ApiClient") -> None:
"""注入 API 客户端。默认可忽略。"""
...
@@ -360,14 +356,18 @@ class BaseNodeHandler(ABC, Generic[T]):
"""
self._collection = collection
def set_target_project_ids(self, target_project_ids: List[str]) -> None:
"""默认忽略项目过滤能力。"""
return
def set_handler_config(self, config: Dict[str, Any]) -> None:
"""默认保存 handler 级别配置,供子类按需读取。"""
self.handler_config = dict(config)
def get_data_id_filter(self) -> List[str]:
value = self.handler_config.get("data_id_filter", [])
if isinstance(value, str):
return [value] if value else []
if isinstance(value, list):
return [str(item) for item in value if str(item)]
return []
def set_api_client(self, client: "ApiClient") -> None:
"""默认忽略 API 客户端注入。"""
return
@@ -479,6 +479,59 @@ class BaseNodeHandler(ABC, Generic[T]):
return self.create_node(data)
class CompatibleNodeHandler(BaseNodeHandler[T]):
def __init__(self, handler: BaseNodeHandler[T]):
super().__init__(handler.node_type, handler.node_class, handler.schema)
self._handler = handler
def set_collection(self, collection: "DataCollection") -> None:
super().set_collection(collection)
self._handler.set_collection(collection)
def set_handler_config(self, config: Dict[str, Any]) -> None:
self._handler.set_handler_config(config)
self.handler_config = dict(self._handler.handler_config)
def set_api_client(self, client: "ApiClient") -> None:
self._handler.set_api_client(client)
def set_poll_mode(self, mode: str) -> None:
self._handler.set_poll_mode(mode)
def get_update_fields(self, *, exclude: frozenset[str] = frozenset({"id"})) -> List[str]:
return self._handler.get_update_fields(exclude=exclude)
async def load(self) -> List['SyncNode']:
return await self._handler.load()
async def create_all(
self,
nodes: List["SyncNode"]
) -> List[TaskResult]:
return await self._handler.create_all(nodes)
async def update_all(
self,
nodes: List["SyncNode"]
) -> List[TaskResult]:
return await self._handler.update_all(nodes)
async def delete_all(
self,
nodes: List["SyncNode"]
) -> List[TaskResult]:
return await self._handler.delete_all(nodes)
async def poll_tasks(self, task_ids: List[str]) -> Dict[str, TaskResult]:
return await self._handler.poll_tasks(task_ids)
async def validate(self, data: Dict[str, Any]) -> ValidationResult:
return await self._handler.validate(data)
def extract_id(self, data: Dict[str, Any]) -> str:
return self._handler.extract_id(data)
class NodeHandlerRegistry:
"""节点处理器注册表"""
@@ -50,22 +50,17 @@ class BaseJsonlHandler(BaseNodeHandler):
):
super().__init__(node_type, node_class, schema)
self.datasource = datasource
self._target_project_ids: List[str] = []
def set_target_project_ids(self, target_project_ids: List[str]) -> None:
"""注入项目过滤列表的本地副本。真正的过滤来源仍然是 datasource。"""
self._target_project_ids = [str(pid) for pid in target_project_ids if str(pid)]
def _should_skip_item_by_project_filter(self, item: Dict[str, Any], item_id: str) -> bool:
target_project_ids = set(self.datasource.target_project_ids)
if not target_project_ids:
data_id_filter = set(self.get_data_id_filter())
if not data_id_filter:
return False
if self.node_type == "project":
return item_id not in target_project_ids
return item_id not in data_id_filter
project_id = item.get("project_id")
if project_id:
return str(project_id) not in target_project_ids
return str(project_id) not in data_id_filter
return False
async def load(self) -> List['SyncNode']: