strepsiades/ecm_sync_system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

清理代码

Browse Source
This commit is contained in:
strepsiades
2026-03-18 17:09:46 +08:00
parent 6ae54e3edd
commit 32fa374e86
20 changed files with 63 additions and 979 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sync_state_machine/__init__.py
+1 -3
View File
@@ -29,7 +29,7 @@ from .config import (
register_datasource_factory,
register_datasource_type,
)
from .datasource import ApiClient, ApiDataSource, BaseApiHandler, BaseDbHandler, BaseJsonlHandler, DbDataSource, JsonlDataSource
from .datasource import ApiClient, ApiDataSource, BaseApiHandler, BaseJsonlHandler, JsonlDataSource
from .pipeline import FullSyncPipeline, create_pipeline_from_config, run_pipeline_from_config
try:
@@ -64,10 +64,8 @@ __all__ = [
"register_datasource_type",
"ApiClient",
"ApiDataSource",
"DbDataSource",
"JsonlDataSource",
"BaseApiHandler",
"BaseDbHandler",
"BaseJsonlHandler",
"FullSyncPipeline",
"create_pipeline_from_config",
sync_state_machine/config/__init__.py
-2
View File
@@ -29,7 +29,6 @@ from .builder import (
build_config,
build_config_from_file,
apply_config_overrides,
apply_env_overrides,
config_snapshot,
)
@@ -63,6 +62,5 @@ __all__ = [
"build_config",
"build_config_from_file",
"apply_config_overrides",
"apply_env_overrides",
"config_snapshot",
]
sync_state_machine/config/builder.py
+1 -14
View File
@@ -107,7 +107,6 @@ def build_default_config(project_root: Path) -> PipelineRunConfig:
"read_only": False,
"handler_configs": {},
},
"target_project_ids": [],
"persist": {
"db_path": str(project_root / "test_results" / "sync_state_machine" / "pipeline_default.db"),
"enable": True,
@@ -129,29 +128,17 @@ def build_default_config(project_root: Path) -> PipelineRunConfig:
def apply_config_overrides(config: PipelineRunConfig, overrides: Dict[str, Any]) -> PipelineRunConfig:
merged = _deep_merge(config.model_dump(), overrides)
raw_strategies = merged.pop("strategies", None)
legacy_strategy_overrides = merged.pop("strategy_overrides", None)
resolved = PipelineRunConfig.model_validate(merged)
if raw_strategies is not None:
resolved.strategies = _apply_strategy_overrides(resolved.node_types, raw_strategies)
else:
resolved.strategies = _apply_strategy_overrides(resolved.node_types, legacy_strategy_overrides or {})
resolved.strategies = _apply_strategy_overrides(resolved.node_types, {})
return resolved
def apply_env_overrides(
config: PipelineRunConfig,
project_root: Path,
*,
dotenv_file: Optional[Union[str, Path]] = None,
) -> PipelineRunConfig:
raise NotImplementedError(
"Environment-based overrides are removed. Use config files via overrides_file instead."
)
def build_config(
*,
project_root: Path,
sync_state_machine/config/runtime_config.py
-2
View File
@@ -19,7 +19,6 @@ from .builder import (
build_config,
build_config_from_file,
apply_config_overrides,
apply_env_overrides,
config_snapshot,
)
@@ -40,6 +39,5 @@ __all__ = [
"build_config",
"build_config_from_file",
"apply_config_overrides",
"apply_env_overrides",
"config_snapshot",
]
sync_state_machine/datasource/__init__.py
-7
View File
@@ -25,9 +25,6 @@ from .task_result import TaskResult, TaskStatus
# API DataSource
from .api import ApiClient, ApiDataSource, BaseApiHandler, register as register_api_datasource
# DB base helpers
from .db import DbDataSource, BaseDbHandler
# JSONL DataSource
from .jsonl import JsonlDataSource, BaseJsonlHandler, register as register_jsonl_datasource
@@ -65,10 +62,6 @@ __all__ = [
"ApiDataSource",
"BaseApiHandler",
# DB base helpers
"DbDataSource",
"BaseDbHandler",
# JSONL DataSource
"JsonlDataSource",
"BaseJsonlHandler",
sync_state_machine/datasource/api/datasource.py
+5 -6
View File
@@ -12,7 +12,7 @@ import json
from typing import Optional
from pathlib import Path
from ..datasource import BaseDataSource
from ..handler import NodeHandler, ensure_handler_compat
from ..handler import NodeHandler
from ..task_result import TaskResult
from .client import ApiClient
@@ -72,16 +72,15 @@ class ApiDataSource(BaseDataSource):
Args:
handler: API Handler 实例
"""
compatible_handler = ensure_handler_compat(handler)
compatible_handler.set_api_client(self.client)
compatible_handler.set_poll_mode(self.poll_mode)
handler.set_api_client(self.client)
handler.set_poll_mode(self.poll_mode)
# 调用基类注册方法
super().register_handler(compatible_handler)
super().register_handler(handler)
# 如果 collection 已设置，确保新 handler 拿到引用
if self._collection is not None:
compatible_handler.set_collection(self._collection) # type: ignore[arg-type]
handler.set_collection(self._collection) # type: ignore[arg-type]
@staticmethod
def _fmt_trace_value(value, max_len: int = 600) -> str:
sync_state_machine/datasource/datasource.py
+2 -4
View File
@@ -31,7 +31,6 @@ if TYPE_CHECKING:
from ..common.types import SyncStatus
from ..common.type_safety import get_pydantic_model_fields
from .handler import ensure_handler_compat
from .task_result import TaskResult
from ..engine import (
StateMachineConfig,
@@ -379,9 +378,8 @@ class BaseDataSource(ABC):
示例：
datasource.register_handler(ContractNodeHandler(api_client))
"""
compatible_handler = ensure_handler_compat(handler)
self._handlers[compatible_handler.node_type] = compatible_handler
self._apply_target_project_ids_to_handler(compatible_handler)
self._handlers[handler.node_type] = handler
self._apply_target_project_ids_to_handler(handler)
def set_target_project_ids(self, target_project_ids: Optional[List[str]]) -> None:
"""设置项目过滤列表，并在注册阶段下发给所有 handler。"""
sync_state_machine/datasource/db/__init__.py
-8
View File
@@ -1,8 +0,0 @@
from __future__ import annotations
"""DB DataSource 基类模块。"""
from .datasource import DbDataSource
from .handler import BaseDbHandler
__all__ = ["DbDataSource", "BaseDbHandler"]
sync_state_machine/datasource/db/datasource.py
-22
View File
@@ -1,22 +0,0 @@
"""DbDataSource - 数据库数据源协调层。"""
from __future__ import annotations
from ..datasource import BaseDataSource
class DbDataSource(BaseDataSource):
"""基于数据库 Handler 的数据源实现。"""
async def close(self) -> None:
return None
def _on_node_loaded(self, handler, node) -> None:
node.append_log(
f"DB链路[load] node_type={node.node_type} data_id={node.data_id or '<empty>'}"
)
def _on_in_progress_result(self, handler, node, result) -> None:
node.append_log(
f"DB链路[submit] action={node.action.value} data_id={node.data_id or '<empty>'}"
)
sync_state_machine/datasource/db/handler.py
-68
View File
@@ -1,68 +0,0 @@
"""BaseDbHandler - DB Handler 基类。"""
from __future__ import annotations
from typing import Any, Dict, List, Optional, Set, TypeVar, TYPE_CHECKING
from pydantic import BaseModel
from ...common.type_safety import get_pydantic_model_fields
from ..handler import BaseNodeHandler
if TYPE_CHECKING:
from ...common.sync_node import SyncNode
from .datasource import DbDataSource
T = TypeVar("T", bound=BaseModel)
class BaseDbHandler(BaseNodeHandler[T]):
"""数据库 Handler 基类。"""
def __init__(
self,
datasource: "DbDataSource",
node_type: str,
node_class: type["SyncNode[T]"],
schema: type[T],
):
super().__init__(node_type, node_class, schema)
self.datasource = datasource
self.update_schemas: List[type] = []
self._target_project_ids: Optional[Set[str]] = None
def set_target_project_ids(self, target_project_ids: List[str]) -> None:
if target_project_ids:
self._target_project_ids = {str(item) for item in target_project_ids if str(item)}
else:
self._target_project_ids = None
def get_update_fields(self, *, exclude: frozenset = frozenset({"id"})) -> List[str]:
fields: set[str] = set()
for schema in self.update_schemas:
for field_name in get_pydantic_model_fields(schema):
if field_name not in exclude:
fields.add(field_name)
return sorted(fields)
def _create_basic_node(self, data: Dict[str, Any], depend_ids: Optional[List[str]] = None) -> "SyncNode[T]":
data_id = self.extract_id(data)
if self._collection is None:
raise RuntimeError("Collection not set. Call set_collection() before creating nodes.")
if data_id:
existing_node = self._collection.get_by_data_id(self.node_type, data_id)
if existing_node is not None:
existing_node.depend_ids = list(depend_ids or [])
existing_node.set_data(data.copy())
existing_node.set_origin_data(data.copy())
return existing_node
node_id = self._collection.generate_node_id()
return self.node_class(
node_id=node_id,
data_id=data_id,
data=data.copy(),
depend_ids=depend_ids or [],
origin_data=data.copy(),
)
sync_state_machine/datasource/handler.py
-81
View File
@@ -479,87 +479,6 @@ class BaseNodeHandler(ABC, Generic[T]):
return self.create_node(data)
class CompatibleNodeHandler(Generic[T]):
"""兼容旧式 Handler，实现扩展后的可选接口并委托给原对象。"""
def __init__(self, handler: NodeHandler[T]):
self._handler = handler
@property
def node_type(self) -> str:
return self._handler.node_type
@property
def node_class(self) -> Type["SyncNode[T]"]:
return self._handler.node_class
@property
def schema(self) -> Type[T]:
return self._handler.schema
def set_collection(self, collection: "DataCollection") -> None:
self._handler.set_collection(collection)
def set_handler_config(self, config: Dict[str, Any]) -> None:
method = getattr(self._handler, "set_handler_config", None)
if callable(method):
method(config)
def set_target_project_ids(self, target_project_ids: List[str]) -> None:
method = getattr(self._handler, "set_target_project_ids", None)
if callable(method):
method(target_project_ids)
def set_api_client(self, client: "ApiClient") -> None:
method = getattr(self._handler, "set_api_client", None)
if callable(method):
method(client)
def set_poll_mode(self, mode: str) -> None:
method = getattr(self._handler, "set_poll_mode", None)
if callable(method):
method(mode)
def get_update_fields(self, *, exclude: frozenset[str] = frozenset({"id"})) -> List[str]:
method = getattr(self._handler, "get_update_fields", None)
if callable(method):
return method(exclude=exclude)
return []
async def load(self) -> List["SyncNode"]:
return await self._handler.load()
async def create_all(self, nodes: List["SyncNode"]) -> List[TaskResult]:
return await self._handler.create_all(nodes)
async def update_all(self, nodes: List["SyncNode"]) -> List[TaskResult]:
return await self._handler.update_all(nodes)
async def delete_all(self, nodes: List["SyncNode"]) -> List[TaskResult]:
return await self._handler.delete_all(nodes)
async def poll_tasks(self, task_ids: List[str]) -> Dict[str, TaskResult]:
return await self._handler.poll_tasks(task_ids)
async def validate(self, data: Dict[str, Any]) -> ValidationResult:
return await self._handler.validate(data)
def extract_id(self, data: Dict[str, Any]) -> str:
return self._handler.extract_id(data)
def _create_node(self, data: Dict[str, Any]) -> "SyncNode[T]":
return self._handler._create_node(data)
def __getattr__(self, item: str) -> Any:
return getattr(self._handler, item)
def ensure_handler_compat(handler: NodeHandler[T]) -> NodeHandler[T]:
if isinstance(handler, BaseNodeHandler) or isinstance(handler, CompatibleNodeHandler):
return handler
return CompatibleNodeHandler(handler)
class NodeHandlerRegistry:
"""节点处理器注册表"""
sync_state_machine/pipeline/__init__.py
-2
View File
@@ -14,7 +14,6 @@ from ..config import (
build_default_config,
build_config,
apply_config_overrides,
apply_env_overrides,
config_snapshot,
)
from .factory import (
@@ -46,6 +45,5 @@ __all__ = [
"build_default_config",
"build_config",
"apply_config_overrides",
"apply_env_overrides",
"config_snapshot",
]