strepsiades/ecm_sync_system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

清理代码

Browse Source
This commit is contained in:
strepsiades
2026-03-18 17:09:46 +08:00
parent 6ae54e3edd
commit 32fa374e86
20 changed files with 63 additions and 979 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/test_datasource_type_registry.py
+10 -36
View File
@@ -18,7 +18,7 @@ from sync_state_machine.config import (
)
from sync_state_machine.datasource import BaseDataSource
from sync_state_machine.datasource import ensure_builtin_datasource_types_registered
from sync_state_machine.datasource.db import BaseDbHandler, DbDataSource
from sync_state_machine.datasource.handler import BaseNodeHandler
from sync_state_machine.pipeline.factory import create_pipeline_from_config
from sync_state_machine.sync_system.strategy import DefaultSyncStrategy
@@ -56,11 +56,12 @@ class MemoryDataSource(BaseDataSource):
self.namespace = namespace
class MemoryHandler(BaseDbHandler[MemorySchema]):
class MemoryHandler(BaseNodeHandler[MemorySchema]):
__test__ = False
def __init__(self, datasource: MemoryDataSource):
super().__init__(datasource, "memory_demo", MemoryNode, MemorySchema)
super().__init__("memory_demo", MemoryNode, MemorySchema)
self.datasource = datasource
def extract_id(self, data: dict[str, str]) -> str:
return str(data.get("id", ""))
@@ -75,11 +76,12 @@ class MemoryHandler(BaseDbHandler[MemorySchema]):
return []
class MemoryRemoteDbHandler(BaseDbHandler[MemorySchema]):
class MemoryRemoteDbHandler(BaseNodeHandler[MemorySchema]):
__test__ = False
def __init__(self, datasource: DbDataSource):
super().__init__(datasource, "memory_demo", MemoryNode, MemorySchema)
def __init__(self, datasource: BaseDataSource):
super().__init__("memory_demo", MemoryNode, MemorySchema)
self.datasource = datasource
def extract_id(self, data: dict[str, str]) -> str:
return str(data.get("id", ""))
@@ -152,36 +154,8 @@ async def test_create_pipeline_from_config_supports_registered_custom_datasource
await pipeline.close()
def test_build_config_supports_same_file_datasource_bootstrap(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
bootstrap_module = tmp_path / "memory_bootstrap_demo.py"
bootstrap_module.write_text(
"from pydantic import ConfigDict\n"
"from sync_state_machine.config import BaseDataSourceConfig, register_datasource_type\n"
"from sync_state_machine.datasource import BaseDataSource\n"
"\n"
"class BootstrapMemoryConfig(BaseDataSourceConfig):\n"
" model_config = ConfigDict(extra=\"forbid\", validate_assignment=True)\n"
" type: str = \"memory_bootstrap\"\n"
" namespace: str = \"demo\"\n"
"\n"
"class BootstrapMemoryDataSource(BaseDataSource):\n"
" def __init__(self, namespace: str):\n"
" super().__init__()\n"
" self.namespace = namespace\n"
"\n"
"def build_memory_datasource(config, endpoint_name):\n"
" return BootstrapMemoryDataSource(namespace=f\"{endpoint_name}:{config.namespace}\")\n"
"\n"
"def register():\n"
" register_datasource_type(\n"
" \"memory_bootstrap\",\n"
" config_model=BootstrapMemoryConfig,\n"
" factory=build_memory_datasource,\n"
" replace=True,\n"
" )\n",
encoding="utf-8",
)
monkeypatch.syspath_prepend(str(tmp_path))
def test_build_config_supports_same_file_datasource_bootstrap(tmp_path: Path) -> None:
register_datasource_type("memory_bootstrap", config_model=MemoryConfig, factory=_memory_factory, replace=True)
profile_path = tmp_path / "custom-memory.yaml"
profile_path.write_text(
tests/unit/test_factory_db_handler_registration.py
+8 -6
View File
@@ -14,7 +14,8 @@ from sync_state_machine.config import (
PipelineRunConfig,
register_datasource_type,
)
from sync_state_machine.datasource.db import BaseDbHandler, DbDataSource
from sync_state_machine.datasource import BaseDataSource
from sync_state_machine.datasource.handler import BaseNodeHandler
from sync_state_machine.datasource.jsonl import BaseJsonlHandler, JsonlDataSource
from sync_state_machine.pipeline.factory import _register_handlers
from sync_state_machine.sync_system.strategy import DefaultSyncStrategy
@@ -42,11 +43,12 @@ class TestDbStrategy(DefaultSyncStrategy[TestDbSchema]):
schema = TestDbSchema
class TestDbHandler(BaseDbHandler[TestDbSchema]):
class TestDbHandler(BaseNodeHandler[TestDbSchema]):
__test__ = False
def __init__(self, datasource: DbDataSource):
super().__init__(datasource, "test_db_handler", TestDbNode, TestDbSchema)
def __init__(self, datasource: BaseDataSource):
super().__init__("test_db_handler", TestDbNode, TestDbSchema)
self.datasource = datasource
def extract_id(self, data: dict[str, str]) -> str:
return str(data.get("id", ""))
@@ -72,7 +74,7 @@ def _register_test_db_datasource_type() -> None:
register_datasource_type(
"test_db",
config_model=TestRegisteredDbConfig,
factory=lambda config, endpoint_name: DbDataSource(),
factory=lambda config, endpoint_name: BaseDataSource(),
replace=True,
)
@@ -103,7 +105,7 @@ def test_register_handlers_uses_db_handler_for_db_datasource(tmp_path: Path) ->
logging=LoggingConfig(initialize=False),
)
local_ds = DbDataSource()
local_ds = BaseDataSource()
remote_ds = JsonlDataSource(tmp_path)
_register_handlers(config, local_ds, remote_ds, ["test_db_handler"])
tests/unit/test_factory_endpoint_initialization.py
+7 -5
View File
@@ -14,7 +14,8 @@ from sync_state_machine.config import (
PipelineRunConfig,
register_datasource_type,
)
from sync_state_machine.datasource.db import BaseDbHandler, DbDataSource
from sync_state_machine.datasource import BaseDataSource
from sync_state_machine.datasource.handler import BaseNodeHandler
from sync_state_machine.pipeline.factory import create_pipeline_from_config
from sync_state_machine.sync_system.strategy import DefaultSyncStrategy
@@ -41,11 +42,12 @@ class EndpointInitStrategy(DefaultSyncStrategy[EndpointInitSchema]):
schema = EndpointInitSchema
class EndpointInitDbHandler(BaseDbHandler[EndpointInitSchema]):
class EndpointInitDbHandler(BaseNodeHandler[EndpointInitSchema]):
__test__ = False
def __init__(self, datasource: DbDataSource):
super().__init__(datasource, "factory_endpoint_demo", EndpointInitNode, EndpointInitSchema)
def __init__(self, datasource: BaseDataSource):
super().__init__("factory_endpoint_demo", EndpointInitNode, EndpointInitSchema)
self.datasource = datasource
def extract_id(self, data: dict[str, str]) -> str:
return str(data.get("id", ""))
@@ -85,7 +87,7 @@ def _register_endpoint_init_datasource_type() -> None:
register_datasource_type(
"endpoint_init_db",
config_model=EndpointInitDbConfig,
factory=lambda config, endpoint_name: DbDataSource(),
factory=lambda config, endpoint_name: BaseDataSource(),
replace=True,
)
tests/unit/test_run_presets_yaml_loader.py
+29 -30
View File
@@ -4,8 +4,11 @@ from pathlib import Path
import pytest
from sync_state_machine.config import build_config_from_file
from pydantic import ConfigDict
from sync_state_machine.config import BaseDataSourceConfig, build_config_from_file, register_datasource_type
from sync_state_machine.config.run_presets import load_overrides_from_file
from sync_state_machine.datasource import BaseDataSource
def test_load_overrides_from_yaml_file(tmp_path: Path) -> None:
@@ -32,7 +35,7 @@ local_datasource:
assert str(tmp_path) in overrides["local_datasource"]["jsonl_dir"]
def test_load_overrides_executes_generic_extensions_bootstraps(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
def test_load_overrides_ignores_extensions_bootstraps(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
bootstrap_module = tmp_path / "profile_bootstrap_demo.py"
bootstrap_module.write_text(
"BOOTSTRAP_CALLED = False\n"
@@ -68,37 +71,33 @@ logging:
import profile_bootstrap_demo
assert profile_bootstrap_demo.BOOTSTRAP_CALLED is True
assert profile_bootstrap_demo.BOOTSTRAP_CALLED is False
def test_build_config_from_file_uses_registered_bootstraps(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
bootstrap_module = tmp_path / "config_from_file_bootstrap.py"
bootstrap_module.write_text(
"from pydantic import ConfigDict\n"
"from sync_state_machine.config import BaseDataSourceConfig, register_datasource_type\n"
"from sync_state_machine.datasource import BaseDataSource\n"
"\n"
"class FileBackedConfig(BaseDataSourceConfig):\n"
" model_config = ConfigDict(extra=\"forbid\", validate_assignment=True)\n"
" type: str = \"file_backed\"\n"
" namespace: str = \"demo\"\n"
"\n"
"class FileBackedDatasource(BaseDataSource):\n"
" pass\n"
"\n"
"def build_datasource(config, endpoint_name):\n"
" return FileBackedDatasource()\n"
"\n"
"def register():\n"
" register_datasource_type(\n"
" \"file_backed\",\n"
" config_model=FileBackedConfig,\n"
" factory=build_datasource,\n"
" replace=True,\n"
" )\n",
encoding="utf-8",
class FileBackedConfig(BaseDataSourceConfig):
model_config = ConfigDict(extra="forbid", validate_assignment=True)
type: str = "file_backed"
namespace: str = "demo"
class FileBackedDatasource(BaseDataSource):
def __init__(self, namespace: str):
super().__init__()
self.namespace = namespace
def _build_file_backed_datasource(config, endpoint_name):
return FileBackedDatasource(namespace=f"{endpoint_name}:{config.namespace}")
def test_build_config_from_file_uses_registered_datasource_type(tmp_path: Path) -> None:
register_datasource_type(
"file_backed",
config_model=FileBackedConfig,
factory=_build_file_backed_datasource,
replace=True,
)
monkeypatch.syspath_prepend(str(tmp_path))
profile = tmp_path / "profile.yaml"
profile.write_text(