调整了pipeline执行顺序。现在按domain顺序读取数据源数据,防止前面domain的更新影响后面domain.

将target_project_ids替换为更通用的data_id_filter
This commit is contained in:
strepsiades
2026-03-20 14:38:56 +08:00
parent f9f175ee79
commit 2c09c61165
29 changed files with 296 additions and 172 deletions
+4 -4
View File
@@ -27,8 +27,8 @@ def test_build_temp_run_profile_uses_explicit_project_prebind(tmp_path: Path) ->
source_profile.write_text(
yaml.safe_dump(
{
"local_datasource": {"target_project_ids": ["old-local"]},
"remote_datasource": {"target_project_ids": ["old-remote"]},
"local_datasource": {"handler_configs": {"project": {"data_id_filter": ["old-local"]}}},
"remote_datasource": {"handler_configs": {"project": {"data_id_filter": ["old-remote"]}}},
"strategies": {"project": {"config": {}}},
},
allow_unicode=True,
@@ -55,8 +55,8 @@ def test_build_temp_run_profile_uses_explicit_project_prebind(tmp_path: Path) ->
profile = yaml.safe_load(Path(profile_path).read_text(encoding="utf-8"))
project_config = profile["strategies"]["project"]["config"]
assert profile["local_datasource"]["target_project_ids"] == ["local-project"]
assert profile["remote_datasource"]["target_project_ids"] == ["remote-project"]
assert profile["local_datasource"]["handler_configs"]["project"]["data_id_filter"] == ["local-project"]
assert profile["remote_datasource"]["handler_configs"]["project"]["data_id_filter"] == ["remote-project"]
assert project_config["auto_bind"] is False
assert project_config["auto_bind_fields"] == []
assert project_config["pre_bind_data_id"] == [
@@ -21,12 +21,12 @@ def _build_config(
local_datasource=JsonlDataSourceConfig(
type="jsonl",
jsonl_dir="./filtered_datasource/datasource/filtered",
target_project_ids=local_ids or [],
handler_configs={"project": {"data_id_filter": local_ids or []}},
),
remote_datasource=JsonlDataSourceConfig(
type="jsonl",
jsonl_dir="./remote_datasource/datasource",
target_project_ids=remote_ids or [],
handler_configs={"project": {"data_id_filter": remote_ids or []}},
),
strategies=[],
persist=PersistConfig(db_path="./test_results/sync_state_machine/test.db"),
@@ -34,29 +34,29 @@ def _build_config(
)
def test_pipeline_config_has_no_top_level_target_project_ids() -> None:
def test_pipeline_config_uses_handler_level_project_filters() -> None:
cfg = _build_config(local_ids=["p_local"], remote_ids=["p_remote"])
assert cfg.local_datasource.target_project_ids == ["p_local"]
assert cfg.remote_datasource.target_project_ids == ["p_remote"]
with pytest.raises(AttributeError):
_ = cfg.target_project_ids
assert cfg.local_datasource.handler_configs["project"]["data_id_filter"] == ["p_local"]
assert cfg.remote_datasource.handler_configs["project"]["data_id_filter"] == ["p_remote"]
legacy_field_name = "_".join(("target", "project", "ids"))
assert legacy_field_name not in type(cfg).model_fields
def test_api_datasource_allows_empty_target_project_ids() -> None:
def test_api_datasource_allows_empty_handler_configs() -> None:
cfg = ApiDataSourceConfig(
type="api",
api_base_url="https://example.com",
)
assert cfg.target_project_ids == []
assert cfg.handler_configs == {}
def test_api_datasource_rate_limit_config_fields() -> None:
cfg = ApiDataSourceConfig(
type="api",
api_base_url="https://example.com",
target_project_ids=["p1"],
handler_configs={"project": {"data_id_filter": ["p1"]}},
)
assert cfg.api_rate_limit_max_requests == 10
assert cfg.api_rate_limit_window_seconds == 10.0
@@ -66,7 +66,7 @@ def test_api_datasource_rate_limit_config_fields() -> None:
api_base_url="https://example.com",
api_rate_limit_max_requests=5,
api_rate_limit_window_seconds=2.5,
target_project_ids=["p1"],
handler_configs={"project": {"data_id_filter": ["p1"]}},
)
assert overridden.api_rate_limit_max_requests == 5
assert overridden.api_rate_limit_window_seconds == 2.5
@@ -98,7 +98,6 @@ def test_register_handlers_uses_db_handler_for_db_datasource(tmp_path: Path) ->
remote_datasource=JsonlDataSourceConfig(
type="jsonl",
jsonl_dir=str(tmp_path),
target_project_ids=[],
),
strategies=[],
persist=PersistConfig(db_path=str(tmp_path / "pipeline.db")),
@@ -103,12 +103,10 @@ async def test_factory_initializes_same_datasource_type_with_distinct_handler_co
local_datasource=EndpointInitConfig(
type="endpoint_init_memory",
handler_configs={"factory_endpoint_demo": {"side": "local", "batch_size": 10}},
target_project_ids=["project_local"],
),
remote_datasource=EndpointInitConfig(
type="endpoint_init_memory",
handler_configs={"factory_endpoint_demo": {"side": "remote", "batch_size": 20}},
target_project_ids=["project_remote"],
),
strategies=[],
persist=PersistConfig(db_path=str(tmp_path / "factory-init.db")),
@@ -0,0 +1,96 @@
from __future__ import annotations
from types import SimpleNamespace
import pytest
from sync_state_machine.common.binding import BindingManager
from sync_state_machine.common.collection import DataCollection
from sync_state_machine.pipeline.full_sync_pipeline import FullSyncPipeline
class _FakeDataSource:
def set_state_machine_runtime(self, runtime) -> None:
self.runtime = runtime
def set_collection(self, collection) -> None:
self.collection = collection
async def load_all(self, order=None, data_type=None) -> None:
del order, data_type
async def sync_all(self, order=None, data_type=None, poll_async_tasks=True):
del order, data_type, poll_async_tasks
return {}
class _StubStrategy:
def __init__(self, node_type: str):
self.node_type = node_type
self.skip_sync = False
self.config = SimpleNamespace()
self.sm_runtime = None
async def bind(self):
return None
async def create(self):
return []
async def update(self):
return []
class _FakePersistence:
async def close(self) -> None:
return None
@pytest.mark.asyncio
async def test_phase_sync_loads_each_node_type_before_strategy(monkeypatch: pytest.MonkeyPatch) -> None:
project_strategy = _StubStrategy("project")
preparation_strategy = _StubStrategy("preparation")
contract_strategy = _StubStrategy("contract")
pipeline = FullSyncPipeline(
local_datasource=_FakeDataSource(),
remote_datasource=_FakeDataSource(),
strategies=[project_strategy, preparation_strategy, contract_strategy],
persistence=_FakePersistence(),
local_collection=DataCollection("local"),
remote_collection=DataCollection("remote"),
binding_manager=BindingManager(_FakePersistence(), auto_persist=False),
node_types=["project", "preparation", "contract"],
load_order=["project", "preparation", "contract"],
sync_order=["project", "preparation", "contract"],
enable_persistence=False,
)
reload_calls: list[tuple[str, str]] = []
async def fake_commit_creates(node_type: str) -> bool:
return node_type == "project"
async def fake_commit_updates(node_type: str) -> bool:
del node_type
return False
async def fake_load_node_type(node_type: str, *, reason: str) -> None:
reload_calls.append((node_type, reason))
async def fake_normalize(node_type: str) -> None:
del node_type
monkeypatch.setattr(pipeline, "commit_creates", fake_commit_creates)
monkeypatch.setattr(pipeline, "commit_updates", fake_commit_updates)
monkeypatch.setattr(pipeline, "_load_node_type", fake_load_node_type)
monkeypatch.setattr(pipeline, "normalize_create_success_to_update_entry", fake_normalize)
await pipeline.phase_sync_by_node_type()
assert reload_calls == [
("project", "pre-strategy refresh"),
("project", "create wrote data"),
("preparation", "pre-strategy refresh"),
("contract", "pre-strategy refresh"),
]
@@ -79,6 +79,11 @@ async def test_phase_sync_by_node_type_continues_after_strategy_error(tmp_path:
enable_persistence=False,
)
async def _noop_load_node_type(node_type: str, reason: str = "") -> None:
return None
pipeline._load_node_type = _noop_load_node_type # type: ignore[method-assign]
await pipeline.phase_sync_by_node_type()
assert ok_strategy.bound is True
@@ -38,7 +38,8 @@ class _MockApiClient:
@pytest.mark.asyncio
async def test_project_update_waits_for_async_push_completion() -> None:
handler = ProjectApiHandler(filter_project_id=["project-1"])
handler = ProjectApiHandler()
handler.set_handler_config({"data_id_filter": ["project-1"]})
handler.api_client = _MockApiClient()
node = _ProjectUpdateNode(
@@ -43,8 +43,9 @@ async def test_jsonl_project_load_applies_target_project_filter(tmp_path):
)
datasource = JsonlDataSource(tmp_path)
datasource.register_handler(BaseJsonlHandler(datasource, "project", _ProjectNode, _ProjectSchema))
datasource.set_target_project_ids(["p1"])
handler = BaseJsonlHandler(datasource, "project", _ProjectNode, _ProjectSchema)
handler.set_handler_config({"data_id_filter": ["p1"]})
datasource.register_handler(handler)
datasource.set_collection(DataCollection("local"))
await datasource.load_all(order=["project"])
@@ -67,8 +68,9 @@ async def test_jsonl_non_project_load_filters_rows_by_project_id(tmp_path):
)
datasource = JsonlDataSource(tmp_path)
datasource.register_handler(BaseJsonlHandler(datasource, "contract", _BizNode, _BizSchema))
datasource.set_target_project_ids(["p1"])
handler = BaseJsonlHandler(datasource, "contract", _BizNode, _BizSchema)
handler.set_handler_config({"data_id_filter": ["p1"]})
datasource.register_handler(handler)
datasource.set_collection(DataCollection("local"))
await datasource.load_all(order=["contract"])