整体同步代码

2026-03-26 14:10:47 +08:00
parent 4b81b83fd8
commit f50dc1aed0
9 changed files with 188 additions and 32 deletions
@@ -51,6 +51,11 @@ class StrategyConfig(BaseModel):
    # 只影响最终一致性评估，不影响 create/update 的差异检测。
    post_check_ignore_fields: List[str] = Field(default_factory=list)

+    # post-check 一致性比较时，忽略列表型字段内各元素的特定子键。
+    # 格式: {字段名: [子键, ...]}，如 {"plan_node": ["is_audit"]}。
+    # 用于排除后端硬写死、永远与本地不一致的字段，避免误报。
+    post_check_ignore_list_item_fields: Dict[str, List[str]] = Field(default_factory=dict)
+
    # schema diff / validate 时忽略的顶层字段。
    compare_ignore_fields: List[str] = Field(default_factory=list)

@@ -382,11 +382,18 @@ class FullSyncPipeline:

    async def _evaluate_consistency_for_node_type(self, node_type: str) -> Dict[str, Any]:
        strategy = next((item for item in self.strategies if item.node_type == node_type), None)
+        if strategy is None:
+            raise RuntimeError(f"Missing strategy for node_type={node_type}")
        depend_fields = dict(strategy.config.depend_fields) if strategy and strategy.config.depend_fields else {}
        compare_to_remote = True
        if strategy and strategy.config.update_direction == UpdateDirection.PULL:
            compare_to_remote = False
-        ignore_fields = list(strategy.config.post_check_ignore_fields) if strategy else []
+        ignore_fields = list(
+            dict.fromkeys([
+                *strategy.config.compare_ignore_fields,
+                *strategy.config.post_check_ignore_fields,
+            ])
+        )

        result = await evaluate_consistency_for_node_type(
            node_type=node_type,
@@ -1,4 +1,5 @@
-from typing import TypeVar, Generic, Type, Dict, List, Optional, Any
+import copy
+from typing import TypeVar, Generic, Type, Dict, List, Optional, Any
 from pydantic import BaseModel
 import logging
 from pathlib import Path
@@ -187,28 +188,12 @@ class BaseSyncStrategy(Generic[T]):
        self._validate_config()
        logger.info(f"[{self.node_type}] Applied preset: {preset_name}")

-    def preprocess_compare_payload(self, payload: Dict[str, Any]) -> Dict[str, Any]:
-        return payload
-
-    def normalize_compare_payload(
-        self,
-        data: Optional[Dict[str, Any]],
-        data_id_map: Optional[Dict[str, str]] = None,
-        *,
-        ignore_fields: Optional[set[str]] = None,
-    ) -> Dict[str, Any]:
-        prepared = self.preprocess_compare_payload(dict(data or {}))
-        return normalized_data_for_compare(
-            prepared,
-            data_id_map,
-            ignore_fields=ignore_fields,
-        )
-
    def get_schema_diff_validator(self) -> SchemaDiffValidator:
        if self._schema_diff_validator is None:
            self._schema_diff_validator = SchemaDiffValidator(
                schema_name=self.schema.__name__,
                ignore_fields=self.config.compare_ignore_fields,
+                ignore_list_item_fields=self.config.post_check_ignore_list_item_fields,
            )
        return self._schema_diff_validator

@@ -222,6 +207,22 @@ class BaseSyncStrategy(Generic[T]):
        for line in validator.format_summary_lines():
            logger.info(f"[{self.node_type}] validate {line}")

+    def preprocess_compare_payload(self, payload: Dict[str, Any]) -> Dict[str, Any]:
+        return payload
+
+    def normalize_compare_payload(
+        self,
+        data: Optional[Dict[str, Any]],
+        data_id_map: Optional[Dict[str, str]] = None,
+    ) -> Dict[str, Any]:
+        normalized = normalized_data_for_compare(
+            data,
+            data_id_map,
+            ignore_fields=set(self.config.compare_ignore_fields),
+            ignore_list_item_fields=self.config.post_check_ignore_list_item_fields,
+        )
+        return self.preprocess_compare_payload(copy.deepcopy(normalized))
+
    async def bind(self, **kwargs):
        """进行绑定逻辑判定，设置 node.binding_status 和 node.action"""
        raise NotImplementedError
@@ -2,7 +2,7 @@ from __future__ import annotations

 import copy
 import inspect
-from typing import Any, Dict, Optional
+from typing import Any, Dict, Mapping, Optional


 async def build_data_id_normalization_map(
@@ -39,18 +39,42 @@ def is_id_like_field(field_name: Optional[str]) -> bool:
    return lowered.endswith("_id") or lowered.endswith("_ids")


+def _strip_ignored_list_item_fields(
+    field_name: Optional[str],
+    value: Any,
+    ignore_list_item_fields: Mapping[str, list[str]],
+) -> Any:
+    if not isinstance(value, list):
+        return value
+
+    ignored_keys = set(ignore_list_item_fields.get(field_name or "", []))
+    if not ignored_keys:
+        return value
+
+    normalized_items = []
+    for item in value:
+        if isinstance(item, dict):
+            normalized_items.append({key: nested for key, nested in item.items() if key not in ignored_keys})
+        else:
+            normalized_items.append(item)
+    return normalized_items
+
+
 def normalize_compare_value(
    field_name: Optional[str],
    value: Any,
    data_id_map: Dict[str, str],
+    ignore_list_item_fields: Optional[Mapping[str, list[str]]] = None,
 ) -> Any:
+    ignored_list_fields = ignore_list_item_fields or {}
    if isinstance(value, dict):
        return {
-            key: normalize_compare_value(key, nested, data_id_map)
+            key: normalize_compare_value(key, nested, data_id_map, ignored_list_fields)
            for key, nested in value.items()
        }
    if isinstance(value, list):
-        return [normalize_compare_value(field_name, item, data_id_map) for item in value]
+        value = _strip_ignored_list_item_fields(field_name, value, ignored_list_fields)
+        return [normalize_compare_value(field_name, item, data_id_map, ignored_list_fields) for item in value]

    if value == "":
        value = None
@@ -64,6 +88,7 @@ def normalized_data_for_compare(
    data: Optional[Dict[str, Any]],
    data_id_map: Optional[Dict[str, str]] = None,
    ignore_fields: Optional[set[str]] = None,
+    ignore_list_item_fields: Optional[Mapping[str, list[str]]] = None,
 ) -> Dict[str, Any]:
    payload = copy.deepcopy(data or {})
    payload.pop("id", None)
@@ -71,7 +96,7 @@ def normalized_data_for_compare(
    ignored = ignore_fields or set()
    mapping = data_id_map or {}
    return {
-        key: normalize_compare_value(key, value, mapping)
+        key: normalize_compare_value(key, value, mapping, ignore_list_item_fields)
        for key, value in payload.items()
        if key not in ignored
    }
@@ -55,15 +55,17 @@ def default_needs_update(
        return False

    ignore_fields = set(strategy.config.compare_ignore_fields)
-    normalized_source = strategy.normalize_compare_payload(
+    normalized_source = normalized_data_for_compare(
        source_data,
        data_id_map,
        ignore_fields=ignore_fields,
+        ignore_list_item_fields=strategy.config.post_check_ignore_list_item_fields,
    )
-    normalized_target = strategy.normalize_compare_payload(
+    normalized_target = normalized_data_for_compare(
        target_data,
        data_id_map,
        ignore_fields=ignore_fields,
+        ignore_list_item_fields=strategy.config.post_check_ignore_list_item_fields,
    )
    diff_map = strategy.get_schema_diff_validator().compare_payloads(
        normalized_source,
@@ -85,26 +87,28 @@ def resolve_needs_update_check(strategy) -> Callable:


 def _collect_diff_descriptions(
-    strategy,
    source_data,
    target_data,
    *,
    data_id_map,
    ignore_fields: set[str],
+    ignore_list_item_fields: dict[str, list[str]],
    max_items: int = 8,
 ) -> List[str]:
    if source_data is None or target_data is None:
        return []

-    normalized_source = strategy.normalize_compare_payload(
+    normalized_source = normalized_data_for_compare(
        source_data,
        data_id_map,
        ignore_fields=ignore_fields,
+        ignore_list_item_fields=ignore_list_item_fields,
    )
-    normalized_target = strategy.normalize_compare_payload(
+    normalized_target = normalized_data_for_compare(
        target_data,
        data_id_map,
        ignore_fields=ignore_fields,
+        ignore_list_item_fields=ignore_list_item_fields,
    )
    diff_map = collect_payload_diffs(
        normalized_source,
@@ -202,11 +206,11 @@ async def add_update_action(
            has_diff = bool(should_update)
            if has_diff:
                diff_descriptions = _collect_diff_descriptions(
-                    strategy,
                    data,
                    target_node.get_data(),
                    data_id_map=data_id_map or {},
                    ignore_fields=set(strategy.config.compare_ignore_fields),
+                    ignore_list_item_fields=dict(strategy.config.post_check_ignore_list_item_fields),
                )

        if steps_changed:
@@ -318,6 +322,7 @@ async def run_update(strategy) -> List["SyncNode"]:
                    f" (Pull, field_direction_key={field_key})"
                )

+        strategy.emit_schema_diff_report()
        return updated_nodes

    # ── 无字段级覆盖：原有逻辑不变 ──
@@ -350,4 +355,5 @@ async def run_update(strategy) -> List["SyncNode"]:
            f"[{strategy.node_type}] update_direction=BIDIRECTIONAL requires custom implementation"
        )

+    strategy.emit_schema_diff_report()
    return updated_nodes
@@ -1,7 +1,7 @@
 from __future__ import annotations

 from dataclasses import dataclass, field
-from typing import Any, Iterable
+from typing import Any, Iterable, Mapping

 from ..sync_system.strategy_ops.compare_ops import normalized_data_for_compare

@@ -28,10 +28,12 @@ class SchemaDiffValidator:
        *,
        schema_name: str,
        ignore_fields: Iterable[str] | None = None,
+        ignore_list_item_fields: Mapping[str, list[str]] | None = None,
        sample_limit: int = 3,
    ) -> None:
        self.schema_name = schema_name
        self.ignore_fields = {str(field) for field in (ignore_fields or []) if str(field)}
+        self.ignore_list_item_fields = dict(ignore_list_item_fields or {})
        self.sample_limit = max(1, sample_limit)
        self.reset()

@@ -54,10 +56,12 @@ class SchemaDiffValidator:
        local_normalized = normalized_data_for_compare(
            dict(local_payload or {}),
            ignore_fields=self.ignore_fields,
+            ignore_list_item_fields=self.ignore_list_item_fields,
        )
        remote_normalized = normalized_data_for_compare(
            dict(remote_payload or {}),
            ignore_fields=self.ignore_fields,
+            ignore_list_item_fields=self.ignore_list_item_fields,
        )

        field_names = sorted(set(local_normalized) | set(remote_normalized))
@@ -8,6 +8,7 @@ import pytest
 from sync_state_machine.common.binding import BindingManager
 from sync_state_machine.common.collection import DataCollection
 from sync_state_machine.datasource.jsonl import JsonlDataSource
+import sync_state_machine.pipeline.full_sync_pipeline as full_sync_pipeline_module
 from sync_state_machine.pipeline.full_sync_pipeline import FullSyncPipeline


@@ -30,7 +31,12 @@ class _StubStrategy:
    def __init__(self, node_type: str):
        self.node_type = node_type
        self.skip_sync = False
-        self.config = SimpleNamespace()
+        self.config = SimpleNamespace(
+            depend_fields={},
+            update_direction=None,
+            compare_ignore_fields=[],
+            post_check_ignore_fields=[],
+        )
        self.sm_runtime = None

    async def bind(self):
@@ -143,4 +149,48 @@ async def test_phase_sync_skips_reload_for_jsonl_datasources(

    assert reload_calls == [
        ("project", "pre-strategy refresh"),
-    ]
+    ]
+
+
+@pytest.mark.asyncio
+async def test_evaluate_consistency_merges_compare_and_post_check_ignore_fields() -> None:
+    project_strategy = _StubStrategy("project")
+    project_strategy.config.compare_ignore_fields = ["updated_at", "code"]
+    project_strategy.config.post_check_ignore_fields = ["code", "audit_status"]
+
+    pipeline = FullSyncPipeline(
+        local_datasource=_FakeDataSource(),
+        remote_datasource=_FakeDataSource(),
+        strategies=[project_strategy],
+        persistence=_FakePersistence(),
+        local_collection=DataCollection("local"),
+        remote_collection=DataCollection("remote"),
+        binding_manager=BindingManager(_FakePersistence(), auto_persist=False),
+        node_types=["project"],
+        load_order=["project"],
+        sync_order=["project"],
+        enable_persistence=False,
+    )
+
+    captured: dict[str, object] = {}
+
+    async def fake_evaluate_consistency_for_node_type(**kwargs):
+        captured.update(kwargs)
+        return {
+            "ok": True,
+            "checked_pairs": 0,
+            "mismatch_count": 0,
+            "fields": [],
+            "schema_name": "project",
+            "ignore_fields": kwargs["ignore_fields"],
+        }
+
+    original = full_sync_pipeline_module.evaluate_consistency_for_node_type
+    full_sync_pipeline_module.evaluate_consistency_for_node_type = fake_evaluate_consistency_for_node_type
+    try:
+        result = await pipeline._evaluate_consistency_for_node_type("project")
+    finally:
+        full_sync_pipeline_module.evaluate_consistency_for_node_type = original
+
+    assert captured["ignore_fields"] == ["updated_at", "code", "audit_status"]
+    assert result["ignore_fields"] == ["updated_at", "code", "audit_status"]
@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from sync_state_machine.config.builder import _apply_strategy_overrides
+from sync_state_machine.config.strategy_config import StrategyConfig, StrategyRuntimeConfig
+
+
+def test_strategy_runtime_config_skip_sync_defaults_false() -> None:
+    runtime_config = StrategyRuntimeConfig(node_type="project")
+
+    assert runtime_config.skip_sync is False
+
+
+def test_apply_strategy_overrides_allows_skip_sync_override() -> None:
+    resolved = _apply_strategy_overrides(
+        node_types=["project"],
+        strategy_overrides={
+            "project": {
+                "skip_sync": True,
+            }
+        },
+    )
+
+    assert len(resolved) == 1
+    assert resolved[0].node_type == "project"
+    assert resolved[0].skip_sync is True
+
+
+def test_strategy_config_supports_both_post_check_ignore_fields() -> None:
+    config = StrategyConfig.model_validate(
+        {
+            "post_check_ignore_fields": ["code"],
+            "post_check_ignore_list_item_fields": {"plan_node": ["is_audit"]},
+        }
+    )
+
+    assert config.post_check_ignore_fields == ["code"]
+    assert config.post_check_ignore_list_item_fields == {"plan_node": ["is_audit"]}
@@ -1,5 +1,6 @@
 import pytest
 from unittest.mock import AsyncMock, patch, MagicMock
+from sync_state_machine.domain.company.sync_strategy import CompanySyncStrategy
 from sync_state_machine.sync_system.config import UpdateDirection
 from sync_state_machine.sync_system.strategy_ops.update_ops import run_update
 from sync_state_machine.domain.construction.sync_strategy import ConstructionTaskSyncStrategy
@@ -86,3 +87,23 @@ def test_construction_task_compare_payload_ignores_plan_node_is_audit():
    }

    assert strategy.normalize_compare_payload(local_payload) == strategy.normalize_compare_payload(remote_payload)
+
+
+def test_default_strategy_exposes_normalize_compare_payload():
+    strategy = CompanySyncStrategy(
+        "company",
+        MagicMock(),
+        MagicMock(),
+        MagicMock(),
+    )
+
+    local_payload = {
+        "id": "remote-company-id",
+        "code": "COMP-001",
+    }
+    remote_payload = {
+        "id": "local-company-id",
+        "code": "COMP-001",
+    }
+
+    assert strategy.normalize_compare_payload(local_payload) == strategy.normalize_compare_payload(remote_payload)