first commit
This commit is contained in:
strepsiades
@@ -0,0 +1,31 @@
|
||||
from .create_ops import add_create_action, run_create, finalize_peer_creating_sources
|
||||
from .update_ops import add_update_action, run_update
|
||||
from .delete_ops import run_delete
|
||||
from .bind_ops import (
|
||||
run_bind,
|
||||
phase_core_state,
|
||||
phase_dependency_check,
|
||||
check_dependencies_for_nodes,
|
||||
phase_auto_binding,
|
||||
check_dependency_satisfied,
|
||||
collect_dependency_errors,
|
||||
)
|
||||
from .reset_ops import get_phase1_reset_defaults, run_phase2_cleanup
|
||||
|
||||
__all__ = [
|
||||
"add_create_action",
|
||||
"run_create",
|
||||
"finalize_peer_creating_sources",
|
||||
"add_update_action",
|
||||
"run_update",
|
||||
"run_delete",
|
||||
"run_bind",
|
||||
"phase_core_state",
|
||||
"phase_dependency_check",
|
||||
"check_dependencies_for_nodes",
|
||||
"phase_auto_binding",
|
||||
"check_dependency_satisfied",
|
||||
"collect_dependency_errors",
|
||||
"get_phase1_reset_defaults",
|
||||
"run_phase2_cleanup",
|
||||
]
|
||||
@@ -0,0 +1,560 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple
|
||||
|
||||
from ...common.types import BindingStatus
|
||||
from ...sync_system.config import OrphanAction
|
||||
from ...sync_system.resolve_ids import IDResolver
|
||||
from ...sync_system.utils import extract_biz_key_dict, dict_to_biz_key_tuple, resolve_id_fields_in_key_dict
|
||||
from ...engine import e10_bind_core, decision_note, e15_bind_dependency, e16_bind_auto
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ...common.collection import DataCollection
|
||||
from ...common.sync_node import SyncNode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def run_bind(strategy, **kwargs):
|
||||
logger.info(f"[{strategy.node_type}] Starting Binding Pipeline...")
|
||||
|
||||
local_nodes: List["SyncNode"] = strategy.local_collection.filter_by_state_ids(
|
||||
node_type=strategy.node_type,
|
||||
state_ids=["S00"],
|
||||
)
|
||||
remote_nodes: List["SyncNode"] = strategy.remote_collection.filter_by_state_ids(
|
||||
node_type=strategy.node_type,
|
||||
state_ids=["S00"],
|
||||
)
|
||||
|
||||
await phase_core_state(strategy, local_nodes, remote_nodes)
|
||||
await phase_dependency_check(strategy, local_nodes, remote_nodes)
|
||||
await phase_auto_binding(strategy, local_nodes, remote_nodes)
|
||||
await refresh_bind_data_id(strategy)
|
||||
|
||||
logger.info(f"[{strategy.node_type}] Binding Pipeline Completed.")
|
||||
|
||||
|
||||
async def refresh_bind_data_id(strategy) -> None:
|
||||
binding_pairs = await strategy.binding_manager.get_all_bindings(strategy.node_type)
|
||||
local_to_remote = {local_id: remote_id for local_id, remote_id in binding_pairs if local_id}
|
||||
remote_to_local = {remote_id: local_id for local_id, remote_id in binding_pairs if local_id and remote_id}
|
||||
|
||||
for local_node in strategy.local_collection.filter(node_type=strategy.node_type):
|
||||
remote_node_id = local_to_remote.get(local_node.node_id)
|
||||
remote_node = strategy.remote_collection.get_by_node_id(remote_node_id) if remote_node_id else None
|
||||
if remote_node and remote_node.data_id:
|
||||
local_node.context["bind_data_id"] = remote_node.data_id
|
||||
else:
|
||||
local_node.context.pop("bind_data_id", None)
|
||||
|
||||
for remote_node in strategy.remote_collection.filter(node_type=strategy.node_type):
|
||||
local_node_id = remote_to_local.get(remote_node.node_id)
|
||||
local_node = strategy.local_collection.get_by_node_id(local_node_id) if local_node_id else None
|
||||
if local_node and local_node.data_id:
|
||||
remote_node.context["bind_data_id"] = local_node.data_id
|
||||
else:
|
||||
remote_node.context.pop("bind_data_id", None)
|
||||
|
||||
|
||||
async def phase_core_state(strategy, local_nodes: List["SyncNode"], remote_nodes: List["SyncNode"]):
|
||||
remote_map = {n.node_id: n for n in remote_nodes}
|
||||
processed_local_ids: Set[str] = set()
|
||||
processed_remote_ids: Set[str] = set()
|
||||
|
||||
for node in local_nodes:
|
||||
remote_id = await strategy.binding_manager.get_remote_id(strategy.node_type, node.node_id)
|
||||
|
||||
if remote_id:
|
||||
remote_node = remote_map.get(remote_id)
|
||||
|
||||
local_data = node.get_data()
|
||||
remote_data = remote_node.get_data() if remote_node else None
|
||||
local_valid = local_data is not None
|
||||
peer_valid = remote_data is not None
|
||||
local_decision = e10_bind_core(
|
||||
strategy.ensure_runtime(),
|
||||
node=node,
|
||||
has_binding_record=True,
|
||||
local_data=local_data,
|
||||
peer_data=remote_data,
|
||||
local_valid=local_valid,
|
||||
peer_valid=peer_valid,
|
||||
)
|
||||
local_reason = decision_note(local_decision)
|
||||
if local_decision.to_state in {"S03", "S04"}:
|
||||
logger.debug(f"[{strategy.node_type}] 核心绑定阻断: node={node.node_id}, reason={local_reason}")
|
||||
else:
|
||||
logger.debug(f"[{strategy.node_type}] 核心绑定完成: node={node.node_id}, reason={local_reason}")
|
||||
processed_local_ids.add(node.node_id)
|
||||
|
||||
if remote_node:
|
||||
peer_decision = e10_bind_core(
|
||||
strategy.ensure_runtime(),
|
||||
node=remote_node,
|
||||
has_binding_record=True,
|
||||
local_data=remote_data,
|
||||
peer_data=local_data,
|
||||
local_valid=peer_valid,
|
||||
peer_valid=local_valid,
|
||||
)
|
||||
peer_reason = decision_note(peer_decision)
|
||||
if peer_decision.to_state in {"S03", "S04"}:
|
||||
logger.debug(
|
||||
f"[{strategy.node_type}] 核心绑定阻断: node={remote_node.node_id}, reason={peer_reason}"
|
||||
)
|
||||
else:
|
||||
logger.debug(f"[{strategy.node_type}] 核心绑定完成: node={remote_node.node_id}, reason={peer_reason}")
|
||||
processed_remote_ids.add(remote_node.node_id)
|
||||
|
||||
for node in local_nodes:
|
||||
if node.node_id not in processed_local_ids:
|
||||
decision = e10_bind_core(
|
||||
strategy.ensure_runtime(),
|
||||
node=node,
|
||||
has_binding_record=False,
|
||||
)
|
||||
reason = decision_note(decision)
|
||||
logger.debug(f"[{strategy.node_type}] 核心绑定完成: node={node.node_id}, reason={reason}")
|
||||
|
||||
for r_node in remote_nodes:
|
||||
if r_node.node_id not in processed_remote_ids:
|
||||
decision = e10_bind_core(
|
||||
strategy.ensure_runtime(),
|
||||
node=r_node,
|
||||
has_binding_record=False,
|
||||
)
|
||||
reason = decision_note(decision)
|
||||
logger.debug(f"[{strategy.node_type}] 核心绑定完成: node={r_node.node_id}, reason={reason}")
|
||||
|
||||
|
||||
def check_dependency_satisfied(
|
||||
dependency_nodes: List[Optional["SyncNode"]],
|
||||
field_values: Optional[List[str]] = None,
|
||||
*,
|
||||
allow_empty_dependency_ref: bool = True,
|
||||
allow_missing_dep_data_id: bool = True,
|
||||
) -> bool:
|
||||
if not dependency_nodes:
|
||||
return True
|
||||
|
||||
values = field_values or []
|
||||
for index, dep_node in enumerate(dependency_nodes):
|
||||
dep_value = values[index] if index < len(values) else ""
|
||||
dep_value_str = "" if dep_value is None else str(dep_value)
|
||||
|
||||
if dep_node is None:
|
||||
if allow_empty_dependency_ref and dep_value_str == "":
|
||||
continue
|
||||
return False
|
||||
if dep_node.binding_status != BindingStatus.NORMAL:
|
||||
return False
|
||||
if not dep_node.data_id and not allow_missing_dep_data_id:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def collect_dependency_errors(
|
||||
dependency_nodes: List[Optional["SyncNode"]],
|
||||
field_names: List[str],
|
||||
field_types: List[str],
|
||||
field_values: List[str],
|
||||
*,
|
||||
allow_empty_dependency_ref: bool = True,
|
||||
allow_missing_dep_data_id: bool = True,
|
||||
) -> List[str]:
|
||||
errors: List[str] = []
|
||||
for index, dep_node in enumerate(dependency_nodes):
|
||||
field_name = field_names[index]
|
||||
field_type = field_types[index] if index < len(field_types) else "unknown"
|
||||
field_value = field_values[index] if index < len(field_values) else ""
|
||||
field_value_str = "" if field_value is None else str(field_value)
|
||||
|
||||
if dep_node is None:
|
||||
if allow_empty_dependency_ref and field_value_str == "":
|
||||
continue
|
||||
errors.append(f"not_found|{field_name}|{field_type}|{field_value_str}")
|
||||
continue
|
||||
|
||||
if dep_node.binding_status != BindingStatus.NORMAL:
|
||||
errors.append(f"bad_status|{field_name}|{dep_node.binding_status.value}")
|
||||
continue
|
||||
|
||||
if not dep_node.data_id and not allow_missing_dep_data_id:
|
||||
status_info = f"status={dep_node.status.value}" if dep_node.status else "status=未知"
|
||||
errors.append(f"no_data_id|{field_name}|{status_info}")
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
async def phase_dependency_check(strategy, local_nodes: List["SyncNode"], remote_nodes: List["SyncNode"]):
|
||||
await check_dependencies_for_nodes(strategy, local_nodes, strategy.local_collection)
|
||||
await check_dependencies_for_nodes(strategy, remote_nodes, strategy.remote_collection)
|
||||
|
||||
|
||||
async def check_dependencies_for_nodes(strategy, nodes: List["SyncNode"], collection: "DataCollection"):
|
||||
for node in nodes:
|
||||
if node.binding_status != BindingStatus.MISSING:
|
||||
continue
|
||||
|
||||
has_depend_fields = bool(strategy.config.depend_fields)
|
||||
node_data = node.get_data()
|
||||
data_present = bool(node_data)
|
||||
|
||||
depend_ids = []
|
||||
depend_nodes = []
|
||||
depend_field_names: List[str] = []
|
||||
depend_field_types: List[str] = []
|
||||
depend_field_values: List[str] = []
|
||||
semantic_errors: List[str] = []
|
||||
|
||||
if has_depend_fields and data_present:
|
||||
for field_name, dep_node_type in strategy.config.depend_fields.items():
|
||||
dep_data_id = node_data.get(field_name)
|
||||
if not dep_data_id:
|
||||
continue
|
||||
|
||||
dep_node = collection.get_by_data_id(dep_node_type, dep_data_id)
|
||||
if dep_node is None:
|
||||
maybe_node_id_ref = collection.get_by_node_id(str(dep_data_id))
|
||||
if maybe_node_id_ref is not None and maybe_node_id_ref.node_type == dep_node_type:
|
||||
semantic_errors.append(f"依赖字段 {field_name} 使用了 node_id={dep_data_id}，期望 data_id 引用")
|
||||
|
||||
depend_nodes.append(dep_node)
|
||||
depend_field_names.append(field_name)
|
||||
depend_field_types.append(dep_node_type)
|
||||
depend_field_values.append(str(dep_data_id))
|
||||
|
||||
if dep_node:
|
||||
depend_ids.append(dep_node.node_id)
|
||||
|
||||
node.depend_ids = depend_ids
|
||||
|
||||
allow_empty_dependency_ref = True
|
||||
allow_missing_dep_data_id = True
|
||||
|
||||
dep_satisfied = (
|
||||
check_dependency_satisfied(
|
||||
depend_nodes,
|
||||
depend_field_values,
|
||||
allow_empty_dependency_ref=allow_empty_dependency_ref,
|
||||
allow_missing_dep_data_id=allow_missing_dep_data_id,
|
||||
)
|
||||
if has_depend_fields and data_present
|
||||
else True
|
||||
)
|
||||
dependency_errors = (
|
||||
collect_dependency_errors(
|
||||
depend_nodes,
|
||||
depend_field_names,
|
||||
depend_field_types,
|
||||
depend_field_values,
|
||||
allow_empty_dependency_ref=allow_empty_dependency_ref,
|
||||
allow_missing_dep_data_id=allow_missing_dep_data_id,
|
||||
)
|
||||
if has_depend_fields and data_present
|
||||
else []
|
||||
)
|
||||
|
||||
decision = e15_bind_dependency(
|
||||
strategy.ensure_runtime(),
|
||||
node=node,
|
||||
data_present=data_present,
|
||||
has_depend_fields=has_depend_fields,
|
||||
dep_satisfied=dep_satisfied,
|
||||
dep_errors=dependency_errors,
|
||||
semantic_errors=semantic_errors,
|
||||
)
|
||||
reason = decision_note(decision)
|
||||
if decision.to_state in {"S03", "S04", "S05"}:
|
||||
detail_reason = node.error or reason
|
||||
logger.debug(f"[{strategy.node_type}] 依赖检查阻断: node={node.node_id}, reason={detail_reason}")
|
||||
|
||||
|
||||
async def phase_auto_binding(strategy, local_nodes: List["SyncNode"], remote_nodes: List["SyncNode"]):
|
||||
pending_auto_bind_updates: List[Dict[str, Any]] = []
|
||||
prebound_node_ids: Set[str] = set()
|
||||
|
||||
local_s02_ids = {
|
||||
n.node_id
|
||||
for n in strategy.local_collection.filter_by_state_ids(
|
||||
node_type=strategy.node_type,
|
||||
state_ids=["S02"],
|
||||
)
|
||||
}
|
||||
remote_s02_ids = {
|
||||
n.node_id
|
||||
for n in strategy.remote_collection.filter_by_state_ids(
|
||||
node_type=strategy.node_type,
|
||||
state_ids=["S02"],
|
||||
)
|
||||
}
|
||||
|
||||
local_create_enabled = strategy.config.local_orphan_action == OrphanAction.CREATE_REMOTE
|
||||
remote_create_enabled = strategy.config.remote_orphan_action == OrphanAction.CREATE_LOCAL
|
||||
|
||||
for prebind_pair in strategy.config.pre_bind_data_id:
|
||||
local_node = strategy.local_collection.get_by_data_id(strategy.node_type, prebind_pair.local_data_id)
|
||||
remote_node = strategy.remote_collection.get_by_data_id(strategy.node_type, prebind_pair.remote_data_id)
|
||||
|
||||
if local_node is None or remote_node is None:
|
||||
continue
|
||||
if local_node.node_id not in local_s02_ids or remote_node.node_id not in remote_s02_ids:
|
||||
continue
|
||||
if local_node.binding_status != BindingStatus.MISSING or remote_node.binding_status != BindingStatus.MISSING:
|
||||
continue
|
||||
|
||||
existing_remote = await strategy.binding_manager.get_remote_id(strategy.node_type, local_node.node_id)
|
||||
existing_local = await strategy.binding_manager.get_local_id(strategy.node_type, remote_node.node_id)
|
||||
if existing_remote or existing_local:
|
||||
continue
|
||||
|
||||
pending_auto_bind_updates.append(
|
||||
{
|
||||
"node": local_node,
|
||||
"outcome": "ONE_TO_ONE",
|
||||
"create_enabled": None,
|
||||
"bind_remote_node_id": remote_node.node_id,
|
||||
}
|
||||
)
|
||||
pending_auto_bind_updates.append(
|
||||
{
|
||||
"node": remote_node,
|
||||
"outcome": "ONE_TO_ONE",
|
||||
"create_enabled": None,
|
||||
"bind_remote_node_id": None,
|
||||
}
|
||||
)
|
||||
prebound_node_ids.add(local_node.node_id)
|
||||
prebound_node_ids.add(remote_node.node_id)
|
||||
|
||||
if not strategy.config.auto_bind or not strategy.config.auto_bind_fields:
|
||||
for node in local_nodes:
|
||||
if node.node_id in prebound_node_ids:
|
||||
continue
|
||||
if node.binding_status == BindingStatus.MISSING:
|
||||
pending_auto_bind_updates.append({"node": node, "outcome": None, "create_enabled": None})
|
||||
for node in remote_nodes:
|
||||
if node.node_id in prebound_node_ids:
|
||||
continue
|
||||
if node.binding_status == BindingStatus.MISSING:
|
||||
pending_auto_bind_updates.append({"node": node, "outcome": None, "create_enabled": None})
|
||||
else:
|
||||
id_resolver = IDResolver(strategy.local_collection, strategy.remote_collection, strategy.binding_manager)
|
||||
|
||||
local_key_map: Dict[Tuple, List["SyncNode"]] = {}
|
||||
remote_key_map: Dict[Tuple, List["SyncNode"]] = {}
|
||||
|
||||
for node in local_nodes:
|
||||
if node.node_id in prebound_node_ids:
|
||||
continue
|
||||
if node.node_id not in local_s02_ids:
|
||||
continue
|
||||
node_data = node.get_data()
|
||||
if not node_data:
|
||||
if node.binding_status == BindingStatus.MISSING:
|
||||
pending_auto_bind_updates.append(
|
||||
{
|
||||
"node": node,
|
||||
"outcome": "DATA_MISSING",
|
||||
"create_enabled": None,
|
||||
"bind_remote_node_id": None,
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
missing_fields = [fn for fn in strategy.config.auto_bind_fields if fn not in node_data]
|
||||
if missing_fields:
|
||||
if node.binding_status == BindingStatus.MISSING:
|
||||
pending_auto_bind_updates.append(
|
||||
{"node": node, "outcome": "KEY_MISSING", "create_enabled": None, "bind_remote_node_id": None}
|
||||
)
|
||||
continue
|
||||
|
||||
key_dict = extract_biz_key_dict(node_data, strategy.config.auto_bind_fields)
|
||||
if not key_dict:
|
||||
continue
|
||||
|
||||
resolved_key_dict, resolve_error = await resolve_id_fields_in_key_dict(
|
||||
key_dict,
|
||||
id_resolver,
|
||||
strategy._get_id_field_hints(),
|
||||
)
|
||||
if not resolved_key_dict:
|
||||
if node.binding_status == BindingStatus.MISSING:
|
||||
pending_auto_bind_updates.append(
|
||||
{
|
||||
"node": node,
|
||||
"outcome": "KEY_ID_RESOLVE_FAIL",
|
||||
"create_enabled": None,
|
||||
"bind_remote_node_id": None,
|
||||
"detail": resolve_error,
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
key = dict_to_biz_key_tuple(resolved_key_dict, strategy.config.auto_bind_fields)
|
||||
local_key_map.setdefault(key, []).append(node)
|
||||
|
||||
for r_node in remote_nodes:
|
||||
if r_node.node_id in prebound_node_ids:
|
||||
continue
|
||||
if r_node.node_id not in remote_s02_ids:
|
||||
continue
|
||||
r_node_data = r_node.get_data()
|
||||
if not r_node_data:
|
||||
if r_node.binding_status == BindingStatus.MISSING:
|
||||
pending_auto_bind_updates.append(
|
||||
{
|
||||
"node": r_node,
|
||||
"outcome": "DATA_MISSING",
|
||||
"create_enabled": None,
|
||||
"bind_remote_node_id": None,
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
missing_fields = [fn for fn in strategy.config.auto_bind_fields if fn not in r_node_data]
|
||||
if missing_fields:
|
||||
if r_node.binding_status == BindingStatus.MISSING:
|
||||
pending_auto_bind_updates.append(
|
||||
{
|
||||
"node": r_node,
|
||||
"outcome": "KEY_MISSING",
|
||||
"create_enabled": None,
|
||||
"bind_remote_node_id": None,
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
key_dict = extract_biz_key_dict(r_node_data, strategy.config.auto_bind_fields)
|
||||
if key_dict:
|
||||
key = dict_to_biz_key_tuple(key_dict, strategy.config.auto_bind_fields)
|
||||
remote_key_map.setdefault(key, []).append(r_node)
|
||||
|
||||
all_keys = set(local_key_map.keys()) | set(remote_key_map.keys())
|
||||
|
||||
for biz_key in all_keys:
|
||||
local_candidates = local_key_map.get(biz_key, [])
|
||||
remote_candidates = remote_key_map.get(biz_key, [])
|
||||
|
||||
excluded_local = set()
|
||||
excluded_remote = set()
|
||||
|
||||
for l_node in local_candidates:
|
||||
if l_node.binding_status != BindingStatus.NORMAL:
|
||||
continue
|
||||
|
||||
remote_id = await strategy.binding_manager.get_remote_id(strategy.node_type, l_node.node_id)
|
||||
if not remote_id:
|
||||
continue
|
||||
|
||||
for r_node in remote_candidates:
|
||||
if r_node.node_id == remote_id and r_node.binding_status == BindingStatus.NORMAL:
|
||||
excluded_local.add(l_node.node_id)
|
||||
excluded_remote.add(r_node.node_id)
|
||||
break
|
||||
|
||||
l_remain = [n for n in local_candidates if n.node_id not in excluded_local]
|
||||
r_remain = [n for n in remote_candidates if n.node_id not in excluded_remote]
|
||||
|
||||
if len(l_remain) == 1 and len(r_remain) == 1:
|
||||
l_node = l_remain[0]
|
||||
r_node = r_remain[0]
|
||||
|
||||
if l_node.binding_status == BindingStatus.MISSING and r_node.binding_status == BindingStatus.MISSING:
|
||||
pending_auto_bind_updates.append(
|
||||
{
|
||||
"node": l_node,
|
||||
"outcome": "ONE_TO_ONE",
|
||||
"create_enabled": None,
|
||||
"bind_remote_node_id": r_node.node_id,
|
||||
}
|
||||
)
|
||||
pending_auto_bind_updates.append(
|
||||
{
|
||||
"node": r_node,
|
||||
"outcome": "ONE_TO_ONE",
|
||||
"create_enabled": None,
|
||||
"bind_remote_node_id": None,
|
||||
}
|
||||
)
|
||||
else:
|
||||
if len(l_remain) >= 1 and len(r_remain) == 0:
|
||||
for n in l_remain:
|
||||
if n.binding_status == BindingStatus.MISSING:
|
||||
pending_auto_bind_updates.append(
|
||||
{
|
||||
"node": n,
|
||||
"outcome": "ZERO",
|
||||
"create_enabled": local_create_enabled,
|
||||
"bind_remote_node_id": None,
|
||||
}
|
||||
)
|
||||
|
||||
if len(r_remain) >= 1 and len(l_remain) == 0:
|
||||
for n in r_remain:
|
||||
if n.binding_status == BindingStatus.MISSING:
|
||||
pending_auto_bind_updates.append(
|
||||
{
|
||||
"node": n,
|
||||
"outcome": "ZERO",
|
||||
"create_enabled": remote_create_enabled,
|
||||
"bind_remote_node_id": None,
|
||||
}
|
||||
)
|
||||
|
||||
if (len(l_remain) > 1 and len(r_remain) >= 1) or (len(l_remain) == 1 and len(r_remain) > 1):
|
||||
for n in l_remain:
|
||||
if n.binding_status == BindingStatus.MISSING:
|
||||
pending_auto_bind_updates.append(
|
||||
{
|
||||
"node": n,
|
||||
"outcome": "AMBIGUOUS",
|
||||
"create_enabled": None,
|
||||
"bind_remote_node_id": None,
|
||||
}
|
||||
)
|
||||
|
||||
if (len(r_remain) > 1 and len(l_remain) >= 1) or (len(r_remain) == 1 and len(l_remain) > 1):
|
||||
for n in r_remain:
|
||||
if n.binding_status == BindingStatus.MISSING:
|
||||
pending_auto_bind_updates.append(
|
||||
{
|
||||
"node": n,
|
||||
"outcome": "AMBIGUOUS",
|
||||
"create_enabled": None,
|
||||
"bind_remote_node_id": None,
|
||||
}
|
||||
)
|
||||
|
||||
for item in pending_auto_bind_updates:
|
||||
node = item["node"]
|
||||
auto_bind_outcome = item["outcome"]
|
||||
create_enabled = item["create_enabled"]
|
||||
bind_remote_node_id = item.get("bind_remote_node_id")
|
||||
detail = item.get("detail")
|
||||
auto_bind_enabled = auto_bind_outcome is not None
|
||||
decision = e16_bind_auto(
|
||||
strategy.ensure_runtime(),
|
||||
node=node,
|
||||
auto_bind_enabled=auto_bind_enabled,
|
||||
auto_bind_outcome=auto_bind_outcome,
|
||||
create_enabled=create_enabled,
|
||||
id_resolve_detail=detail,
|
||||
)
|
||||
reason = decision_note(decision)
|
||||
if decision.to_state == "S01" and auto_bind_outcome == "ONE_TO_ONE":
|
||||
if bind_remote_node_id:
|
||||
await strategy.binding_manager.bind(
|
||||
strategy.node_type,
|
||||
node.node_id,
|
||||
bind_remote_node_id,
|
||||
)
|
||||
logger.debug(f"[{strategy.node_type}] 自动绑定成功: node={node.node_id}")
|
||||
if decision.to_state in {"S04", "S05"}:
|
||||
if detail:
|
||||
reason = f"{reason} | detail={detail}"
|
||||
node.error = reason
|
||||
logger.debug(f"[{strategy.node_type}] 自动绑定阻断: node={node.node_id}, reason={reason}")
|
||||
@@ -0,0 +1,89 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import inspect
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
|
||||
async def build_data_id_normalization_map(
|
||||
*,
|
||||
node_type: str,
|
||||
local_collection,
|
||||
remote_collection,
|
||||
binding_manager,
|
||||
) -> Dict[str, str]:
|
||||
mapping: Dict[str, str] = {}
|
||||
records_or_awaitable = binding_manager.get_all_records(node_type)
|
||||
records = await records_or_awaitable if inspect.isawaitable(records_or_awaitable) else records_or_awaitable
|
||||
for record in records:
|
||||
if not record.local_id or not record.remote_id:
|
||||
continue
|
||||
local_node = local_collection.get_by_node_id(record.local_id)
|
||||
remote_node = remote_collection.get_by_node_id(record.remote_id)
|
||||
if local_node is None or remote_node is None:
|
||||
continue
|
||||
local_data_id = local_node.data_id
|
||||
remote_data_id = remote_node.data_id
|
||||
if not local_data_id:
|
||||
continue
|
||||
mapping[local_data_id] = local_data_id
|
||||
if remote_data_id:
|
||||
mapping[remote_data_id] = local_data_id
|
||||
return mapping
|
||||
|
||||
|
||||
def is_id_like_field(field_name: Optional[str]) -> bool:
|
||||
if not field_name:
|
||||
return False
|
||||
lowered = field_name.lower()
|
||||
return lowered.endswith("_id") or lowered.endswith("_ids")
|
||||
|
||||
|
||||
def normalize_compare_value(field_name: Optional[str], value: Any, data_id_map: Dict[str, str]) -> Any:
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
key: normalize_compare_value(key, nested, data_id_map)
|
||||
for key, nested in value.items()
|
||||
}
|
||||
if isinstance(value, list):
|
||||
return [normalize_compare_value(field_name, item, data_id_map) for item in value]
|
||||
|
||||
if value == "":
|
||||
value = None
|
||||
|
||||
if is_id_like_field(field_name) and isinstance(value, str):
|
||||
return data_id_map.get(value, value)
|
||||
return value
|
||||
|
||||
|
||||
def normalized_data_for_compare(
|
||||
data: Optional[Dict[str, Any]],
|
||||
data_id_map: Optional[Dict[str, str]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
payload = copy.deepcopy(data or {})
|
||||
payload.pop("id", None)
|
||||
payload.pop("_id", None)
|
||||
mapping = data_id_map or {}
|
||||
return {
|
||||
key: normalize_compare_value(key, value, mapping)
|
||||
for key, value in payload.items()
|
||||
}
|
||||
|
||||
|
||||
def collect_payload_diffs(
|
||||
source_payload: Dict[str, Any],
|
||||
target_payload: Dict[str, Any],
|
||||
*,
|
||||
max_items: int = 6,
|
||||
) -> Dict[str, Dict[str, Any]]:
|
||||
out: Dict[str, Dict[str, Any]] = {}
|
||||
keys = list(dict.fromkeys(list(source_payload.keys()) + list(target_payload.keys())))
|
||||
for key in keys:
|
||||
source_value = source_payload.get(key)
|
||||
target_value = target_payload.get(key)
|
||||
if source_value == target_value:
|
||||
continue
|
||||
out[key] = {"source": source_value, "target": target_value}
|
||||
if len(out) >= max_items:
|
||||
break
|
||||
return out
|
||||
@@ -0,0 +1,204 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, List
|
||||
|
||||
from ...common.registry import DomainRegistry
|
||||
from ...common.types import BindingStatus, SyncAction
|
||||
from ...sync_system.config import OrphanAction
|
||||
from ...sync_system.resolve_ids import IDResolver
|
||||
from ...engine import (
|
||||
decision_note,
|
||||
e20_create_prepare,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ...common.binding import BindingManager
|
||||
from ...common.collection import DataCollection
|
||||
from ...common.sync_node import SyncNode
|
||||
from ...engine import StateMachineRuntime
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def add_create_action(
|
||||
strategy,
|
||||
*,
|
||||
from_collection: "DataCollection",
|
||||
to_collection: "DataCollection",
|
||||
source_is_local: bool,
|
||||
) -> List["SyncNode"]:
|
||||
nodes_to_create = from_collection.filter_by_state_ids(
|
||||
node_type=strategy.node_type,
|
||||
state_ids=["S13"],
|
||||
)
|
||||
|
||||
if not nodes_to_create:
|
||||
return []
|
||||
|
||||
id_resolver = IDResolver(strategy.local_collection, strategy.remote_collection, strategy.binding_manager)
|
||||
created_nodes = []
|
||||
|
||||
for src_node in nodes_to_create:
|
||||
src_data = src_node.get_data()
|
||||
if not src_data:
|
||||
logger.debug(
|
||||
f"[{strategy.node_type}] {'本地' if source_is_local else '远程'}节点 {src_node.node_id} "
|
||||
"状态为 MISSING 但数据为空，跳过创建"
|
||||
)
|
||||
continue
|
||||
|
||||
resolved_data, _ = await id_resolver.resolve_and_validate(
|
||||
data=src_data,
|
||||
node_type=strategy.node_type,
|
||||
depend_fields=dict(strategy.config.depend_fields) if strategy.config.depend_fields else {},
|
||||
to_remote=source_is_local,
|
||||
)
|
||||
if resolved_data is None:
|
||||
commit = e20_create_prepare(
|
||||
strategy.ensure_runtime(),
|
||||
node=src_node,
|
||||
spawn_success=False,
|
||||
)
|
||||
if commit.to_state is not None:
|
||||
raise RuntimeError(f"[{strategy.node_type}] E20 fail expected no-transition, got: {commit.to_state}")
|
||||
src_node.error = "spawn_failed: 创建前依赖ID解析失败"
|
||||
logger.debug(f"[{strategy.node_type}] 创建副作用失败: node={src_node.node_id}, error=创建前依赖ID解析失败")
|
||||
continue
|
||||
|
||||
node_class = DomainRegistry.get_node_class(strategy.node_type)
|
||||
new_node = None
|
||||
try:
|
||||
new_node = node_class(
|
||||
node_id=to_collection.generate_node_id(),
|
||||
data_id="",
|
||||
data=resolved_data,
|
||||
depend_ids=list(src_node.depend_ids or []),
|
||||
binding_status=BindingStatus.NORMAL,
|
||||
action=SyncAction.CREATE,
|
||||
)
|
||||
await to_collection.add(new_node)
|
||||
|
||||
if source_is_local:
|
||||
local_id, remote_id = src_node.node_id, new_node.node_id
|
||||
else:
|
||||
local_id, remote_id = new_node.node_id, src_node.node_id
|
||||
|
||||
await strategy.binding_manager.bind(strategy.node_type, local_id, remote_id)
|
||||
created_nodes.append(new_node)
|
||||
new_node.append_log(f"准备创建: 源节点 {src_node.node_id} 复制数据")
|
||||
logger.debug(f"[{strategy.node_type}] 创建准备完成: source={src_node.node_id}, target={new_node.node_id}")
|
||||
except Exception as exc:
|
||||
if new_node is not None:
|
||||
try:
|
||||
await to_collection.delete(new_node.node_id)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
commit = e20_create_prepare(
|
||||
strategy.ensure_runtime(),
|
||||
node=src_node,
|
||||
spawn_success=False,
|
||||
)
|
||||
if commit.to_state is not None:
|
||||
raise RuntimeError(f"[{strategy.node_type}] E20 fail expected no-transition, got: {commit.to_state}")
|
||||
|
||||
src_node.error = f"spawn_failed: {exc}"
|
||||
logger.debug(f"[{strategy.node_type}] 创建副作用失败: node={src_node.node_id}, error={exc}")
|
||||
continue
|
||||
|
||||
return created_nodes
|
||||
|
||||
|
||||
async def run_create(strategy) -> List["SyncNode"]:
|
||||
created_nodes: List["SyncNode"] = []
|
||||
|
||||
if strategy.config.local_orphan_action == OrphanAction.CREATE_REMOTE:
|
||||
local_nodes = await add_create_action(
|
||||
strategy,
|
||||
from_collection=strategy.local_collection,
|
||||
to_collection=strategy.remote_collection,
|
||||
source_is_local=True,
|
||||
)
|
||||
created_nodes.extend(local_nodes)
|
||||
logger.info(f"[{strategy.node_type}] Created {len(local_nodes)} nodes in remote collection (Push)")
|
||||
elif strategy.config.local_orphan_action == OrphanAction.DELETE_LOCAL:
|
||||
raise NotImplementedError(
|
||||
f"[{strategy.node_type}] local_orphan_action=DELETE_LOCAL is not supported in run_create; "
|
||||
"use delete stage (delete_ops) explicitly."
|
||||
)
|
||||
|
||||
if strategy.config.remote_orphan_action == OrphanAction.CREATE_LOCAL:
|
||||
remote_nodes = await add_create_action(
|
||||
strategy,
|
||||
from_collection=strategy.remote_collection,
|
||||
to_collection=strategy.local_collection,
|
||||
source_is_local=False,
|
||||
)
|
||||
created_nodes.extend(remote_nodes)
|
||||
logger.info(f"[{strategy.node_type}] Created {len(remote_nodes)} nodes in local collection (Pull)")
|
||||
elif strategy.config.remote_orphan_action == OrphanAction.DELETE_REMOTE:
|
||||
raise NotImplementedError(
|
||||
f"[{strategy.node_type}] remote_orphan_action=DELETE_REMOTE is not supported in run_create; "
|
||||
"use delete stage (delete_ops) explicitly."
|
||||
)
|
||||
|
||||
return created_nodes
|
||||
|
||||
|
||||
async def finalize_peer_creating_sources(
|
||||
*,
|
||||
node_type: str,
|
||||
local_collection: "DataCollection",
|
||||
remote_collection: "DataCollection",
|
||||
binding_manager: "BindingManager",
|
||||
runtime: "StateMachineRuntime",
|
||||
) -> tuple[int, int]:
|
||||
finalized_success = 0
|
||||
finalized_failed = 0
|
||||
|
||||
async def _handle_target_node(*, target_node: "SyncNode", target_is_remote: bool) -> None:
|
||||
nonlocal finalized_success, finalized_failed
|
||||
|
||||
if target_is_remote:
|
||||
local_id = await binding_manager.get_local_id(node_type, target_node.node_id)
|
||||
if not local_id:
|
||||
return
|
||||
source_node = local_collection.get(local_id)
|
||||
else:
|
||||
remote_id = await binding_manager.get_remote_id(node_type, target_node.node_id)
|
||||
if not remote_id:
|
||||
return
|
||||
source_node = remote_collection.get(remote_id)
|
||||
|
||||
if source_node is None:
|
||||
return
|
||||
if source_node.binding_status != BindingStatus.PEER_CREATING:
|
||||
return
|
||||
|
||||
if target_node.status.value == "SUCCESS":
|
||||
commit = e20_create_prepare(runtime, node=source_node, spawn_success=True)
|
||||
if commit.to_state != "S01":
|
||||
raise RuntimeError(
|
||||
f"[{node_type}] E20 success expected S01, got {commit.to_state} for source={source_node.node_id}"
|
||||
)
|
||||
source_node.error = None
|
||||
finalized_success += 1
|
||||
return
|
||||
|
||||
if target_node.status.value == "FAILED":
|
||||
source_node.error = (
|
||||
f"peer_create_failed: peer_node={target_node.node_id}, detail={target_node.error or 'unknown'}"
|
||||
)
|
||||
source_node.append_log(
|
||||
f"对侧创建失败，保持 PEER_CREATING: peer={target_node.node_id}, detail={target_node.error or 'unknown'}"
|
||||
)
|
||||
finalized_failed += 1
|
||||
|
||||
for target_node in remote_collection.filter_by_state_ids(node_type=node_type, state_ids=["S11C", "S12C"]):
|
||||
await _handle_target_node(target_node=target_node, target_is_remote=True)
|
||||
|
||||
for target_node in local_collection.filter_by_state_ids(node_type=node_type, state_ids=["S11C", "S12C"]):
|
||||
await _handle_target_node(target_node=target_node, target_is_remote=False)
|
||||
|
||||
return finalized_success, finalized_failed
|
||||
@@ -0,0 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, List
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ...common.sync_node import SyncNode
|
||||
|
||||
|
||||
async def run_delete(strategy) -> List["SyncNode"]:
|
||||
raise NotImplementedError(f"[{strategy.node_type}] delete path is not implemented yet")
|
||||
@@ -0,0 +1,159 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from ...sync_system.resolve_ids import IDResolver
|
||||
from .compare_ops import (
|
||||
build_data_id_normalization_map,
|
||||
normalized_data_for_compare,
|
||||
collect_payload_diffs,
|
||||
)
|
||||
|
||||
|
||||
def _strip_list_item_fields(payload: Dict, ignore_map: Dict[str, List[str]]) -> Dict:
|
||||
"""从 payload 中列表型字段的每个元素里，剔除指定子键。
|
||||
|
||||
例如 ignore_map={"plan_node": ["is_audit"]} 会把 payload["plan_node"][*]["is_audit"] 全部去掉。
|
||||
payload 本身不会被修改（浅拷贝）。
|
||||
"""
|
||||
if not ignore_map:
|
||||
return payload
|
||||
result = dict(payload)
|
||||
for field_name, sub_keys in ignore_map.items():
|
||||
if field_name not in result:
|
||||
continue
|
||||
items = result[field_name]
|
||||
if not isinstance(items, list):
|
||||
continue
|
||||
stripped = []
|
||||
for item in items:
|
||||
if isinstance(item, dict):
|
||||
item = {k: v for k, v in item.items() if k not in sub_keys}
|
||||
stripped.append(item)
|
||||
result[field_name] = stripped
|
||||
return result
|
||||
|
||||
|
||||
def _compact_repr(value: Any, *, max_len: int = 80) -> str:
|
||||
text = repr(value)
|
||||
if len(text) <= max_len:
|
||||
return text
|
||||
return text[: max_len - 3] + "..."
|
||||
|
||||
|
||||
def _append_post_check_error(node, message: str) -> None:
|
||||
if not message:
|
||||
return
|
||||
if node.error:
|
||||
if message not in node.error:
|
||||
node.error = f"{node.error} | {message}"
|
||||
return
|
||||
node.error = message
|
||||
|
||||
|
||||
async def evaluate_consistency_for_node_type(
|
||||
*,
|
||||
node_type: str,
|
||||
local_collection,
|
||||
remote_collection,
|
||||
binding_manager,
|
||||
logger,
|
||||
depend_fields: Dict[str, str] | None = None,
|
||||
compare_to_remote: bool = True,
|
||||
ignore_list_item_fields: Dict[str, List[str]] | None = None,
|
||||
post_check_fields: List[str] | None = None,
|
||||
) -> Dict[str, Any]:
|
||||
records = await binding_manager.get_all_records(node_type)
|
||||
mismatches: List[str] = []
|
||||
checked_pairs = 0
|
||||
depend_fields = dict(depend_fields or {})
|
||||
id_resolver = IDResolver(local_collection, remote_collection, binding_manager)
|
||||
data_id_map = await build_data_id_normalization_map(
|
||||
node_type=node_type,
|
||||
local_collection=local_collection,
|
||||
remote_collection=remote_collection,
|
||||
binding_manager=binding_manager,
|
||||
)
|
||||
|
||||
for record in records:
|
||||
local_id = record.local_id
|
||||
remote_id = record.remote_id
|
||||
if not local_id or not remote_id:
|
||||
continue
|
||||
|
||||
local_node = local_collection.get_by_node_id(local_id)
|
||||
remote_node = remote_collection.get_by_node_id(remote_id)
|
||||
checked_pairs += 1
|
||||
|
||||
if local_node is None or remote_node is None:
|
||||
mismatches.append(
|
||||
f"pair_missing(local_node={bool(local_node)}, remote_node={bool(remote_node)}, local_id={local_id}, remote_id={remote_id})"
|
||||
)
|
||||
continue
|
||||
|
||||
local_data = local_node.get_data() or {}
|
||||
remote_data = remote_node.get_data() or {}
|
||||
|
||||
# 与 create/update 保持同一依赖ID解析函数：IDResolver.resolve_and_validate。
|
||||
# compare_to_remote=True -> local(data) 先转 remote 口径后比较
|
||||
# compare_to_remote=False -> remote(data) 先转 local 口径后比较
|
||||
if depend_fields:
|
||||
if compare_to_remote:
|
||||
resolved_local, _ = await id_resolver.resolve_and_validate(
|
||||
data=local_data,
|
||||
node_type=node_type,
|
||||
depend_fields=depend_fields,
|
||||
to_remote=True,
|
||||
)
|
||||
if resolved_local is not None:
|
||||
local_data = resolved_local
|
||||
else:
|
||||
resolved_remote, _ = await id_resolver.resolve_and_validate(
|
||||
data=remote_data,
|
||||
node_type=node_type,
|
||||
depend_fields=depend_fields,
|
||||
to_remote=False,
|
||||
)
|
||||
if resolved_remote is not None:
|
||||
remote_data = resolved_remote
|
||||
|
||||
local_payload = normalized_data_for_compare(local_data, data_id_map)
|
||||
remote_payload = normalized_data_for_compare(remote_data, data_id_map)
|
||||
if post_check_fields:
|
||||
local_payload = {k: v for k, v in local_payload.items() if k in post_check_fields}
|
||||
remote_payload = {k: v for k, v in remote_payload.items() if k in post_check_fields}
|
||||
if ignore_list_item_fields:
|
||||
local_payload = _strip_list_item_fields(local_payload, ignore_list_item_fields)
|
||||
remote_payload = _strip_list_item_fields(remote_payload, ignore_list_item_fields)
|
||||
if local_payload != remote_payload:
|
||||
diff_map = collect_payload_diffs(local_payload, remote_payload)
|
||||
diff_parts = []
|
||||
for field_name, values in diff_map.items():
|
||||
local_text = _compact_repr(values.get("source"))
|
||||
remote_text = _compact_repr(values.get("target"))
|
||||
diff_parts.append(f"{field_name}{{local:{local_text},remote:{remote_text}}}")
|
||||
detail = "; ".join(diff_parts) if diff_parts else "payload differs"
|
||||
|
||||
mismatch_text = f"pair_diff(local_id={local_id}, remote_id={remote_id}, {detail})"
|
||||
mismatches.append(mismatch_text)
|
||||
_append_post_check_error(local_node, f"post_check diff: {detail}")
|
||||
_append_post_check_error(remote_node, f"post_check diff: {detail}")
|
||||
|
||||
result = {
|
||||
"ok": len(mismatches) == 0,
|
||||
"checked_pairs": checked_pairs,
|
||||
"mismatch_count": len(mismatches),
|
||||
"samples": mismatches[:10],
|
||||
}
|
||||
|
||||
if result["ok"]:
|
||||
logger.info(f"✅ Consistency check passed: node_type={node_type}, checked_pairs={checked_pairs}")
|
||||
else:
|
||||
logger.error(
|
||||
f"❌ Consistency check failed: node_type={node_type}, checked_pairs={checked_pairs}, mismatches={len(mismatches)}"
|
||||
)
|
||||
logger.error(" ↳ consistent语义: 比较绑定对(local_node.data vs remote_node.data)，非 original_data")
|
||||
for item in result["samples"]:
|
||||
logger.error(f" - {item}")
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,131 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional
|
||||
|
||||
from ...common.types import BindingStatus, SyncAction, SyncStatus
|
||||
from ...engine import e01_bootstrap
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ...common.binding import BindingManager
|
||||
from ...common.collection import DataCollection
|
||||
from ...engine.dispatcher import StateMachineRuntime
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_phase1_reset_defaults() -> Dict[str, Any]:
|
||||
return {
|
||||
"binding_status": BindingStatus.UNCHECKED,
|
||||
"error": None,
|
||||
}
|
||||
|
||||
|
||||
async def _resolve_local_id_for_unbind(
|
||||
*,
|
||||
node_type: str,
|
||||
any_side_node_id: str,
|
||||
binding_manager: "BindingManager",
|
||||
) -> Optional[str]:
|
||||
remote_id = await binding_manager.get_remote_id(node_type, any_side_node_id)
|
||||
if remote_id:
|
||||
return any_side_node_id
|
||||
local_id = await binding_manager.get_local_id(node_type, any_side_node_id)
|
||||
if local_id:
|
||||
return local_id
|
||||
return None
|
||||
|
||||
|
||||
def _is_create_zombie(node) -> bool:
|
||||
if node.action != SyncAction.CREATE:
|
||||
return False
|
||||
if node.status == SyncStatus.FAILED:
|
||||
return True
|
||||
return not bool(node.data_id)
|
||||
|
||||
|
||||
async def _cleanup_one_side(
|
||||
*,
|
||||
node_type: str,
|
||||
collection: "DataCollection",
|
||||
label: str,
|
||||
cleaned_ids: List[str],
|
||||
binding_manager: "BindingManager",
|
||||
runtime: "StateMachineRuntime",
|
||||
) -> None:
|
||||
nodes = collection.filter(node_type=node_type)
|
||||
for node in nodes:
|
||||
if collection.is_bootstrap_blocked(node):
|
||||
logger.debug(
|
||||
f"[{node_type}] Skip bootstrap-blocked node during E01 cleanup: node_id={node.node_id}, error={node.error}"
|
||||
)
|
||||
continue
|
||||
|
||||
decision = e01_bootstrap(runtime, node=node, is_create_zombie=_is_create_zombie(node))
|
||||
if decision.to_state != "S15":
|
||||
continue
|
||||
|
||||
local_id_for_unbind = await _resolve_local_id_for_unbind(
|
||||
node_type=node_type,
|
||||
any_side_node_id=node.node_id,
|
||||
binding_manager=binding_manager,
|
||||
)
|
||||
|
||||
if local_id_for_unbind:
|
||||
try:
|
||||
await binding_manager.unbind(node_type, local_id_for_unbind)
|
||||
bind_note = f"unbound(local_id={local_id_for_unbind})"
|
||||
except Exception as exc:
|
||||
bind_note = f"unbind_failed(local_id={local_id_for_unbind}, err={exc})"
|
||||
else:
|
||||
bind_note = "no_binding"
|
||||
|
||||
reason = "status=FAILED" if node.status == SyncStatus.FAILED else "data_id empty"
|
||||
logger.info(
|
||||
f"[{node_type}] Cleaning up CREATE zombie ({label}): "
|
||||
f"node_id={node.node_id}, reason={reason}, {bind_note}"
|
||||
)
|
||||
|
||||
await collection.delete(node.node_id)
|
||||
cleaned_ids.append(node.node_id)
|
||||
|
||||
|
||||
async def run_phase2_cleanup(
|
||||
*,
|
||||
node_types: List[str],
|
||||
local_collection: "DataCollection",
|
||||
remote_collection: "DataCollection",
|
||||
binding_manager: "BindingManager",
|
||||
runtime: "StateMachineRuntime",
|
||||
) -> int:
|
||||
total_cleaned = 0
|
||||
|
||||
for node_type in node_types:
|
||||
cleaned_local: List[str] = []
|
||||
cleaned_remote: List[str] = []
|
||||
await _cleanup_one_side(
|
||||
node_type=node_type,
|
||||
collection=local_collection,
|
||||
label="local",
|
||||
cleaned_ids=cleaned_local,
|
||||
binding_manager=binding_manager,
|
||||
runtime=runtime,
|
||||
)
|
||||
await _cleanup_one_side(
|
||||
node_type=node_type,
|
||||
collection=remote_collection,
|
||||
label="remote",
|
||||
cleaned_ids=cleaned_remote,
|
||||
binding_manager=binding_manager,
|
||||
runtime=runtime,
|
||||
)
|
||||
|
||||
cleaned_count = len(cleaned_local) + len(cleaned_remote)
|
||||
total_cleaned += cleaned_count
|
||||
if cleaned_count > 0:
|
||||
logger.warning(
|
||||
f"[{node_type}] Cleaned up CREATE-failed zombies: "
|
||||
f"local_deleted={len(cleaned_local)}, remote_deleted={len(cleaned_remote)}"
|
||||
)
|
||||
|
||||
return total_cleaned
|
||||
@@ -0,0 +1,302 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Callable, List, Optional
|
||||
|
||||
from ...common.types import SyncAction
|
||||
from ...sync_system.config import UpdateDirection
|
||||
from ...sync_system.resolve_ids import IDResolver
|
||||
from ...engine import decision_note, e30_update_prepare
|
||||
from .compare_ops import (
|
||||
build_data_id_normalization_map,
|
||||
normalized_data_for_compare,
|
||||
collect_payload_diffs,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ...common.collection import DataCollection
|
||||
from ...common.sync_node import SyncNode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _safe_repr(value, *, max_len: int = 120) -> str:
|
||||
text = repr(value)
|
||||
if len(text) <= max_len:
|
||||
return text
|
||||
return text[: max_len - 3] + "..."
|
||||
|
||||
|
||||
def _collect_diff_descriptions(source_data, target_data, *, max_items: int = 8) -> List[str]:
|
||||
if source_data is None or target_data is None:
|
||||
return []
|
||||
|
||||
descriptions: List[str] = []
|
||||
for key, source_value in source_data.items():
|
||||
if key in {"id", "_id"}:
|
||||
continue
|
||||
target_value = target_data.get(key)
|
||||
if source_value == target_value:
|
||||
continue
|
||||
descriptions.append(f"{key}: {_safe_repr(target_value)} -> {_safe_repr(source_value)}")
|
||||
if len(descriptions) >= max_items:
|
||||
break
|
||||
|
||||
return descriptions
|
||||
|
||||
|
||||
def default_needs_update(
|
||||
strategy,
|
||||
source_node,
|
||||
target_node,
|
||||
resolved_data=None,
|
||||
data_id_map=None,
|
||||
) -> bool:
|
||||
if resolved_data is None:
|
||||
raise ValueError(f"[{strategy.node_type}] resolved_data is required for differential sync.")
|
||||
|
||||
source_data = resolved_data
|
||||
target_data = target_node.get_data()
|
||||
if source_data is None or target_data is None:
|
||||
return False
|
||||
|
||||
normalized_source = normalized_data_for_compare(source_data, data_id_map)
|
||||
normalized_target = normalized_data_for_compare(target_data, data_id_map)
|
||||
diff_map = collect_payload_diffs(normalized_source, normalized_target, max_items=8)
|
||||
is_different = bool(diff_map)
|
||||
|
||||
if is_different and diff_map:
|
||||
previews = [f"{key}: {values.get('source')} != {values.get('target')}" for key, values in list(diff_map.items())[:5]]
|
||||
logger.debug(f"[{strategy.node_type}] Node diff detected: {', '.join(previews)}")
|
||||
|
||||
return is_different
|
||||
|
||||
|
||||
def resolve_needs_update_check(strategy) -> Callable:
|
||||
return strategy._needs_update
|
||||
|
||||
|
||||
def _collect_diff_descriptions(source_data, target_data, *, data_id_map, max_items: int = 8) -> List[str]:
|
||||
if source_data is None or target_data is None:
|
||||
return []
|
||||
|
||||
normalized_source = normalized_data_for_compare(source_data, data_id_map)
|
||||
normalized_target = normalized_data_for_compare(target_data, data_id_map)
|
||||
diff_map = collect_payload_diffs(normalized_source, normalized_target, max_items=max_items)
|
||||
|
||||
descriptions: List[str] = []
|
||||
for key, values in diff_map.items():
|
||||
target_value = values.get("target")
|
||||
source_value = values.get("source")
|
||||
descriptions.append(f"{key}: {_safe_repr(target_value)} -> {_safe_repr(source_value)}")
|
||||
return descriptions
|
||||
|
||||
|
||||
async def add_update_action(
|
||||
strategy,
|
||||
*,
|
||||
from_collection: "DataCollection",
|
||||
to_collection: "DataCollection",
|
||||
needs_update_check,
|
||||
source_is_local: bool,
|
||||
data_id_map: Optional[dict] = None,
|
||||
node_filter: Optional[Callable[["SyncNode"], bool]] = None,
|
||||
) -> List["SyncNode"]:
|
||||
nodes_to_update = from_collection.filter_by_state_ids(
|
||||
node_type=strategy.node_type,
|
||||
state_ids=["S01"],
|
||||
)
|
||||
if not nodes_to_update:
|
||||
return []
|
||||
|
||||
if node_filter is not None:
|
||||
nodes_to_update = [n for n in nodes_to_update if node_filter(n)]
|
||||
if not nodes_to_update:
|
||||
return []
|
||||
|
||||
id_resolver = IDResolver(strategy.local_collection, strategy.remote_collection, strategy.binding_manager)
|
||||
updated_nodes: List["SyncNode"] = []
|
||||
total_nodes = len(nodes_to_update)
|
||||
|
||||
for idx, src_node in enumerate(nodes_to_update, start=1):
|
||||
if idx % 100 == 0 or idx == total_nodes:
|
||||
logger.debug(f"[{strategy.node_type}] 更新检查 {idx}/{total_nodes}")
|
||||
|
||||
src_data = src_node.get_data()
|
||||
if not src_data:
|
||||
continue
|
||||
|
||||
if source_is_local:
|
||||
target_node_id = await strategy.binding_manager.get_remote_id(strategy.node_type, src_node.node_id)
|
||||
else:
|
||||
target_node_id = await strategy.binding_manager.get_local_id(strategy.node_type, src_node.node_id)
|
||||
if not target_node_id:
|
||||
continue
|
||||
|
||||
target_node = to_collection.get_by_node_id(target_node_id)
|
||||
if not target_node or target_node.action == SyncAction.CREATE:
|
||||
continue
|
||||
|
||||
target_has_data_id = bool(target_node.data_id)
|
||||
data = None
|
||||
resolve_error = None
|
||||
if target_has_data_id:
|
||||
data, resolve_error = await id_resolver.resolve_and_validate(
|
||||
data=src_data,
|
||||
node_type=strategy.node_type,
|
||||
depend_fields=dict(strategy.config.depend_fields) if strategy.config.depend_fields else {},
|
||||
to_remote=source_is_local,
|
||||
)
|
||||
|
||||
update_id_resolve_ok = data is not None
|
||||
has_diff = False
|
||||
diff_descriptions: List[str] = []
|
||||
if update_id_resolve_ok:
|
||||
try:
|
||||
should_update = needs_update_check(src_node, target_node, resolved_data=data, data_id_map=data_id_map)
|
||||
except TypeError:
|
||||
try:
|
||||
should_update = needs_update_check(src_node, target_node)
|
||||
except TypeError:
|
||||
should_update = needs_update_check(src_node)
|
||||
has_diff = bool(should_update)
|
||||
if has_diff:
|
||||
diff_descriptions = _collect_diff_descriptions(data, target_node.get_data(), data_id_map=data_id_map or {})
|
||||
|
||||
if target_has_data_id and update_id_resolve_ok and not has_diff:
|
||||
continue
|
||||
|
||||
decision = e30_update_prepare(
|
||||
strategy.ensure_runtime(),
|
||||
source_node=src_node,
|
||||
target_node=target_node,
|
||||
update_enabled=True,
|
||||
target_has_data_id=target_has_data_id,
|
||||
update_id_resolve_ok=update_id_resolve_ok,
|
||||
has_diff=has_diff,
|
||||
id_resolve_detail=resolve_error,
|
||||
)
|
||||
reason = decision_note(decision)
|
||||
if decision.to_state == "S08":
|
||||
target_node.error = reason
|
||||
logger.debug(f"[{strategy.node_type}] 更新准备阻断: target={target_node.node_id}, reason={reason}")
|
||||
continue
|
||||
if decision.to_state != "S07":
|
||||
raise RuntimeError(f"[{strategy.node_type}] unexpected update_prepare state: {decision.to_state}")
|
||||
if data is None:
|
||||
raise RuntimeError(f"[{strategy.node_type}] E30(S07) requires resolved update data")
|
||||
|
||||
target_node.set_data(data)
|
||||
target_node.error = None
|
||||
diff_text = "; ".join(diff_descriptions) if diff_descriptions else "(未生成差异明细)"
|
||||
target_node.append_log(f"检测到差异: 从节点 {src_node.node_id} 更新数据 | fields={diff_text}")
|
||||
logger.info(f"[{strategy.node_type}] 更新差异字段: target={target_node.node_id}, {diff_text}")
|
||||
logger.debug(f"[{strategy.node_type}] 更新准备完成: target={target_node.node_id}, reason={reason}")
|
||||
updated_nodes.append(target_node)
|
||||
|
||||
return updated_nodes
|
||||
|
||||
|
||||
async def run_update(strategy) -> List["SyncNode"]:
|
||||
updated_nodes: List["SyncNode"] = []
|
||||
needs_update_check = resolve_needs_update_check(strategy)
|
||||
data_id_map = await build_data_id_normalization_map(
|
||||
node_type=strategy.node_type,
|
||||
local_collection=strategy.local_collection,
|
||||
remote_collection=strategy.remote_collection,
|
||||
binding_manager=strategy.binding_manager,
|
||||
)
|
||||
|
||||
field_key = strategy.config.field_direction_key
|
||||
overrides: dict = strategy.config.field_direction_overrides
|
||||
|
||||
if field_key and overrides:
|
||||
# 按方向归组：对每种出现的方向各调一次 add_update_action，传入过滤器。
|
||||
# 覆盖表中未命中的节点沿用 update_direction 作为默认方向。
|
||||
default_direction = strategy.config.update_direction
|
||||
active_directions = set(overrides.values()) | {default_direction}
|
||||
|
||||
for direction in (UpdateDirection.PUSH, UpdateDirection.PULL):
|
||||
if direction not in active_directions:
|
||||
continue
|
||||
|
||||
captured_direction = direction # 避免闭包捕获循环变量
|
||||
captured_overrides = overrides
|
||||
captured_default = default_direction
|
||||
captured_field_key = field_key
|
||||
|
||||
def make_filter(d=captured_direction, fk=captured_field_key, ov=captured_overrides, dflt=captured_default):
|
||||
def _filter(node: "SyncNode") -> bool:
|
||||
node_data = node.get_data() or {}
|
||||
key_val = node_data.get(fk)
|
||||
effective = ov.get(key_val, dflt)
|
||||
return effective == d
|
||||
return _filter
|
||||
|
||||
node_filter = make_filter()
|
||||
|
||||
if direction == UpdateDirection.PUSH:
|
||||
nodes = await add_update_action(
|
||||
strategy,
|
||||
from_collection=strategy.local_collection,
|
||||
to_collection=strategy.remote_collection,
|
||||
needs_update_check=needs_update_check,
|
||||
source_is_local=True,
|
||||
data_id_map=data_id_map,
|
||||
node_filter=node_filter,
|
||||
)
|
||||
updated_nodes.extend(nodes)
|
||||
logger.info(
|
||||
f"[{strategy.node_type}] Updated {len(nodes)} nodes in remote collection"
|
||||
f" (Push, field_direction_key={field_key})"
|
||||
)
|
||||
|
||||
elif direction == UpdateDirection.PULL:
|
||||
nodes = await add_update_action(
|
||||
strategy,
|
||||
from_collection=strategy.remote_collection,
|
||||
to_collection=strategy.local_collection,
|
||||
needs_update_check=needs_update_check,
|
||||
source_is_local=False,
|
||||
data_id_map=data_id_map,
|
||||
node_filter=node_filter,
|
||||
)
|
||||
updated_nodes.extend(nodes)
|
||||
logger.info(
|
||||
f"[{strategy.node_type}] Updated {len(nodes)} nodes in local collection"
|
||||
f" (Pull, field_direction_key={field_key})"
|
||||
)
|
||||
|
||||
return updated_nodes
|
||||
|
||||
# ── 无字段级覆盖：原有逻辑不变 ──
|
||||
if strategy.config.update_direction == UpdateDirection.PUSH:
|
||||
nodes = await add_update_action(
|
||||
strategy,
|
||||
from_collection=strategy.local_collection,
|
||||
to_collection=strategy.remote_collection,
|
||||
needs_update_check=needs_update_check,
|
||||
source_is_local=True,
|
||||
data_id_map=data_id_map,
|
||||
)
|
||||
updated_nodes.extend(nodes)
|
||||
logger.info(f"[{strategy.node_type}] Updated {len(nodes)} nodes in remote collection (Push)")
|
||||
|
||||
elif strategy.config.update_direction == UpdateDirection.PULL:
|
||||
nodes = await add_update_action(
|
||||
strategy,
|
||||
from_collection=strategy.remote_collection,
|
||||
to_collection=strategy.local_collection,
|
||||
needs_update_check=needs_update_check,
|
||||
source_is_local=False,
|
||||
data_id_map=data_id_map,
|
||||
)
|
||||
updated_nodes.extend(nodes)
|
||||
logger.info(f"[{strategy.node_type}] Updated {len(nodes)} nodes in local collection (Pull)")
|
||||
|
||||
elif strategy.config.update_direction == UpdateDirection.BIDIRECTIONAL:
|
||||
raise NotImplementedError(
|
||||
f"[{strategy.node_type}] update_direction=BIDIRECTIONAL requires custom implementation"
|
||||
)
|
||||
|
||||
return updated_nodes
|
||||
Reference in New Issue
Block a user