将persist和后端分离,方便替换为其他数据库

This commit is contained in:
strepsiades
2026-03-24 10:15:16 +08:00
parent 7d92b3f1fe
commit 03a22b0c1c
19 changed files with 665 additions and 471 deletions
+41
View File
@@ -28,6 +28,8 @@ remote_datasource:
jsonl_dir: ${PROJECT_ROOT}/remote_datasource/datasource
persist:
db_path: ${PROJECT_ROOT}/test_results/sync_state_machine/demo.db
backend: sqlite
backend_options: {}
strategies:
project:
config:
@@ -109,6 +111,45 @@ logging:
- 每个业务类型的行为策略(依赖、自动绑定、创建方向、更新方向、跳过等)。
- `persist`
- 持久化开关与数据库位置。
- `persist.backend`
- 持久化后端注册表里的名称,例如 `sqlite`
- backend 实现会自行解释 `persist` 的其余字段。
- `persist.backend_options`
- 后端私有配置,后端自己决定如何解释。
### 6. 持久化后端注册
如果你希望在 backend 初始化阶段替换持久化后端,可以先在启动代码里按名称注册,再通过配置选择,不需要改调用方。
#### 6.1 内置 SQLite
```yaml
persist:
backend: sqlite
db_path: ${PROJECT_ROOT}/test_results/sync_state_machine/demo.db
backend_options: {}
```
#### 6.2 注册自定义后端
```python
from sync_state_machine.common.persistence import register_persistence_backend
register_persistence_backend("mysql", lambda config: MySQLPersistenceBackend(config))
```
然后在配置里选择这个名字:
```yaml
persist:
backend: mysql
db_path: ${PROJECT_ROOT}/test_results/sync_state_machine/demo.db
backend_options:
url: mysql+aiomysql://user:pass@127.0.0.1/depm_sync?charset=utf8mb4
pool_size: 5
```
后端实现会收到整份 `PersistConfig`,自行决定是读取 `db_path``backend_options`,还是额外扩展别的字段。
- `logging`
- 运行日志输出位置与级别。
+11 -1
View File
@@ -1,6 +1,12 @@
from .types import SyncAction, SyncStatus, BindingStatus
from .sync_node import SyncNode
from .persistence import PersistenceBackend
from .persistence import (
PersistenceBackend,
create_persistence_backend,
get_registered_persistence_backends,
register_persistence_backend,
)
from .persistence_backends import SQLitePersistenceBackend
from .collection import DataCollection
from .binding import BindingManager
@@ -10,6 +16,10 @@ __all__ = [
"BindingStatus",
"SyncNode",
"PersistenceBackend",
"SQLitePersistenceBackend",
"create_persistence_backend",
"register_persistence_backend",
"get_registered_persistence_backends",
"DataCollection",
"BindingManager",
]
+103 -435
View File
@@ -1,21 +1,20 @@
import aiosqlite
import json
from __future__ import annotations
import os
import sqlite3
from typing import Optional, Dict, List, Any, TYPE_CHECKING
from abc import ABC, abstractmethod
from typing import Any, Callable, ClassVar, Dict, List, Optional, TYPE_CHECKING
from ..config.persist_config import PersistConfig
if TYPE_CHECKING:
from .sync_node import SyncNode
class PersistenceBackend:
"""
Persistence 负责同步系统运行时数据的持久化存储(aiosqlite 实现)。
主要作为 DataCollection 和 BindingManager 的后端存储。
"""
def __init__(self, db_path: str = ":memory:", backend: str = "sqlite"):
self.db_path = db_path
self.backend = backend
self.conn: Optional[aiosqlite.Connection] = None
class PersistenceBackend(ABC):
"""同步系统持久化契约。"""
backend_name: ClassVar[str] = "abstract"
@staticmethod
def _is_read_only_sql(sql: str) -> bool:
@@ -93,250 +92,27 @@ class PersistenceBackend:
]
return summary
async def initialize(self):
"""异步初始化数据库连接和表结构"""
if self.backend != "sqlite":
raise NotImplementedError(f"Persistence backend not supported yet: {self.backend}")
@abstractmethod
async def initialize(self) -> None:
raise NotImplementedError
# 确保目录存在
if self.db_path != ":memory:":
dir_path = os.path.dirname(self.db_path)
if dir_path:
os.makedirs(dir_path, exist_ok=True)
self.conn = await aiosqlite.connect(self.db_path)
self.conn.row_factory = aiosqlite.Row
await self._init_db()
@abstractmethod
async def save_node(self, collection_id: str, scope: str, node: "SyncNode") -> None:
raise NotImplementedError
async def _init_db(self):
# 1. 节点表: 仅用于持久化存储 DataCollection 中的 SyncNode
# 注意: depend_ids不持久化,每次从data实时计算
await self.conn.execute("""
CREATE TABLE IF NOT EXISTS nodes (
collection_id TEXT NOT NULL,
scope TEXT NOT NULL DEFAULT 'global',
node_id TEXT NOT NULL,
node_type TEXT NOT NULL,
data_id TEXT,
bind_data_id TEXT,
data TEXT,
origin_data TEXT,
action TEXT,
status TEXT,
binding_status TEXT,
error TEXT,
sync_log TEXT,
context TEXT,
PRIMARY KEY (collection_id, scope, node_id)
)
""")
await self._ensure_nodes_schema()
# 2. 绑定表: 存储本地到远程的映射关系
# 只保留 local_id / remote_id 两字段(remote_id 可空)
await self._ensure_bindings_schema()
await self.conn.commit()
@abstractmethod
async def save_nodes_bulk(self, collection_id: str, scope: str, nodes: List["SyncNode"]) -> None:
raise NotImplementedError
async def _ensure_nodes_schema(self) -> None:
cursor = await self.conn.execute("PRAGMA table_info(nodes)")
rows = await cursor.fetchall()
columns = [r[1] for r in rows] if rows else []
if "scope" not in columns:
await self.conn.execute("""
CREATE TABLE IF NOT EXISTS nodes_new (
collection_id TEXT NOT NULL,
scope TEXT NOT NULL DEFAULT 'global',
node_id TEXT NOT NULL,
node_type TEXT NOT NULL,
data_id TEXT,
bind_data_id TEXT,
data TEXT,
origin_data TEXT,
action TEXT,
status TEXT,
binding_status TEXT,
error TEXT,
sync_log TEXT,
context TEXT,
PRIMARY KEY (collection_id, scope, node_id)
)
""")
await self.conn.execute("""
INSERT INTO nodes_new (
collection_id, scope, node_id, node_type, data_id, bind_data_id,
data, origin_data, action, status, binding_status, error, sync_log, context
)
SELECT
collection_id, 'global', node_id, node_type, data_id, bind_data_id,
data, origin_data, action, status, binding_status, error, sync_log, context
FROM nodes
""")
await self.conn.execute("DROP TABLE nodes")
await self.conn.execute("ALTER TABLE nodes_new RENAME TO nodes")
cursor = await self.conn.execute("PRAGMA table_info(nodes)")
rows = await cursor.fetchall()
columns = [r[1] for r in rows] if rows else []
if "bind_data_id" not in columns:
await self.conn.execute("ALTER TABLE nodes ADD COLUMN bind_data_id TEXT")
await self.conn.execute(
"CREATE INDEX IF NOT EXISTS idx_nodes_collection_scope_type ON nodes (collection_id, scope, node_type)"
)
await self.conn.execute(
"CREATE INDEX IF NOT EXISTS idx_nodes_collection_scope_data_id ON nodes (collection_id, scope, data_id)"
)
@abstractmethod
async def delete_node(self, collection_id: str, scope: str, node_id: str) -> None:
raise NotImplementedError
async def _ensure_bindings_schema(self) -> None:
"""确保 bindings 表为 (node_type, local_id, remote_id) 结构。
若检测到旧表含 payload 列,则进行迁移。
"""
cursor = await self.conn.execute("PRAGMA table_info(bindings)")
rows = await cursor.fetchall()
columns = [r[1] for r in rows] if rows else []
if not columns:
await self.conn.execute("""
CREATE TABLE IF NOT EXISTS bindings (
scope TEXT NOT NULL DEFAULT 'global',
node_type TEXT NOT NULL,
local_id TEXT NOT NULL,
remote_id TEXT,
last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (scope, node_type, local_id)
)
""")
return
if "scope" not in columns or ("payload" in columns and "remote_id" not in columns):
# 迁移旧结构到 scope + remote_id 新结构
await self.conn.execute("""
CREATE TABLE IF NOT EXISTS bindings_new (
scope TEXT NOT NULL DEFAULT 'global',
node_type TEXT NOT NULL,
local_id TEXT NOT NULL,
remote_id TEXT,
last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (scope, node_type, local_id)
)
""")
if "payload" in columns and "remote_id" not in columns:
async with self.conn.execute("SELECT node_type, local_id, payload FROM bindings") as cur:
old_rows = await cur.fetchall()
for row in old_rows:
try:
payload_dict = json.loads(row[2]) if row[2] else {}
except json.JSONDecodeError:
payload_dict = {}
remote_id = payload_dict.get("remote_id")
await self.conn.execute(
"INSERT OR REPLACE INTO bindings_new (scope, node_type, local_id, remote_id) VALUES (?, ?, ?, ?)",
("global", row[0], row[1], remote_id),
)
else:
async with self.conn.execute("SELECT node_type, local_id, remote_id FROM bindings") as cur:
old_rows = await cur.fetchall()
for row in old_rows:
await self.conn.execute(
"INSERT OR REPLACE INTO bindings_new (scope, node_type, local_id, remote_id) VALUES (?, ?, ?, ?)",
("global", row[0], row[1], row[2]),
)
await self.conn.execute("DROP TABLE bindings")
await self.conn.execute("ALTER TABLE bindings_new RENAME TO bindings")
await self.conn.execute(
"CREATE INDEX IF NOT EXISTS idx_bindings_scope_type_remote ON bindings (scope, node_type, remote_id)"
)
# --- DataCollection 支持 ---
async def save_node(self, collection_id: str, scope: str, node: "SyncNode"):
"""保存或更新单个节点(depend_ids不持久化,运行时从data实时计算)"""
bind_data_id = None
if isinstance(node.context, dict):
value = node.context.get("bind_data_id")
if value is not None and value != "":
bind_data_id = str(value)
await self.conn.execute("""
INSERT OR REPLACE INTO nodes (
collection_id, scope, node_id, node_type, data_id, bind_data_id,
data, origin_data, action, status, binding_status, error, sync_log, context
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""", (
collection_id,
scope,
node.node_id,
node.node_type,
node.data_id,
bind_data_id,
json.dumps(node.data) if node.data else None,
json.dumps(node.origin_data) if node.origin_data else None,
node.action.value,
node.status.value,
node.binding_status.value,
node.error,
node.sync_log,
json.dumps(node.context) if node.context else None
))
await self.conn.commit()
async def save_nodes_bulk(self, collection_id: str, scope: str, nodes: List["SyncNode"]):
"""批量保存或更新节点 (单次提交,depend_ids不持久化)"""
if not nodes:
return
rows = []
for node in nodes:
bind_data_id = None
if isinstance(node.context, dict):
value = node.context.get("bind_data_id")
if value is not None and value != "":
bind_data_id = str(value)
rows.append((
collection_id,
scope,
node.node_id,
node.node_type,
node.data_id,
bind_data_id,
json.dumps(node.data) if node.data else None,
json.dumps(node.origin_data) if node.origin_data else None,
node.action.value,
node.status.value,
node.binding_status.value,
node.error,
node.sync_log,
json.dumps(node.context) if node.context else None
))
await self.conn.executemany("""
INSERT OR REPLACE INTO nodes (
collection_id, scope, node_id, node_type, data_id, bind_data_id,
data, origin_data, action, status, binding_status, error, sync_log, context
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""", rows)
await self.conn.commit()
async def delete_node(self, collection_id: str, scope: str, node_id: str):
"""删除单个节点"""
await self.conn.execute(
"DELETE FROM nodes WHERE collection_id = ? AND scope = ? AND node_id = ?",
(collection_id, scope, node_id),
)
await self.conn.commit()
async def delete_nodes_bulk(self, collection_id: str, scope: str, node_ids: List[str]):
"""批量删除节点 (单次提交)"""
if not node_ids:
return
rows = [(collection_id, scope, node_id) for node_id in node_ids]
await self.conn.executemany(
"DELETE FROM nodes WHERE collection_id = ? AND scope = ? AND node_id = ?",
rows,
)
await self.conn.commit()
@abstractmethod
async def delete_nodes_bulk(self, collection_id: str, scope: str, node_ids: List[str]) -> None:
raise NotImplementedError
@abstractmethod
async def load_nodes(
self,
collection_id: str,
@@ -344,153 +120,37 @@ class PersistenceBackend:
*,
exclude_node_types: Optional[List[str]] = None,
) -> List[Dict[str, Any]]:
"""从特定 collection 中加载所有节点(depend_ids初始化为空,稍后从data实时计算)"""
sql = "SELECT * FROM nodes WHERE collection_id = ? AND scope = ?"
params: List[Any] = [collection_id, scope]
filtered_types = [node_type for node_type in (exclude_node_types or []) if node_type]
if filtered_types:
placeholders = ", ".join("?" for _ in filtered_types)
sql += f" AND node_type NOT IN ({placeholders})"
params.extend(filtered_types)
raise NotImplementedError
async with self.conn.execute(sql, tuple(params)) as cursor:
rows = await cursor.fetchall()
results = []
for row in rows:
results.append({
"node_id": row["node_id"],
"node_type": row["node_type"],
"data_id": row["data_id"],
"bind_data_id": row["bind_data_id"],
"depend_ids": [], # 不从数据库加载,运行时从data实时计算
"data": json.loads(row["data"]) if row["data"] else None,
"origin_data": json.loads(row["origin_data"]) if row["origin_data"] else None,
"action": row["action"],
"status": row["status"],
"binding_status": row["binding_status"],
"error": row["error"],
"sync_log": row["sync_log"],
"context": json.loads(row["context"]) if row["context"] else {}
})
if results[-1]["bind_data_id"] and isinstance(results[-1]["context"], dict):
results[-1]["context"]["bind_data_id"] = results[-1]["bind_data_id"]
return results
@abstractmethod
async def save_binding(self, scope: str, node_type: str, local_id: str, payload_dict: Dict[str, Any]) -> None:
raise NotImplementedError
# --- BindingManager 支持 ---
@abstractmethod
async def save_bindings_bulk(self, scope: str, node_type: str, bindings: Dict[str, Optional[str]]) -> None:
raise NotImplementedError
async def save_binding(self, scope: str, node_type: str, local_id: str, payload_dict: Dict[str, Any]):
"""保存或更新绑定关系及其元数据"""
remote_id = payload_dict.get("remote_id") if payload_dict else None
await self.conn.execute("""
INSERT OR REPLACE INTO bindings (scope, node_type, local_id, remote_id)
VALUES (?, ?, ?, ?)
""", (scope, node_type, local_id, remote_id))
await self.conn.commit()
@abstractmethod
async def delete_binding(self, scope: str, node_type: str, local_id: str) -> None:
raise NotImplementedError
async def save_bindings_bulk(self, scope: str, node_type: str, bindings: Dict[str, Optional[str]]):
"""批量保存或更新绑定关系 (单次提交)"""
if not bindings:
return
rows = []
for local_id, remote_id in bindings.items():
rows.append((scope, node_type, local_id, remote_id))
await self.conn.executemany(
"INSERT OR REPLACE INTO bindings (scope, node_type, local_id, remote_id) VALUES (?, ?, ?, ?)",
rows,
)
await self.conn.commit()
async def delete_binding(self, scope: str, node_type: str, local_id: str):
"""删除绑定关系"""
await self.conn.execute(
"DELETE FROM bindings WHERE scope = ? AND node_type = ? AND local_id = ?",
(scope, node_type, local_id)
)
await self.conn.commit()
async def delete_bindings_bulk(self, scope: str, node_type: str, local_ids: List[str]):
"""批量删除绑定关系 (单次提交)"""
if not local_ids:
return
rows = [(scope, node_type, local_id) for local_id in local_ids]
await self.conn.executemany(
"DELETE FROM bindings WHERE scope = ? AND node_type = ? AND local_id = ?",
rows,
)
await self.conn.commit()
@abstractmethod
async def delete_bindings_bulk(self, scope: str, node_type: str, local_ids: List[str]) -> None:
raise NotImplementedError
@abstractmethod
async def load_bindings(self, scope: str, node_type: str) -> List[Dict[str, Any]]:
"""回写该类型下的所有绑定关系到内存"""
async with self.conn.execute(
"SELECT local_id, remote_id FROM bindings WHERE scope = ? AND node_type = ?",
(scope, node_type)
) as cursor:
rows = await cursor.fetchall()
return [
{"local_id": r["local_id"], "payload": {"remote_id": r["remote_id"]}}
for r in rows
]
raise NotImplementedError
@abstractmethod
async def load_bind_data_id_overrides(self, node_type: str, scope: str) -> Dict[str, Optional[str]]:
"""基于 bindings 表计算 node_id -> bind_data_id 覆盖映射。
raise NotImplementedError
规则:
- local 节点的 bind_data_id 取其绑定 remote 节点的 data_id
- remote 节点的 bind_data_id 取其绑定 local 节点的 data_id
"""
sql = """
SELECT b.local_id AS src_node_id, rn.data_id AS bind_data_id
FROM bindings b
LEFT JOIN nodes rn
ON rn.node_id = b.remote_id
AND rn.node_type = b.node_type
AND rn.scope = b.scope
WHERE b.node_type = ?
AND b.scope = ?
UNION ALL
SELECT b.remote_id AS src_node_id, ln.data_id AS bind_data_id
FROM bindings b
LEFT JOIN nodes ln
ON ln.node_id = b.local_id
AND ln.node_type = b.node_type
AND ln.scope = b.scope
WHERE b.node_type = ?
AND b.scope = ?
AND b.remote_id IS NOT NULL
"""
async with self.conn.execute(sql, (node_type, scope, node_type, scope)) as cursor:
rows = await cursor.fetchall()
result: Dict[str, Optional[str]] = {}
for row in rows:
src_node_id = row["src_node_id"]
if not src_node_id:
continue
result[str(src_node_id)] = row["bind_data_id"]
return result
async def delete_scope_node_types(
self,
*,
scope: str,
node_types: List[str],
) -> None:
filtered_types = [node_type for node_type in node_types if node_type]
if not filtered_types:
return
placeholders = ", ".join("?" for _ in filtered_types)
await self.conn.execute(
f"DELETE FROM nodes WHERE scope = ? AND node_type IN ({placeholders})",
(scope, *filtered_types),
)
await self.conn.execute(
f"DELETE FROM bindings WHERE scope = ? AND node_type IN ({placeholders})",
(scope, *filtered_types),
)
await self.conn.commit()
@abstractmethod
async def delete_scope_node_types(self, *, scope: str, node_types: List[str]) -> None:
raise NotImplementedError
@abstractmethod
async def copy_nodes_between_scopes(
self,
*,
@@ -499,25 +159,9 @@ class PersistenceBackend:
target_scope: str,
node_types: List[str],
) -> None:
if source_scope == target_scope or not node_types:
return
placeholders = ", ".join("?" for _ in node_types)
sql = f"""
INSERT OR REPLACE INTO nodes (
collection_id, scope, node_id, node_type, data_id, bind_data_id,
data, origin_data, action, status, binding_status, error, sync_log, context
)
SELECT
collection_id, ?, node_id, node_type, data_id, bind_data_id,
data, origin_data, action, status, binding_status, error, sync_log, context
FROM nodes
WHERE collection_id = ?
AND scope = ?
AND node_type IN ({placeholders})
"""
await self.conn.execute(sql, (target_scope, collection_id, source_scope, *node_types))
await self.conn.commit()
raise NotImplementedError
@abstractmethod
async def copy_bindings_between_scopes(
self,
*,
@@ -525,34 +169,58 @@ class PersistenceBackend:
target_scope: str,
node_types: List[str],
) -> None:
if source_scope == target_scope or not node_types:
return
placeholders = ", ".join("?" for _ in node_types)
sql = f"""
INSERT OR REPLACE INTO bindings (scope, node_type, local_id, remote_id)
SELECT ?, node_type, local_id, remote_id
FROM bindings
WHERE scope = ?
AND node_type IN ({placeholders})
"""
await self.conn.execute(sql, (target_scope, source_scope, *node_types))
await self.conn.commit()
raise NotImplementedError
async def dump_to_runtime(self, filename: str):
"""将当前状态转储到 _runtime 文件夹 (使用 VACUUM INTO)"""
runtime_path = os.path.join("_runtime", filename)
if os.path.exists(runtime_path):
os.remove(runtime_path)
# VACUUM INTO 是备份 SQLite 的推荐方式 (SQLite 3.27+)
await self.conn.execute("VACUUM INTO ?", (runtime_path,))
@abstractmethod
async def dump_to_runtime(self, filename: str) -> None:
raise NotImplementedError
async def close(self):
"""关闭数据库连接(包含异常处理)"""
if self.conn:
try:
await self.conn.close()
except Exception as e:
print(f"⚠️ Error closing persistence: {e}")
finally:
self.conn = None
@abstractmethod
async def close(self) -> None:
raise NotImplementedError
PersistenceBackendFactory = Callable[[PersistConfig], PersistenceBackend]
_PERSISTENCE_BACKEND_REGISTRY: dict[str, PersistenceBackendFactory] = {}
def register_persistence_backend(name: str, factory: PersistenceBackendFactory, *, override: bool = False) -> None:
backend_name = name.strip()
if not backend_name:
raise ValueError("backend name cannot be empty")
if not override and backend_name in _PERSISTENCE_BACKEND_REGISTRY:
raise ValueError(f"persistence backend already registered: {backend_name}")
_PERSISTENCE_BACKEND_REGISTRY[backend_name] = factory
def get_registered_persistence_backends() -> Dict[str, PersistenceBackendFactory]:
return dict(_PERSISTENCE_BACKEND_REGISTRY)
def create_persistence_backend(
persist_config: PersistConfig,
) -> PersistenceBackend:
backend_name = (persist_config.backend or "").strip()
if not backend_name:
raise ValueError("persist.backend cannot be empty")
if not _PERSISTENCE_BACKEND_REGISTRY:
register_persistence_backend(
"sqlite",
lambda config: SQLitePersistenceBackend(config),
)
factory = _PERSISTENCE_BACKEND_REGISTRY.get(backend_name)
if factory is None:
available = ", ".join(sorted(_PERSISTENCE_BACKEND_REGISTRY)) or "none"
raise NotImplementedError(f"Persistence backend not supported yet: {backend_name}. Available: {available}")
return factory(persist_config)
def _register_builtin_persistence_backends() -> None:
from .persistence_backends.sqlite_backend import SQLitePersistenceBackend
register_persistence_backend("sqlite", lambda config: SQLitePersistenceBackend(config), override=True)
_register_builtin_persistence_backends()
@@ -0,0 +1,3 @@
from .sqlite_backend import SQLitePersistenceBackend
__all__ = ["SQLitePersistenceBackend"]
@@ -0,0 +1,464 @@
from __future__ import annotations
import aiosqlite
import json
import os
from typing import Any, Dict, List, Optional, TYPE_CHECKING
from ...config.persist_config import PersistConfig
from ..persistence import PersistenceBackend
if TYPE_CHECKING:
from ..sync_node import SyncNode
class SQLitePersistenceBackend(PersistenceBackend):
"""SQLite implementation of the persistence contract."""
backend_name = "sqlite"
def __init__(self, persist_config: PersistConfig | str | None = None, **_: Any) -> None:
if isinstance(persist_config, PersistConfig):
self.persist_config = persist_config
self.db_path = persist_config.db_path
elif isinstance(persist_config, str):
self.persist_config = None
self.db_path = persist_config
else:
self.persist_config = None
self.db_path = ":memory:"
self.conn: Optional[aiosqlite.Connection] = None
async def initialize(self) -> None:
if self.db_path != ":memory:":
dir_path = os.path.dirname(self.db_path)
if dir_path:
os.makedirs(dir_path, exist_ok=True)
self.conn = await aiosqlite.connect(self.db_path)
self.conn.row_factory = aiosqlite.Row
await self._init_db()
async def _init_db(self) -> None:
assert self.conn is not None
await self.conn.execute("""
CREATE TABLE IF NOT EXISTS nodes (
collection_id TEXT NOT NULL,
scope TEXT NOT NULL DEFAULT 'global',
node_id TEXT NOT NULL,
node_type TEXT NOT NULL,
data_id TEXT,
bind_data_id TEXT,
data TEXT,
origin_data TEXT,
action TEXT,
status TEXT,
binding_status TEXT,
error TEXT,
sync_log TEXT,
context TEXT,
PRIMARY KEY (collection_id, scope, node_id)
)
""")
await self._ensure_nodes_schema()
await self._ensure_bindings_schema()
await self.conn.commit()
async def _ensure_nodes_schema(self) -> None:
assert self.conn is not None
cursor = await self.conn.execute("PRAGMA table_info(nodes)")
rows = await cursor.fetchall()
columns = [r[1] for r in rows] if rows else []
if "scope" not in columns:
await self.conn.execute("""
CREATE TABLE IF NOT EXISTS nodes_new (
collection_id TEXT NOT NULL,
scope TEXT NOT NULL DEFAULT 'global',
node_id TEXT NOT NULL,
node_type TEXT NOT NULL,
data_id TEXT,
bind_data_id TEXT,
data TEXT,
origin_data TEXT,
action TEXT,
status TEXT,
binding_status TEXT,
error TEXT,
sync_log TEXT,
context TEXT,
PRIMARY KEY (collection_id, scope, node_id)
)
""")
await self.conn.execute("""
INSERT INTO nodes_new (
collection_id, scope, node_id, node_type, data_id, bind_data_id,
data, origin_data, action, status, binding_status, error, sync_log, context
)
SELECT
collection_id, 'global', node_id, node_type, data_id, bind_data_id,
data, origin_data, action, status, binding_status, error, sync_log, context
FROM nodes
""")
await self.conn.execute("DROP TABLE nodes")
await self.conn.execute("ALTER TABLE nodes_new RENAME TO nodes")
cursor = await self.conn.execute("PRAGMA table_info(nodes)")
rows = await cursor.fetchall()
columns = [r[1] for r in rows] if rows else []
if "bind_data_id" not in columns:
await self.conn.execute("ALTER TABLE nodes ADD COLUMN bind_data_id TEXT")
await self.conn.execute(
"CREATE INDEX IF NOT EXISTS idx_nodes_collection_scope_type ON nodes (collection_id, scope, node_type)"
)
await self.conn.execute(
"CREATE INDEX IF NOT EXISTS idx_nodes_collection_scope_data_id ON nodes (collection_id, scope, data_id)"
)
async def _ensure_bindings_schema(self) -> None:
assert self.conn is not None
cursor = await self.conn.execute("PRAGMA table_info(bindings)")
rows = await cursor.fetchall()
columns = [r[1] for r in rows] if rows else []
if not columns:
await self.conn.execute("""
CREATE TABLE IF NOT EXISTS bindings (
scope TEXT NOT NULL DEFAULT 'global',
node_type TEXT NOT NULL,
local_id TEXT NOT NULL,
remote_id TEXT,
last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (scope, node_type, local_id)
)
""")
return
if "scope" not in columns or ("payload" in columns and "remote_id" not in columns):
await self.conn.execute("""
CREATE TABLE IF NOT EXISTS bindings_new (
scope TEXT NOT NULL DEFAULT 'global',
node_type TEXT NOT NULL,
local_id TEXT NOT NULL,
remote_id TEXT,
last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (scope, node_type, local_id)
)
""")
if "payload" in columns and "remote_id" not in columns:
async with self.conn.execute("SELECT node_type, local_id, payload FROM bindings") as cur:
old_rows = await cur.fetchall()
for row in old_rows:
try:
payload_dict = json.loads(row[2]) if row[2] else {}
except json.JSONDecodeError:
payload_dict = {}
remote_id = payload_dict.get("remote_id")
await self.conn.execute(
"INSERT OR REPLACE INTO bindings_new (scope, node_type, local_id, remote_id) VALUES (?, ?, ?, ?)",
("global", row[0], row[1], remote_id),
)
else:
async with self.conn.execute("SELECT node_type, local_id, remote_id FROM bindings") as cur:
old_rows = await cur.fetchall()
for row in old_rows:
await self.conn.execute(
"INSERT OR REPLACE INTO bindings_new (scope, node_type, local_id, remote_id) VALUES (?, ?, ?, ?)",
("global", row[0], row[1], row[2]),
)
await self.conn.execute("DROP TABLE bindings")
await self.conn.execute("ALTER TABLE bindings_new RENAME TO bindings")
await self.conn.execute(
"CREATE INDEX IF NOT EXISTS idx_bindings_scope_type_remote ON bindings (scope, node_type, remote_id)"
)
async def save_node(self, collection_id: str, scope: str, node: "SyncNode") -> None:
assert self.conn is not None
bind_data_id = None
if isinstance(node.context, dict):
value = node.context.get("bind_data_id")
if value is not None and value != "":
bind_data_id = str(value)
await self.conn.execute("""
INSERT OR REPLACE INTO nodes (
collection_id, scope, node_id, node_type, data_id, bind_data_id,
data, origin_data, action, status, binding_status, error, sync_log, context
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""", (
collection_id,
scope,
node.node_id,
node.node_type,
node.data_id,
bind_data_id,
json.dumps(node.data) if node.data else None,
json.dumps(node.origin_data) if node.origin_data else None,
node.action.value,
node.status.value,
node.binding_status.value,
node.error,
node.sync_log,
json.dumps(node.context) if node.context else None,
))
await self.conn.commit()
async def save_nodes_bulk(self, collection_id: str, scope: str, nodes: List["SyncNode"]) -> None:
assert self.conn is not None
if not nodes:
return
rows = []
for node in nodes:
bind_data_id = None
if isinstance(node.context, dict):
value = node.context.get("bind_data_id")
if value is not None and value != "":
bind_data_id = str(value)
rows.append((
collection_id,
scope,
node.node_id,
node.node_type,
node.data_id,
bind_data_id,
json.dumps(node.data) if node.data else None,
json.dumps(node.origin_data) if node.origin_data else None,
node.action.value,
node.status.value,
node.binding_status.value,
node.error,
node.sync_log,
json.dumps(node.context) if node.context else None,
))
await self.conn.executemany("""
INSERT OR REPLACE INTO nodes (
collection_id, scope, node_id, node_type, data_id, bind_data_id,
data, origin_data, action, status, binding_status, error, sync_log, context
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""", rows)
await self.conn.commit()
async def delete_node(self, collection_id: str, scope: str, node_id: str) -> None:
assert self.conn is not None
await self.conn.execute(
"DELETE FROM nodes WHERE collection_id = ? AND scope = ? AND node_id = ?",
(collection_id, scope, node_id),
)
await self.conn.commit()
async def delete_nodes_bulk(self, collection_id: str, scope: str, node_ids: List[str]) -> None:
assert self.conn is not None
if not node_ids:
return
rows = [(collection_id, scope, node_id) for node_id in node_ids]
await self.conn.executemany(
"DELETE FROM nodes WHERE collection_id = ? AND scope = ? AND node_id = ?",
rows,
)
await self.conn.commit()
async def load_nodes(
self,
collection_id: str,
scope: str,
*,
exclude_node_types: Optional[List[str]] = None,
) -> List[Dict[str, Any]]:
assert self.conn is not None
sql = "SELECT * FROM nodes WHERE collection_id = ? AND scope = ?"
params: List[Any] = [collection_id, scope]
filtered_types = [node_type for node_type in (exclude_node_types or []) if node_type]
if filtered_types:
placeholders = ", ".join("?" for _ in filtered_types)
sql += f" AND node_type NOT IN ({placeholders})"
params.extend(filtered_types)
async with self.conn.execute(sql, tuple(params)) as cursor:
rows = await cursor.fetchall()
results = []
for row in rows:
item = {
"node_id": row["node_id"],
"node_type": row["node_type"],
"data_id": row["data_id"],
"bind_data_id": row["bind_data_id"],
"depend_ids": [],
"data": json.loads(row["data"]) if row["data"] else None,
"origin_data": json.loads(row["origin_data"]) if row["origin_data"] else None,
"action": row["action"],
"status": row["status"],
"binding_status": row["binding_status"],
"error": row["error"],
"sync_log": row["sync_log"],
"context": json.loads(row["context"]) if row["context"] else {},
}
if item["bind_data_id"] and isinstance(item["context"], dict):
item["context"]["bind_data_id"] = item["bind_data_id"]
results.append(item)
return results
async def save_binding(self, scope: str, node_type: str, local_id: str, payload_dict: Dict[str, Any]) -> None:
assert self.conn is not None
remote_id = payload_dict.get("remote_id") if payload_dict else None
await self.conn.execute("""
INSERT OR REPLACE INTO bindings (scope, node_type, local_id, remote_id)
VALUES (?, ?, ?, ?)
""", (scope, node_type, local_id, remote_id))
await self.conn.commit()
async def save_bindings_bulk(self, scope: str, node_type: str, bindings: Dict[str, Optional[str]]) -> None:
assert self.conn is not None
if not bindings:
return
rows = [(scope, node_type, local_id, remote_id) for local_id, remote_id in bindings.items()]
await self.conn.executemany(
"INSERT OR REPLACE INTO bindings (scope, node_type, local_id, remote_id) VALUES (?, ?, ?, ?)",
rows,
)
await self.conn.commit()
async def delete_binding(self, scope: str, node_type: str, local_id: str) -> None:
assert self.conn is not None
await self.conn.execute(
"DELETE FROM bindings WHERE scope = ? AND node_type = ? AND local_id = ?",
(scope, node_type, local_id),
)
await self.conn.commit()
async def delete_bindings_bulk(self, scope: str, node_type: str, local_ids: List[str]) -> None:
assert self.conn is not None
if not local_ids:
return
rows = [(scope, node_type, local_id) for local_id in local_ids]
await self.conn.executemany(
"DELETE FROM bindings WHERE scope = ? AND node_type = ? AND local_id = ?",
rows,
)
await self.conn.commit()
async def load_bindings(self, scope: str, node_type: str) -> List[Dict[str, Any]]:
assert self.conn is not None
async with self.conn.execute(
"SELECT local_id, remote_id FROM bindings WHERE scope = ? AND node_type = ?",
(scope, node_type),
) as cursor:
rows = await cursor.fetchall()
return [{"local_id": r["local_id"], "payload": {"remote_id": r["remote_id"]}} for r in rows]
async def load_bind_data_id_overrides(self, node_type: str, scope: str) -> Dict[str, Optional[str]]:
assert self.conn is not None
sql = """
SELECT b.local_id AS src_node_id, rn.data_id AS bind_data_id
FROM bindings b
LEFT JOIN nodes rn
ON rn.node_id = b.remote_id
AND rn.node_type = b.node_type
AND rn.scope = b.scope
WHERE b.node_type = ?
AND b.scope = ?
UNION ALL
SELECT b.remote_id AS src_node_id, ln.data_id AS bind_data_id
FROM bindings b
LEFT JOIN nodes ln
ON ln.node_id = b.local_id
AND ln.node_type = b.node_type
AND ln.scope = b.scope
WHERE b.node_type = ?
AND b.scope = ?
AND b.remote_id IS NOT NULL
"""
async with self.conn.execute(sql, (node_type, scope, node_type, scope)) as cursor:
rows = await cursor.fetchall()
result: Dict[str, Optional[str]] = {}
for row in rows:
src_node_id = row["src_node_id"]
if not src_node_id:
continue
result[str(src_node_id)] = row["bind_data_id"]
return result
async def delete_scope_node_types(self, *, scope: str, node_types: List[str]) -> None:
assert self.conn is not None
filtered_types = [node_type for node_type in node_types if node_type]
if not filtered_types:
return
placeholders = ", ".join("?" for _ in filtered_types)
await self.conn.execute(
f"DELETE FROM nodes WHERE scope = ? AND node_type IN ({placeholders})",
(scope, *filtered_types),
)
await self.conn.execute(
f"DELETE FROM bindings WHERE scope = ? AND node_type IN ({placeholders})",
(scope, *filtered_types),
)
await self.conn.commit()
async def copy_nodes_between_scopes(
self,
*,
collection_id: str,
source_scope: str,
target_scope: str,
node_types: List[str],
) -> None:
assert self.conn is not None
if source_scope == target_scope or not node_types:
return
placeholders = ", ".join("?" for _ in node_types)
sql = f"""
INSERT OR REPLACE INTO nodes (
collection_id, scope, node_id, node_type, data_id, bind_data_id,
data, origin_data, action, status, binding_status, error, sync_log, context
)
SELECT
collection_id, ?, node_id, node_type, data_id, bind_data_id,
data, origin_data, action, status, binding_status, error, sync_log, context
FROM nodes
WHERE collection_id = ?
AND scope = ?
AND node_type IN ({placeholders})
"""
await self.conn.execute(sql, (target_scope, collection_id, source_scope, *node_types))
await self.conn.commit()
async def copy_bindings_between_scopes(
self,
*,
source_scope: str,
target_scope: str,
node_types: List[str],
) -> None:
assert self.conn is not None
if source_scope == target_scope or not node_types:
return
placeholders = ", ".join("?" for _ in node_types)
sql = f"""
INSERT OR REPLACE INTO bindings (scope, node_type, local_id, remote_id)
SELECT ?, node_type, local_id, remote_id
FROM bindings
WHERE scope = ?
AND node_type IN ({placeholders})
"""
await self.conn.execute(sql, (target_scope, source_scope, *node_types))
await self.conn.commit()
async def dump_to_runtime(self, filename: str) -> None:
assert self.conn is not None
runtime_path = os.path.join("_runtime", filename)
if os.path.exists(runtime_path):
os.remove(runtime_path)
await self.conn.execute("VACUUM INTO ?", (runtime_path,))
async def close(self) -> None:
if self.conn:
try:
await self.conn.close()
except Exception as exc:
print(f"⚠️ Error closing persistence: {exc}")
finally:
self.conn = None
@@ -0,0 +1,3 @@
from .persistence_backends.sqlite_backend import SQLitePersistenceBackend
__all__ = ["SQLitePersistenceBackend"]
+2 -2
View File
@@ -1,4 +1,4 @@
from typing import Any, Dict, Literal
from typing import Any, Dict
from pydantic import BaseModel, ConfigDict, Field
@@ -6,7 +6,7 @@ from pydantic import BaseModel, ConfigDict, Field
class PersistConfig(BaseModel):
model_config = ConfigDict(extra="forbid", validate_assignment=True)
backend: Literal["sqlite", "sqlalchemy", "custom"] = "sqlite"
backend: str = "sqlite"
db_path: str
backend_options: Dict[str, Any] = Field(default_factory=dict)
enable: bool = True
+2 -5
View File
@@ -21,7 +21,7 @@ from typing import Any, Dict, List, Optional
from ..common.binding import BindingManager
from ..common.collection import DataCollection
from ..common.persistence import PersistenceBackend
from ..common.persistence import PersistenceBackend, create_persistence_backend
from ..common.registry import DomainRegistry
from ..datasource import ensure_builtin_datasource_types_registered
from ..datasource.type_registry import get_datasource_factory
@@ -292,9 +292,6 @@ async def create_pipeline_from_config(
all_types = list(config.node_types)
strategy_map = _resolve_strategy_map(config, all_types)
if config.persist.backend != "sqlite":
raise NotImplementedError(f"Persistence backend not supported yet: {config.persist.backend}")
db_path = Path(config.persist.db_path)
if config.persist.wipe_on_start and db_path.exists():
db_path.unlink()
@@ -303,7 +300,7 @@ async def create_pipeline_from_config(
local_ds = None
remote_ds = None
try:
persistence = PersistenceBackend(str(config.persist.db_path), backend=config.persist.backend)
persistence = create_persistence_backend(config.persist)
await persistence.initialize()
local_ds = await _create_datasource(
@@ -11,6 +11,7 @@ import pytest_asyncio
from sync_state_machine.common.binding import BindingManager
from sync_state_machine.common.collection import DataCollection
from sync_state_machine.common.persistence import PersistenceBackend
from sync_state_machine.common.sqlite_backend import SQLitePersistenceBackend
from sync_state_machine.common.types import BindingStatus, SyncAction, SyncStatus
from sync_state_machine.pipeline.full_sync_pipeline import FullSyncPipeline
from sync_state_machine.sync_system.strategy_ops.compare_ops import (
@@ -39,7 +40,7 @@ async def pipeline_bundle():
register_test_domain()
db = Path(tempfile.mkdtemp(prefix="sync_state_machine_pipeline_") ) / "state.db"
persistence = PersistenceBackend(str(db))
persistence = SQLitePersistenceBackend(str(db))
await persistence.initialize()
local_collection = DataCollection("local", persistence=persistence, auto_persist=False)
@@ -304,7 +305,7 @@ async def test_load_overrides_bind_data_id_from_bindings(pipeline_bundle):
existing_persistence = local_collection.persistence
assert existing_persistence is not None
persistence = PersistenceBackend(existing_persistence.db_path)
persistence = SQLitePersistenceBackend(existing_persistence.db_path)
await persistence.initialize()
await persistence.conn.execute(
@@ -443,7 +444,7 @@ async def test_pipeline_create_then_update_completes_in_single_run_with_reload()
register_test_domain()
db = Path(tempfile.mkdtemp(prefix="sync_state_machine_pipeline_reload_") ) / "state.db"
persistence = PersistenceBackend(str(db))
persistence = SQLitePersistenceBackend(str(db))
await persistence.initialize()
local_collection = DataCollection("local", persistence=persistence, auto_persist=False)
@@ -8,6 +8,7 @@ import pytest
from sync_state_machine.common.binding import BindingManager
from sync_state_machine.common.collection import DataCollection
from sync_state_machine.common.persistence import PersistenceBackend
from sync_state_machine.common.sqlite_backend import SQLitePersistenceBackend
from sync_state_machine.pipeline.full_sync_pipeline import FullSyncPipeline
from sync_state_machine.sync_system.config import OrphanAction, UpdateDirection
from tests.fixtures.mock_domain import (
@@ -36,7 +37,7 @@ async def test_full_pipeline_multinode_mock_persisted_db_snapshot() -> None:
if db_path.exists():
db_path.unlink()
persistence = PersistenceBackend(str(db_path))
persistence = SQLitePersistenceBackend(str(db_path))
await persistence.initialize()
local_collection = DataCollection("local", persistence=persistence, auto_persist=False)
@@ -7,6 +7,7 @@ import pytest
from sync_state_machine.common.collection import DataCollection
from sync_state_machine.common.persistence import PersistenceBackend
from sync_state_machine.common.sqlite_backend import SQLitePersistenceBackend
from sync_state_machine.common.types import BindingStatus, SyncAction, SyncStatus
from sync_state_machine.engine.dispatcher import StateMachineRuntime
from sync_state_machine.engine.model import StateMachineConfig
@@ -21,7 +22,7 @@ from tests.mocks.mock_datasource import (
@pytest.fixture
def datasource_bundle():
db = Path(tempfile.mkdtemp(prefix="sync_state_machine_mock_ds_")) / "state.db"
persistence = PersistenceBackend(str(db))
persistence = SQLitePersistenceBackend(str(db))
cfg_path = Path(__file__).resolve().parents[2] / "sync_state_machine" / "config" / "node_state_machine.yaml"
runtime = StateMachineRuntime(StateMachineConfig.load(cfg_path))
collection = DataCollection("mock", persistence=None, auto_persist=False)
@@ -49,7 +50,7 @@ async def test_create_success_to_s11(datasource_bundle):
node = _new_node("n1", SyncAction.CREATE)
await collection.add(node)
datasource.register_handler(ScriptedMockHandler(HandlerScript(create="success")))
datasource.add_handler(ScriptedMockHandler(HandlerScript(create="success")))
datasource.set_collection(collection)
await datasource.sync_all(data_type="mock")
@@ -62,7 +63,7 @@ async def test_create_skipped_is_rejected(datasource_bundle):
node = _new_node("n2", SyncAction.CREATE)
await collection.add(node)
datasource.register_handler(ScriptedMockHandler(HandlerScript(create="skipped")))
datasource.add_handler(ScriptedMockHandler(HandlerScript(create="skipped")))
datasource.set_collection(collection)
await datasource.sync_all(data_type="mock")
@@ -76,7 +77,7 @@ async def test_update_skip_downgrade_to_s14(datasource_bundle):
node = _new_node("n3", SyncAction.UPDATE)
await collection.add(node)
datasource.register_handler(
datasource.add_handler(
ScriptedMockHandler(HandlerScript(update="skipped"))
)
datasource.set_collection(collection)
@@ -94,7 +95,7 @@ async def test_update_success_writes_back_origin_data(datasource_bundle):
node.set_data({"id": "n3b", "name": "after"})
await collection.add(node)
datasource.register_handler(ScriptedMockHandler(HandlerScript(update="success")))
datasource.add_handler(ScriptedMockHandler(HandlerScript(update="success")))
datasource.set_collection(collection)
await datasource.sync_all(data_type="mock")
@@ -109,7 +110,7 @@ async def test_in_progress_poll_success_to_s11(datasource_bundle):
node = _new_node("n4", SyncAction.CREATE)
await collection.add(node)
datasource.register_handler(
datasource.add_handler(
ScriptedMockHandler(HandlerScript(create="in_progress", poll_sequence=["success"]))
)
datasource.set_collection(collection)
@@ -124,7 +125,7 @@ async def test_in_progress_poll_timeout_to_s12(datasource_bundle):
node = _new_node("n5", SyncAction.CREATE)
await collection.add(node)
datasource.register_handler(
datasource.add_handler(
ScriptedMockHandler(HandlerScript(create="in_progress", poll_sequence=["in_progress", "in_progress"]))
)
datasource.set_collection(collection)
@@ -10,6 +10,7 @@ import pytest
from sync_state_machine.common.collection import DataCollection
from sync_state_machine.common.binding import BindingManager
from sync_state_machine.common.persistence import PersistenceBackend
from sync_state_machine.common.sqlite_backend import SQLitePersistenceBackend
from sync_state_machine.common.types import BindingStatus, SyncAction, SyncStatus
from sync_state_machine.engine import StateMachineConfig, StateMachineRuntime
from sync_state_machine.engine.events import (
@@ -464,7 +465,7 @@ async def test_sync_log_cleared_on_initial_load(tmp_path: Path) -> None:
REPORT_DIR.mkdir(parents=True, exist_ok=True)
db = tmp_path / "sync_log_clear_state.db"
persistence = PersistenceBackend(str(db))
persistence = SQLitePersistenceBackend(str(db))
await persistence.initialize()
collection = DataCollection("project-log-clear", persistence=persistence, auto_persist=False)
@@ -531,7 +532,7 @@ async def test_load_preserves_state_then_e01_normalizes_to_s00(tmp_path: Path) -
REPORT_DIR.mkdir(parents=True, exist_ok=True)
db = tmp_path / "load_preserve_then_e01_normalize.db"
persistence = PersistenceBackend(str(db))
persistence = SQLitePersistenceBackend(str(db))
await persistence.initialize()
collection = DataCollection("project-e01-normalize", persistence=persistence, auto_persist=False)
@@ -583,7 +584,7 @@ async def test_invalid_persisted_enum_is_bootstrap_blocked_and_skipped_in_reset(
REPORT_DIR.mkdir(parents=True, exist_ok=True)
db = tmp_path / "invalid_enum_bootstrap_blocked.db"
persistence = PersistenceBackend(str(db))
persistence = SQLitePersistenceBackend(str(db))
await persistence.initialize()
collection_id = "project-invalid-enum"
@@ -9,6 +9,7 @@ import pytest_asyncio
from sync_state_machine.common.binding import BindingManager
from sync_state_machine.common.collection import DataCollection
from sync_state_machine.common.persistence import PersistenceBackend
from sync_state_machine.common.sqlite_backend import SQLitePersistenceBackend
from sync_state_machine.common.types import BindingStatus, SyncAction, SyncStatus
from sync_state_machine.sync_system.resolve_ids import IDResolver
from sync_state_machine.sync_system.config import OrphanAction, UpdateDirection
@@ -32,7 +33,7 @@ from tests.fixtures.mock_domain import (
async def setup_env():
register_test_domain()
db = Path(tempfile.mkdtemp(prefix="sync_state_machine_biz_") ) / "state.db"
persistence = PersistenceBackend(str(db))
persistence = SQLitePersistenceBackend(str(db))
await persistence.initialize()
local = DataCollection("local", persistence=persistence, auto_persist=False)
@@ -9,7 +9,7 @@ from pydantic import BaseModel
from sync_state_machine.config.multi_project_config import build_multi_project_config_from_file
from sync_state_machine.common.binding import BindingManager
from sync_state_machine.common.collection import DataCollection
from sync_state_machine.common.persistence import PersistenceBackend
from sync_state_machine.common.sqlite_backend import SQLitePersistenceBackend
from sync_state_machine.common.sync_node import SyncNode
from sync_state_machine.pipeline.multi_project_pipeline import (
_project_scope_overrides,
@@ -88,7 +88,7 @@ class _ProjectNode(SyncNode[dict]):
@pytest.mark.asyncio
async def test_binding_manager_prunes_stale_bindings_after_project_scope_cleanup() -> None:
persistence = PersistenceBackend(":memory:")
persistence = SQLitePersistenceBackend(":memory:")
await persistence.initialize()
try:
local_collection = DataCollection("local", persistence=persistence, auto_persist=False)
+2 -1
View File
@@ -7,6 +7,7 @@ import pytest
from sync_state_machine.common.binding import BindingManager
from sync_state_machine.common.collection import DataCollection
from sync_state_machine.common.persistence import PersistenceBackend
from sync_state_machine.common.sqlite_backend import SQLitePersistenceBackend
from sync_state_machine.common.types import BindingStatus, SyncAction, SyncStatus
from sync_state_machine.datasource.datasource import BaseDataSource
from sync_state_machine.engine.state_apply import apply_state_to_node
@@ -38,7 +39,7 @@ def _set_state(pipeline: FullSyncPipeline, node: MockSyncNode, state_id: str, *,
async def _new_pipeline(tmp_path: Path) -> tuple[FullSyncPipeline, DataCollection, DataCollection, BindingManager, PersistenceBackend]:
db_path = tmp_path / "state.db"
persistence = PersistenceBackend(str(db_path))
persistence = SQLitePersistenceBackend(str(db_path))
await persistence.initialize()
local_collection = DataCollection("local", persistence=persistence, auto_persist=False)
@@ -7,6 +7,7 @@ import pytest
from sync_state_machine.common.binding import BindingManager
from sync_state_machine.common.collection import DataCollection
from sync_state_machine.common.persistence import PersistenceBackend
from sync_state_machine.common.sqlite_backend import SQLitePersistenceBackend
from sync_state_machine.datasource.datasource import BaseDataSource
from sync_state_machine.pipeline.full_sync_pipeline import FullSyncPipeline
from sync_state_machine.sync_system.strategy import BaseSyncStrategy
@@ -53,7 +54,7 @@ class _OkStrategy(BaseSyncStrategy):
@pytest.mark.asyncio
async def test_phase_sync_by_node_type_continues_after_strategy_error(tmp_path: Path) -> None:
db_path = tmp_path / "state.db"
persistence = PersistenceBackend(str(db_path))
persistence = SQLitePersistenceBackend(str(db_path))
await persistence.initialize()
local_collection = DataCollection("local", persistence=persistence, auto_persist=False)
@@ -101,7 +102,7 @@ async def test_phase_sync_by_node_type_continues_after_strategy_error(tmp_path:
@pytest.mark.asyncio
async def test_phase_sync_by_node_type_skip_sync_keeps_bind_but_skips_writes(tmp_path: Path) -> None:
db_path = tmp_path / "state.db"
persistence = PersistenceBackend(str(db_path))
persistence = SQLitePersistenceBackend(str(db_path))
await persistence.initialize()
local_collection = DataCollection("local", persistence=persistence, auto_persist=False)
@@ -8,6 +8,7 @@ import pytest
from sync_state_machine.common.binding import BindingManager
from sync_state_machine.common.collection import DataCollection
from sync_state_machine.common.persistence import PersistenceBackend
from sync_state_machine.common.sqlite_backend import SQLitePersistenceBackend
from sync_state_machine.common.types import BindingStatus, SyncAction, SyncStatus
from sync_state_machine.domain.construction.sync_node import ConstructionTaskSyncNode
from sync_state_machine.domain.construction.sync_strategy import ConstructionTaskSyncStrategy
@@ -19,7 +20,7 @@ from sync_state_machine.sync_system.strategy_ops.post_check_ops import evaluate_
@pytest.mark.asyncio
async def test_post_check_uses_same_id_resolver_as_create_update_for_depend_ids(tmp_path: Path) -> None:
db_path = tmp_path / "state.db"
persistence = PersistenceBackend(str(db_path))
persistence = SQLitePersistenceBackend(str(db_path))
await persistence.initialize()
local_collection = DataCollection("local", persistence=persistence, auto_persist=False)
@@ -167,7 +168,7 @@ async def test_post_check_uses_same_id_resolver_as_create_update_for_depend_ids(
@pytest.mark.asyncio
async def test_post_check_ignores_only_explicit_configured_fields(tmp_path: Path) -> None:
db_path = tmp_path / "state.db"
persistence = PersistenceBackend(str(db_path))
persistence = SQLitePersistenceBackend(str(db_path))
await persistence.initialize()
local_collection = DataCollection("local", persistence=persistence, auto_persist=False)
@@ -7,6 +7,7 @@ import pytest
from sync_state_machine.common.binding import BindingManager
from sync_state_machine.common.collection import DataCollection
from sync_state_machine.common.persistence import PersistenceBackend
from sync_state_machine.common.sqlite_backend import SQLitePersistenceBackend
from tests.fixtures.mock_domain import build_project_node, register_test_domain
@@ -15,7 +16,7 @@ async def test_scope_partitioning_and_copy_between_scopes(tmp_path: Path) -> Non
register_test_domain()
db_path = tmp_path / "scope_partitioning.db"
persistence = PersistenceBackend(str(db_path))
persistence = SQLitePersistenceBackend(str(db_path))
await persistence.initialize()
global_collection = DataCollection("local", scope="global", persistence=persistence, auto_persist=False)
+4 -5
View File
@@ -89,11 +89,10 @@
<div class="row">
<label>Persist Backend
<select id="persistBackend">
<option value="sqlite">sqlite</option>
<option value="sqlalchemy">sqlalchemy</option>
<option value="custom">custom</option>
</select>
<input id="persistBackend" list="persistBackendOptions" />
<datalist id="persistBackendOptions">
<option value="sqlite"></option>
</datalist>
</label>
<label>持久化 DB 路径
<input id="persistenceDb" />