将persist和后端分离,方便替换为其他数据库

This commit is contained in:
strepsiades
2026-03-24 10:15:16 +08:00
parent 7d92b3f1fe
commit 03a22b0c1c
19 changed files with 665 additions and 471 deletions
+11 -1
View File
@@ -1,6 +1,12 @@
from .types import SyncAction, SyncStatus, BindingStatus
from .sync_node import SyncNode
from .persistence import PersistenceBackend
from .persistence import (
PersistenceBackend,
create_persistence_backend,
get_registered_persistence_backends,
register_persistence_backend,
)
from .persistence_backends import SQLitePersistenceBackend
from .collection import DataCollection
from .binding import BindingManager
@@ -10,6 +16,10 @@ __all__ = [
"BindingStatus",
"SyncNode",
"PersistenceBackend",
"SQLitePersistenceBackend",
"create_persistence_backend",
"register_persistence_backend",
"get_registered_persistence_backends",
"DataCollection",
"BindingManager",
]
+103 -435
View File
@@ -1,21 +1,20 @@
import aiosqlite
import json
from __future__ import annotations
import os
import sqlite3
from typing import Optional, Dict, List, Any, TYPE_CHECKING
from abc import ABC, abstractmethod
from typing import Any, Callable, ClassVar, Dict, List, Optional, TYPE_CHECKING
from ..config.persist_config import PersistConfig
if TYPE_CHECKING:
from .sync_node import SyncNode
class PersistenceBackend:
"""
Persistence 负责同步系统运行时数据的持久化存储(aiosqlite 实现)。
主要作为 DataCollection 和 BindingManager 的后端存储。
"""
def __init__(self, db_path: str = ":memory:", backend: str = "sqlite"):
self.db_path = db_path
self.backend = backend
self.conn: Optional[aiosqlite.Connection] = None
class PersistenceBackend(ABC):
"""同步系统持久化契约。"""
backend_name: ClassVar[str] = "abstract"
@staticmethod
def _is_read_only_sql(sql: str) -> bool:
@@ -93,250 +92,27 @@ class PersistenceBackend:
]
return summary
async def initialize(self):
"""异步初始化数据库连接和表结构"""
if self.backend != "sqlite":
raise NotImplementedError(f"Persistence backend not supported yet: {self.backend}")
@abstractmethod
async def initialize(self) -> None:
raise NotImplementedError
# 确保目录存在
if self.db_path != ":memory:":
dir_path = os.path.dirname(self.db_path)
if dir_path:
os.makedirs(dir_path, exist_ok=True)
self.conn = await aiosqlite.connect(self.db_path)
self.conn.row_factory = aiosqlite.Row
await self._init_db()
@abstractmethod
async def save_node(self, collection_id: str, scope: str, node: "SyncNode") -> None:
raise NotImplementedError
async def _init_db(self):
# 1. 节点表: 仅用于持久化存储 DataCollection 中的 SyncNode
# 注意: depend_ids不持久化,每次从data实时计算
await self.conn.execute("""
CREATE TABLE IF NOT EXISTS nodes (
collection_id TEXT NOT NULL,
scope TEXT NOT NULL DEFAULT 'global',
node_id TEXT NOT NULL,
node_type TEXT NOT NULL,
data_id TEXT,
bind_data_id TEXT,
data TEXT,
origin_data TEXT,
action TEXT,
status TEXT,
binding_status TEXT,
error TEXT,
sync_log TEXT,
context TEXT,
PRIMARY KEY (collection_id, scope, node_id)
)
""")
await self._ensure_nodes_schema()
# 2. 绑定表: 存储本地到远程的映射关系
# 只保留 local_id / remote_id 两字段(remote_id 可空)
await self._ensure_bindings_schema()
await self.conn.commit()
@abstractmethod
async def save_nodes_bulk(self, collection_id: str, scope: str, nodes: List["SyncNode"]) -> None:
raise NotImplementedError
async def _ensure_nodes_schema(self) -> None:
cursor = await self.conn.execute("PRAGMA table_info(nodes)")
rows = await cursor.fetchall()
columns = [r[1] for r in rows] if rows else []
if "scope" not in columns:
await self.conn.execute("""
CREATE TABLE IF NOT EXISTS nodes_new (
collection_id TEXT NOT NULL,
scope TEXT NOT NULL DEFAULT 'global',
node_id TEXT NOT NULL,
node_type TEXT NOT NULL,
data_id TEXT,
bind_data_id TEXT,
data TEXT,
origin_data TEXT,
action TEXT,
status TEXT,
binding_status TEXT,
error TEXT,
sync_log TEXT,
context TEXT,
PRIMARY KEY (collection_id, scope, node_id)
)
""")
await self.conn.execute("""
INSERT INTO nodes_new (
collection_id, scope, node_id, node_type, data_id, bind_data_id,
data, origin_data, action, status, binding_status, error, sync_log, context
)
SELECT
collection_id, 'global', node_id, node_type, data_id, bind_data_id,
data, origin_data, action, status, binding_status, error, sync_log, context
FROM nodes
""")
await self.conn.execute("DROP TABLE nodes")
await self.conn.execute("ALTER TABLE nodes_new RENAME TO nodes")
cursor = await self.conn.execute("PRAGMA table_info(nodes)")
rows = await cursor.fetchall()
columns = [r[1] for r in rows] if rows else []
if "bind_data_id" not in columns:
await self.conn.execute("ALTER TABLE nodes ADD COLUMN bind_data_id TEXT")
await self.conn.execute(
"CREATE INDEX IF NOT EXISTS idx_nodes_collection_scope_type ON nodes (collection_id, scope, node_type)"
)
await self.conn.execute(
"CREATE INDEX IF NOT EXISTS idx_nodes_collection_scope_data_id ON nodes (collection_id, scope, data_id)"
)
@abstractmethod
async def delete_node(self, collection_id: str, scope: str, node_id: str) -> None:
raise NotImplementedError
async def _ensure_bindings_schema(self) -> None:
"""确保 bindings 表为 (node_type, local_id, remote_id) 结构。
若检测到旧表含 payload 列,则进行迁移。
"""
cursor = await self.conn.execute("PRAGMA table_info(bindings)")
rows = await cursor.fetchall()
columns = [r[1] for r in rows] if rows else []
if not columns:
await self.conn.execute("""
CREATE TABLE IF NOT EXISTS bindings (
scope TEXT NOT NULL DEFAULT 'global',
node_type TEXT NOT NULL,
local_id TEXT NOT NULL,
remote_id TEXT,
last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (scope, node_type, local_id)
)
""")
return
if "scope" not in columns or ("payload" in columns and "remote_id" not in columns):
# 迁移旧结构到 scope + remote_id 新结构
await self.conn.execute("""
CREATE TABLE IF NOT EXISTS bindings_new (
scope TEXT NOT NULL DEFAULT 'global',
node_type TEXT NOT NULL,
local_id TEXT NOT NULL,
remote_id TEXT,
last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (scope, node_type, local_id)
)
""")
if "payload" in columns and "remote_id" not in columns:
async with self.conn.execute("SELECT node_type, local_id, payload FROM bindings") as cur:
old_rows = await cur.fetchall()
for row in old_rows:
try:
payload_dict = json.loads(row[2]) if row[2] else {}
except json.JSONDecodeError:
payload_dict = {}
remote_id = payload_dict.get("remote_id")
await self.conn.execute(
"INSERT OR REPLACE INTO bindings_new (scope, node_type, local_id, remote_id) VALUES (?, ?, ?, ?)",
("global", row[0], row[1], remote_id),
)
else:
async with self.conn.execute("SELECT node_type, local_id, remote_id FROM bindings") as cur:
old_rows = await cur.fetchall()
for row in old_rows:
await self.conn.execute(
"INSERT OR REPLACE INTO bindings_new (scope, node_type, local_id, remote_id) VALUES (?, ?, ?, ?)",
("global", row[0], row[1], row[2]),
)
await self.conn.execute("DROP TABLE bindings")
await self.conn.execute("ALTER TABLE bindings_new RENAME TO bindings")
await self.conn.execute(
"CREATE INDEX IF NOT EXISTS idx_bindings_scope_type_remote ON bindings (scope, node_type, remote_id)"
)
# --- DataCollection 支持 ---
async def save_node(self, collection_id: str, scope: str, node: "SyncNode"):
"""保存或更新单个节点(depend_ids不持久化,运行时从data实时计算)"""
bind_data_id = None
if isinstance(node.context, dict):
value = node.context.get("bind_data_id")
if value is not None and value != "":
bind_data_id = str(value)
await self.conn.execute("""
INSERT OR REPLACE INTO nodes (
collection_id, scope, node_id, node_type, data_id, bind_data_id,
data, origin_data, action, status, binding_status, error, sync_log, context
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""", (
collection_id,
scope,
node.node_id,
node.node_type,
node.data_id,
bind_data_id,
json.dumps(node.data) if node.data else None,
json.dumps(node.origin_data) if node.origin_data else None,
node.action.value,
node.status.value,
node.binding_status.value,
node.error,
node.sync_log,
json.dumps(node.context) if node.context else None
))
await self.conn.commit()
async def save_nodes_bulk(self, collection_id: str, scope: str, nodes: List["SyncNode"]):
"""批量保存或更新节点 (单次提交,depend_ids不持久化)"""
if not nodes:
return
rows = []
for node in nodes:
bind_data_id = None
if isinstance(node.context, dict):
value = node.context.get("bind_data_id")
if value is not None and value != "":
bind_data_id = str(value)
rows.append((
collection_id,
scope,
node.node_id,
node.node_type,
node.data_id,
bind_data_id,
json.dumps(node.data) if node.data else None,
json.dumps(node.origin_data) if node.origin_data else None,
node.action.value,
node.status.value,
node.binding_status.value,
node.error,
node.sync_log,
json.dumps(node.context) if node.context else None
))
await self.conn.executemany("""
INSERT OR REPLACE INTO nodes (
collection_id, scope, node_id, node_type, data_id, bind_data_id,
data, origin_data, action, status, binding_status, error, sync_log, context
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""", rows)
await self.conn.commit()
async def delete_node(self, collection_id: str, scope: str, node_id: str):
"""删除单个节点"""
await self.conn.execute(
"DELETE FROM nodes WHERE collection_id = ? AND scope = ? AND node_id = ?",
(collection_id, scope, node_id),
)
await self.conn.commit()
async def delete_nodes_bulk(self, collection_id: str, scope: str, node_ids: List[str]):
"""批量删除节点 (单次提交)"""
if not node_ids:
return
rows = [(collection_id, scope, node_id) for node_id in node_ids]
await self.conn.executemany(
"DELETE FROM nodes WHERE collection_id = ? AND scope = ? AND node_id = ?",
rows,
)
await self.conn.commit()
@abstractmethod
async def delete_nodes_bulk(self, collection_id: str, scope: str, node_ids: List[str]) -> None:
raise NotImplementedError
@abstractmethod
async def load_nodes(
self,
collection_id: str,
@@ -344,153 +120,37 @@ class PersistenceBackend:
*,
exclude_node_types: Optional[List[str]] = None,
) -> List[Dict[str, Any]]:
"""从特定 collection 中加载所有节点(depend_ids初始化为空,稍后从data实时计算)"""
sql = "SELECT * FROM nodes WHERE collection_id = ? AND scope = ?"
params: List[Any] = [collection_id, scope]
filtered_types = [node_type for node_type in (exclude_node_types or []) if node_type]
if filtered_types:
placeholders = ", ".join("?" for _ in filtered_types)
sql += f" AND node_type NOT IN ({placeholders})"
params.extend(filtered_types)
raise NotImplementedError
async with self.conn.execute(sql, tuple(params)) as cursor:
rows = await cursor.fetchall()
results = []
for row in rows:
results.append({
"node_id": row["node_id"],
"node_type": row["node_type"],
"data_id": row["data_id"],
"bind_data_id": row["bind_data_id"],
"depend_ids": [], # 不从数据库加载,运行时从data实时计算
"data": json.loads(row["data"]) if row["data"] else None,
"origin_data": json.loads(row["origin_data"]) if row["origin_data"] else None,
"action": row["action"],
"status": row["status"],
"binding_status": row["binding_status"],
"error": row["error"],
"sync_log": row["sync_log"],
"context": json.loads(row["context"]) if row["context"] else {}
})
if results[-1]["bind_data_id"] and isinstance(results[-1]["context"], dict):
results[-1]["context"]["bind_data_id"] = results[-1]["bind_data_id"]
return results
@abstractmethod
async def save_binding(self, scope: str, node_type: str, local_id: str, payload_dict: Dict[str, Any]) -> None:
raise NotImplementedError
# --- BindingManager 支持 ---
@abstractmethod
async def save_bindings_bulk(self, scope: str, node_type: str, bindings: Dict[str, Optional[str]]) -> None:
raise NotImplementedError
async def save_binding(self, scope: str, node_type: str, local_id: str, payload_dict: Dict[str, Any]):
"""保存或更新绑定关系及其元数据"""
remote_id = payload_dict.get("remote_id") if payload_dict else None
await self.conn.execute("""
INSERT OR REPLACE INTO bindings (scope, node_type, local_id, remote_id)
VALUES (?, ?, ?, ?)
""", (scope, node_type, local_id, remote_id))
await self.conn.commit()
@abstractmethod
async def delete_binding(self, scope: str, node_type: str, local_id: str) -> None:
raise NotImplementedError
async def save_bindings_bulk(self, scope: str, node_type: str, bindings: Dict[str, Optional[str]]):
"""批量保存或更新绑定关系 (单次提交)"""
if not bindings:
return
rows = []
for local_id, remote_id in bindings.items():
rows.append((scope, node_type, local_id, remote_id))
await self.conn.executemany(
"INSERT OR REPLACE INTO bindings (scope, node_type, local_id, remote_id) VALUES (?, ?, ?, ?)",
rows,
)
await self.conn.commit()
async def delete_binding(self, scope: str, node_type: str, local_id: str):
"""删除绑定关系"""
await self.conn.execute(
"DELETE FROM bindings WHERE scope = ? AND node_type = ? AND local_id = ?",
(scope, node_type, local_id)
)
await self.conn.commit()
async def delete_bindings_bulk(self, scope: str, node_type: str, local_ids: List[str]):
"""批量删除绑定关系 (单次提交)"""
if not local_ids:
return
rows = [(scope, node_type, local_id) for local_id in local_ids]
await self.conn.executemany(
"DELETE FROM bindings WHERE scope = ? AND node_type = ? AND local_id = ?",
rows,
)
await self.conn.commit()
@abstractmethod
async def delete_bindings_bulk(self, scope: str, node_type: str, local_ids: List[str]) -> None:
raise NotImplementedError
@abstractmethod
async def load_bindings(self, scope: str, node_type: str) -> List[Dict[str, Any]]:
"""回写该类型下的所有绑定关系到内存"""
async with self.conn.execute(
"SELECT local_id, remote_id FROM bindings WHERE scope = ? AND node_type = ?",
(scope, node_type)
) as cursor:
rows = await cursor.fetchall()
return [
{"local_id": r["local_id"], "payload": {"remote_id": r["remote_id"]}}
for r in rows
]
raise NotImplementedError
@abstractmethod
async def load_bind_data_id_overrides(self, node_type: str, scope: str) -> Dict[str, Optional[str]]:
"""基于 bindings 表计算 node_id -> bind_data_id 覆盖映射。
raise NotImplementedError
规则:
- local 节点的 bind_data_id 取其绑定 remote 节点的 data_id
- remote 节点的 bind_data_id 取其绑定 local 节点的 data_id
"""
sql = """
SELECT b.local_id AS src_node_id, rn.data_id AS bind_data_id
FROM bindings b
LEFT JOIN nodes rn
ON rn.node_id = b.remote_id
AND rn.node_type = b.node_type
AND rn.scope = b.scope
WHERE b.node_type = ?
AND b.scope = ?
UNION ALL
SELECT b.remote_id AS src_node_id, ln.data_id AS bind_data_id
FROM bindings b
LEFT JOIN nodes ln
ON ln.node_id = b.local_id
AND ln.node_type = b.node_type
AND ln.scope = b.scope
WHERE b.node_type = ?
AND b.scope = ?
AND b.remote_id IS NOT NULL
"""
async with self.conn.execute(sql, (node_type, scope, node_type, scope)) as cursor:
rows = await cursor.fetchall()
result: Dict[str, Optional[str]] = {}
for row in rows:
src_node_id = row["src_node_id"]
if not src_node_id:
continue
result[str(src_node_id)] = row["bind_data_id"]
return result
async def delete_scope_node_types(
self,
*,
scope: str,
node_types: List[str],
) -> None:
filtered_types = [node_type for node_type in node_types if node_type]
if not filtered_types:
return
placeholders = ", ".join("?" for _ in filtered_types)
await self.conn.execute(
f"DELETE FROM nodes WHERE scope = ? AND node_type IN ({placeholders})",
(scope, *filtered_types),
)
await self.conn.execute(
f"DELETE FROM bindings WHERE scope = ? AND node_type IN ({placeholders})",
(scope, *filtered_types),
)
await self.conn.commit()
@abstractmethod
async def delete_scope_node_types(self, *, scope: str, node_types: List[str]) -> None:
raise NotImplementedError
@abstractmethod
async def copy_nodes_between_scopes(
self,
*,
@@ -499,25 +159,9 @@ class PersistenceBackend:
target_scope: str,
node_types: List[str],
) -> None:
if source_scope == target_scope or not node_types:
return
placeholders = ", ".join("?" for _ in node_types)
sql = f"""
INSERT OR REPLACE INTO nodes (
collection_id, scope, node_id, node_type, data_id, bind_data_id,
data, origin_data, action, status, binding_status, error, sync_log, context
)
SELECT
collection_id, ?, node_id, node_type, data_id, bind_data_id,
data, origin_data, action, status, binding_status, error, sync_log, context
FROM nodes
WHERE collection_id = ?
AND scope = ?
AND node_type IN ({placeholders})
"""
await self.conn.execute(sql, (target_scope, collection_id, source_scope, *node_types))
await self.conn.commit()
raise NotImplementedError
@abstractmethod
async def copy_bindings_between_scopes(
self,
*,
@@ -525,34 +169,58 @@ class PersistenceBackend:
target_scope: str,
node_types: List[str],
) -> None:
if source_scope == target_scope or not node_types:
return
placeholders = ", ".join("?" for _ in node_types)
sql = f"""
INSERT OR REPLACE INTO bindings (scope, node_type, local_id, remote_id)
SELECT ?, node_type, local_id, remote_id
FROM bindings
WHERE scope = ?
AND node_type IN ({placeholders})
"""
await self.conn.execute(sql, (target_scope, source_scope, *node_types))
await self.conn.commit()
raise NotImplementedError
async def dump_to_runtime(self, filename: str):
"""将当前状态转储到 _runtime 文件夹 (使用 VACUUM INTO)"""
runtime_path = os.path.join("_runtime", filename)
if os.path.exists(runtime_path):
os.remove(runtime_path)
# VACUUM INTO 是备份 SQLite 的推荐方式 (SQLite 3.27+)
await self.conn.execute("VACUUM INTO ?", (runtime_path,))
@abstractmethod
async def dump_to_runtime(self, filename: str) -> None:
raise NotImplementedError
async def close(self):
"""关闭数据库连接(包含异常处理)"""
if self.conn:
try:
await self.conn.close()
except Exception as e:
print(f"⚠️ Error closing persistence: {e}")
finally:
self.conn = None
@abstractmethod
async def close(self) -> None:
raise NotImplementedError
PersistenceBackendFactory = Callable[[PersistConfig], PersistenceBackend]
_PERSISTENCE_BACKEND_REGISTRY: dict[str, PersistenceBackendFactory] = {}
def register_persistence_backend(name: str, factory: PersistenceBackendFactory, *, override: bool = False) -> None:
backend_name = name.strip()
if not backend_name:
raise ValueError("backend name cannot be empty")
if not override and backend_name in _PERSISTENCE_BACKEND_REGISTRY:
raise ValueError(f"persistence backend already registered: {backend_name}")
_PERSISTENCE_BACKEND_REGISTRY[backend_name] = factory
def get_registered_persistence_backends() -> Dict[str, PersistenceBackendFactory]:
return dict(_PERSISTENCE_BACKEND_REGISTRY)
def create_persistence_backend(
persist_config: PersistConfig,
) -> PersistenceBackend:
backend_name = (persist_config.backend or "").strip()
if not backend_name:
raise ValueError("persist.backend cannot be empty")
if not _PERSISTENCE_BACKEND_REGISTRY:
register_persistence_backend(
"sqlite",
lambda config: SQLitePersistenceBackend(config),
)
factory = _PERSISTENCE_BACKEND_REGISTRY.get(backend_name)
if factory is None:
available = ", ".join(sorted(_PERSISTENCE_BACKEND_REGISTRY)) or "none"
raise NotImplementedError(f"Persistence backend not supported yet: {backend_name}. Available: {available}")
return factory(persist_config)
def _register_builtin_persistence_backends() -> None:
from .persistence_backends.sqlite_backend import SQLitePersistenceBackend
register_persistence_backend("sqlite", lambda config: SQLitePersistenceBackend(config), override=True)
_register_builtin_persistence_backends()
@@ -0,0 +1,3 @@
from .sqlite_backend import SQLitePersistenceBackend
__all__ = ["SQLitePersistenceBackend"]
@@ -0,0 +1,464 @@
from __future__ import annotations
import aiosqlite
import json
import os
from typing import Any, Dict, List, Optional, TYPE_CHECKING
from ...config.persist_config import PersistConfig
from ..persistence import PersistenceBackend
if TYPE_CHECKING:
from ..sync_node import SyncNode
class SQLitePersistenceBackend(PersistenceBackend):
"""SQLite implementation of the persistence contract."""
backend_name = "sqlite"
def __init__(self, persist_config: PersistConfig | str | None = None, **_: Any) -> None:
if isinstance(persist_config, PersistConfig):
self.persist_config = persist_config
self.db_path = persist_config.db_path
elif isinstance(persist_config, str):
self.persist_config = None
self.db_path = persist_config
else:
self.persist_config = None
self.db_path = ":memory:"
self.conn: Optional[aiosqlite.Connection] = None
async def initialize(self) -> None:
if self.db_path != ":memory:":
dir_path = os.path.dirname(self.db_path)
if dir_path:
os.makedirs(dir_path, exist_ok=True)
self.conn = await aiosqlite.connect(self.db_path)
self.conn.row_factory = aiosqlite.Row
await self._init_db()
async def _init_db(self) -> None:
assert self.conn is not None
await self.conn.execute("""
CREATE TABLE IF NOT EXISTS nodes (
collection_id TEXT NOT NULL,
scope TEXT NOT NULL DEFAULT 'global',
node_id TEXT NOT NULL,
node_type TEXT NOT NULL,
data_id TEXT,
bind_data_id TEXT,
data TEXT,
origin_data TEXT,
action TEXT,
status TEXT,
binding_status TEXT,
error TEXT,
sync_log TEXT,
context TEXT,
PRIMARY KEY (collection_id, scope, node_id)
)
""")
await self._ensure_nodes_schema()
await self._ensure_bindings_schema()
await self.conn.commit()
async def _ensure_nodes_schema(self) -> None:
assert self.conn is not None
cursor = await self.conn.execute("PRAGMA table_info(nodes)")
rows = await cursor.fetchall()
columns = [r[1] for r in rows] if rows else []
if "scope" not in columns:
await self.conn.execute("""
CREATE TABLE IF NOT EXISTS nodes_new (
collection_id TEXT NOT NULL,
scope TEXT NOT NULL DEFAULT 'global',
node_id TEXT NOT NULL,
node_type TEXT NOT NULL,
data_id TEXT,
bind_data_id TEXT,
data TEXT,
origin_data TEXT,
action TEXT,
status TEXT,
binding_status TEXT,
error TEXT,
sync_log TEXT,
context TEXT,
PRIMARY KEY (collection_id, scope, node_id)
)
""")
await self.conn.execute("""
INSERT INTO nodes_new (
collection_id, scope, node_id, node_type, data_id, bind_data_id,
data, origin_data, action, status, binding_status, error, sync_log, context
)
SELECT
collection_id, 'global', node_id, node_type, data_id, bind_data_id,
data, origin_data, action, status, binding_status, error, sync_log, context
FROM nodes
""")
await self.conn.execute("DROP TABLE nodes")
await self.conn.execute("ALTER TABLE nodes_new RENAME TO nodes")
cursor = await self.conn.execute("PRAGMA table_info(nodes)")
rows = await cursor.fetchall()
columns = [r[1] for r in rows] if rows else []
if "bind_data_id" not in columns:
await self.conn.execute("ALTER TABLE nodes ADD COLUMN bind_data_id TEXT")
await self.conn.execute(
"CREATE INDEX IF NOT EXISTS idx_nodes_collection_scope_type ON nodes (collection_id, scope, node_type)"
)
await self.conn.execute(
"CREATE INDEX IF NOT EXISTS idx_nodes_collection_scope_data_id ON nodes (collection_id, scope, data_id)"
)
async def _ensure_bindings_schema(self) -> None:
assert self.conn is not None
cursor = await self.conn.execute("PRAGMA table_info(bindings)")
rows = await cursor.fetchall()
columns = [r[1] for r in rows] if rows else []
if not columns:
await self.conn.execute("""
CREATE TABLE IF NOT EXISTS bindings (
scope TEXT NOT NULL DEFAULT 'global',
node_type TEXT NOT NULL,
local_id TEXT NOT NULL,
remote_id TEXT,
last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (scope, node_type, local_id)
)
""")
return
if "scope" not in columns or ("payload" in columns and "remote_id" not in columns):
await self.conn.execute("""
CREATE TABLE IF NOT EXISTS bindings_new (
scope TEXT NOT NULL DEFAULT 'global',
node_type TEXT NOT NULL,
local_id TEXT NOT NULL,
remote_id TEXT,
last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (scope, node_type, local_id)
)
""")
if "payload" in columns and "remote_id" not in columns:
async with self.conn.execute("SELECT node_type, local_id, payload FROM bindings") as cur:
old_rows = await cur.fetchall()
for row in old_rows:
try:
payload_dict = json.loads(row[2]) if row[2] else {}
except json.JSONDecodeError:
payload_dict = {}
remote_id = payload_dict.get("remote_id")
await self.conn.execute(
"INSERT OR REPLACE INTO bindings_new (scope, node_type, local_id, remote_id) VALUES (?, ?, ?, ?)",
("global", row[0], row[1], remote_id),
)
else:
async with self.conn.execute("SELECT node_type, local_id, remote_id FROM bindings") as cur:
old_rows = await cur.fetchall()
for row in old_rows:
await self.conn.execute(
"INSERT OR REPLACE INTO bindings_new (scope, node_type, local_id, remote_id) VALUES (?, ?, ?, ?)",
("global", row[0], row[1], row[2]),
)
await self.conn.execute("DROP TABLE bindings")
await self.conn.execute("ALTER TABLE bindings_new RENAME TO bindings")
await self.conn.execute(
"CREATE INDEX IF NOT EXISTS idx_bindings_scope_type_remote ON bindings (scope, node_type, remote_id)"
)
async def save_node(self, collection_id: str, scope: str, node: "SyncNode") -> None:
assert self.conn is not None
bind_data_id = None
if isinstance(node.context, dict):
value = node.context.get("bind_data_id")
if value is not None and value != "":
bind_data_id = str(value)
await self.conn.execute("""
INSERT OR REPLACE INTO nodes (
collection_id, scope, node_id, node_type, data_id, bind_data_id,
data, origin_data, action, status, binding_status, error, sync_log, context
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""", (
collection_id,
scope,
node.node_id,
node.node_type,
node.data_id,
bind_data_id,
json.dumps(node.data) if node.data else None,
json.dumps(node.origin_data) if node.origin_data else None,
node.action.value,
node.status.value,
node.binding_status.value,
node.error,
node.sync_log,
json.dumps(node.context) if node.context else None,
))
await self.conn.commit()
async def save_nodes_bulk(self, collection_id: str, scope: str, nodes: List["SyncNode"]) -> None:
assert self.conn is not None
if not nodes:
return
rows = []
for node in nodes:
bind_data_id = None
if isinstance(node.context, dict):
value = node.context.get("bind_data_id")
if value is not None and value != "":
bind_data_id = str(value)
rows.append((
collection_id,
scope,
node.node_id,
node.node_type,
node.data_id,
bind_data_id,
json.dumps(node.data) if node.data else None,
json.dumps(node.origin_data) if node.origin_data else None,
node.action.value,
node.status.value,
node.binding_status.value,
node.error,
node.sync_log,
json.dumps(node.context) if node.context else None,
))
await self.conn.executemany("""
INSERT OR REPLACE INTO nodes (
collection_id, scope, node_id, node_type, data_id, bind_data_id,
data, origin_data, action, status, binding_status, error, sync_log, context
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""", rows)
await self.conn.commit()
async def delete_node(self, collection_id: str, scope: str, node_id: str) -> None:
assert self.conn is not None
await self.conn.execute(
"DELETE FROM nodes WHERE collection_id = ? AND scope = ? AND node_id = ?",
(collection_id, scope, node_id),
)
await self.conn.commit()
async def delete_nodes_bulk(self, collection_id: str, scope: str, node_ids: List[str]) -> None:
assert self.conn is not None
if not node_ids:
return
rows = [(collection_id, scope, node_id) for node_id in node_ids]
await self.conn.executemany(
"DELETE FROM nodes WHERE collection_id = ? AND scope = ? AND node_id = ?",
rows,
)
await self.conn.commit()
async def load_nodes(
self,
collection_id: str,
scope: str,
*,
exclude_node_types: Optional[List[str]] = None,
) -> List[Dict[str, Any]]:
assert self.conn is not None
sql = "SELECT * FROM nodes WHERE collection_id = ? AND scope = ?"
params: List[Any] = [collection_id, scope]
filtered_types = [node_type for node_type in (exclude_node_types or []) if node_type]
if filtered_types:
placeholders = ", ".join("?" for _ in filtered_types)
sql += f" AND node_type NOT IN ({placeholders})"
params.extend(filtered_types)
async with self.conn.execute(sql, tuple(params)) as cursor:
rows = await cursor.fetchall()
results = []
for row in rows:
item = {
"node_id": row["node_id"],
"node_type": row["node_type"],
"data_id": row["data_id"],
"bind_data_id": row["bind_data_id"],
"depend_ids": [],
"data": json.loads(row["data"]) if row["data"] else None,
"origin_data": json.loads(row["origin_data"]) if row["origin_data"] else None,
"action": row["action"],
"status": row["status"],
"binding_status": row["binding_status"],
"error": row["error"],
"sync_log": row["sync_log"],
"context": json.loads(row["context"]) if row["context"] else {},
}
if item["bind_data_id"] and isinstance(item["context"], dict):
item["context"]["bind_data_id"] = item["bind_data_id"]
results.append(item)
return results
async def save_binding(self, scope: str, node_type: str, local_id: str, payload_dict: Dict[str, Any]) -> None:
assert self.conn is not None
remote_id = payload_dict.get("remote_id") if payload_dict else None
await self.conn.execute("""
INSERT OR REPLACE INTO bindings (scope, node_type, local_id, remote_id)
VALUES (?, ?, ?, ?)
""", (scope, node_type, local_id, remote_id))
await self.conn.commit()
async def save_bindings_bulk(self, scope: str, node_type: str, bindings: Dict[str, Optional[str]]) -> None:
assert self.conn is not None
if not bindings:
return
rows = [(scope, node_type, local_id, remote_id) for local_id, remote_id in bindings.items()]
await self.conn.executemany(
"INSERT OR REPLACE INTO bindings (scope, node_type, local_id, remote_id) VALUES (?, ?, ?, ?)",
rows,
)
await self.conn.commit()
async def delete_binding(self, scope: str, node_type: str, local_id: str) -> None:
assert self.conn is not None
await self.conn.execute(
"DELETE FROM bindings WHERE scope = ? AND node_type = ? AND local_id = ?",
(scope, node_type, local_id),
)
await self.conn.commit()
async def delete_bindings_bulk(self, scope: str, node_type: str, local_ids: List[str]) -> None:
assert self.conn is not None
if not local_ids:
return
rows = [(scope, node_type, local_id) for local_id in local_ids]
await self.conn.executemany(
"DELETE FROM bindings WHERE scope = ? AND node_type = ? AND local_id = ?",
rows,
)
await self.conn.commit()
async def load_bindings(self, scope: str, node_type: str) -> List[Dict[str, Any]]:
assert self.conn is not None
async with self.conn.execute(
"SELECT local_id, remote_id FROM bindings WHERE scope = ? AND node_type = ?",
(scope, node_type),
) as cursor:
rows = await cursor.fetchall()
return [{"local_id": r["local_id"], "payload": {"remote_id": r["remote_id"]}} for r in rows]
async def load_bind_data_id_overrides(self, node_type: str, scope: str) -> Dict[str, Optional[str]]:
assert self.conn is not None
sql = """
SELECT b.local_id AS src_node_id, rn.data_id AS bind_data_id
FROM bindings b
LEFT JOIN nodes rn
ON rn.node_id = b.remote_id
AND rn.node_type = b.node_type
AND rn.scope = b.scope
WHERE b.node_type = ?
AND b.scope = ?
UNION ALL
SELECT b.remote_id AS src_node_id, ln.data_id AS bind_data_id
FROM bindings b
LEFT JOIN nodes ln
ON ln.node_id = b.local_id
AND ln.node_type = b.node_type
AND ln.scope = b.scope
WHERE b.node_type = ?
AND b.scope = ?
AND b.remote_id IS NOT NULL
"""
async with self.conn.execute(sql, (node_type, scope, node_type, scope)) as cursor:
rows = await cursor.fetchall()
result: Dict[str, Optional[str]] = {}
for row in rows:
src_node_id = row["src_node_id"]
if not src_node_id:
continue
result[str(src_node_id)] = row["bind_data_id"]
return result
async def delete_scope_node_types(self, *, scope: str, node_types: List[str]) -> None:
assert self.conn is not None
filtered_types = [node_type for node_type in node_types if node_type]
if not filtered_types:
return
placeholders = ", ".join("?" for _ in filtered_types)
await self.conn.execute(
f"DELETE FROM nodes WHERE scope = ? AND node_type IN ({placeholders})",
(scope, *filtered_types),
)
await self.conn.execute(
f"DELETE FROM bindings WHERE scope = ? AND node_type IN ({placeholders})",
(scope, *filtered_types),
)
await self.conn.commit()
async def copy_nodes_between_scopes(
self,
*,
collection_id: str,
source_scope: str,
target_scope: str,
node_types: List[str],
) -> None:
assert self.conn is not None
if source_scope == target_scope or not node_types:
return
placeholders = ", ".join("?" for _ in node_types)
sql = f"""
INSERT OR REPLACE INTO nodes (
collection_id, scope, node_id, node_type, data_id, bind_data_id,
data, origin_data, action, status, binding_status, error, sync_log, context
)
SELECT
collection_id, ?, node_id, node_type, data_id, bind_data_id,
data, origin_data, action, status, binding_status, error, sync_log, context
FROM nodes
WHERE collection_id = ?
AND scope = ?
AND node_type IN ({placeholders})
"""
await self.conn.execute(sql, (target_scope, collection_id, source_scope, *node_types))
await self.conn.commit()
async def copy_bindings_between_scopes(
self,
*,
source_scope: str,
target_scope: str,
node_types: List[str],
) -> None:
assert self.conn is not None
if source_scope == target_scope or not node_types:
return
placeholders = ", ".join("?" for _ in node_types)
sql = f"""
INSERT OR REPLACE INTO bindings (scope, node_type, local_id, remote_id)
SELECT ?, node_type, local_id, remote_id
FROM bindings
WHERE scope = ?
AND node_type IN ({placeholders})
"""
await self.conn.execute(sql, (target_scope, source_scope, *node_types))
await self.conn.commit()
async def dump_to_runtime(self, filename: str) -> None:
assert self.conn is not None
runtime_path = os.path.join("_runtime", filename)
if os.path.exists(runtime_path):
os.remove(runtime_path)
await self.conn.execute("VACUUM INTO ?", (runtime_path,))
async def close(self) -> None:
if self.conn:
try:
await self.conn.close()
except Exception as exc:
print(f"⚠️ Error closing persistence: {exc}")
finally:
self.conn = None
@@ -0,0 +1,3 @@
from .persistence_backends.sqlite_backend import SQLitePersistenceBackend
__all__ = ["SQLitePersistenceBackend"]