将persist和后端分离，方便替换为其他数据库
This commit is contained in:
strepsiades
@@ -0,0 +1,3 @@
|
||||
from .sqlite_backend import SQLitePersistenceBackend
|
||||
|
||||
__all__ = ["SQLitePersistenceBackend"]
|
||||
@@ -0,0 +1,464 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import aiosqlite
|
||||
import json
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional, TYPE_CHECKING
|
||||
|
||||
from ...config.persist_config import PersistConfig
|
||||
from ..persistence import PersistenceBackend
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..sync_node import SyncNode
|
||||
|
||||
|
||||
class SQLitePersistenceBackend(PersistenceBackend):
|
||||
"""SQLite implementation of the persistence contract."""
|
||||
|
||||
backend_name = "sqlite"
|
||||
|
||||
def __init__(self, persist_config: PersistConfig | str | None = None, **_: Any) -> None:
|
||||
if isinstance(persist_config, PersistConfig):
|
||||
self.persist_config = persist_config
|
||||
self.db_path = persist_config.db_path
|
||||
elif isinstance(persist_config, str):
|
||||
self.persist_config = None
|
||||
self.db_path = persist_config
|
||||
else:
|
||||
self.persist_config = None
|
||||
self.db_path = ":memory:"
|
||||
self.conn: Optional[aiosqlite.Connection] = None
|
||||
|
||||
async def initialize(self) -> None:
|
||||
if self.db_path != ":memory:":
|
||||
dir_path = os.path.dirname(self.db_path)
|
||||
if dir_path:
|
||||
os.makedirs(dir_path, exist_ok=True)
|
||||
|
||||
self.conn = await aiosqlite.connect(self.db_path)
|
||||
self.conn.row_factory = aiosqlite.Row
|
||||
await self._init_db()
|
||||
|
||||
async def _init_db(self) -> None:
|
||||
assert self.conn is not None
|
||||
await self.conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS nodes (
|
||||
collection_id TEXT NOT NULL,
|
||||
scope TEXT NOT NULL DEFAULT 'global',
|
||||
node_id TEXT NOT NULL,
|
||||
node_type TEXT NOT NULL,
|
||||
data_id TEXT,
|
||||
bind_data_id TEXT,
|
||||
data TEXT,
|
||||
origin_data TEXT,
|
||||
action TEXT,
|
||||
status TEXT,
|
||||
binding_status TEXT,
|
||||
error TEXT,
|
||||
sync_log TEXT,
|
||||
context TEXT,
|
||||
PRIMARY KEY (collection_id, scope, node_id)
|
||||
)
|
||||
""")
|
||||
await self._ensure_nodes_schema()
|
||||
await self._ensure_bindings_schema()
|
||||
await self.conn.commit()
|
||||
|
||||
async def _ensure_nodes_schema(self) -> None:
|
||||
assert self.conn is not None
|
||||
cursor = await self.conn.execute("PRAGMA table_info(nodes)")
|
||||
rows = await cursor.fetchall()
|
||||
columns = [r[1] for r in rows] if rows else []
|
||||
if "scope" not in columns:
|
||||
await self.conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS nodes_new (
|
||||
collection_id TEXT NOT NULL,
|
||||
scope TEXT NOT NULL DEFAULT 'global',
|
||||
node_id TEXT NOT NULL,
|
||||
node_type TEXT NOT NULL,
|
||||
data_id TEXT,
|
||||
bind_data_id TEXT,
|
||||
data TEXT,
|
||||
origin_data TEXT,
|
||||
action TEXT,
|
||||
status TEXT,
|
||||
binding_status TEXT,
|
||||
error TEXT,
|
||||
sync_log TEXT,
|
||||
context TEXT,
|
||||
PRIMARY KEY (collection_id, scope, node_id)
|
||||
)
|
||||
""")
|
||||
await self.conn.execute("""
|
||||
INSERT INTO nodes_new (
|
||||
collection_id, scope, node_id, node_type, data_id, bind_data_id,
|
||||
data, origin_data, action, status, binding_status, error, sync_log, context
|
||||
)
|
||||
SELECT
|
||||
collection_id, 'global', node_id, node_type, data_id, bind_data_id,
|
||||
data, origin_data, action, status, binding_status, error, sync_log, context
|
||||
FROM nodes
|
||||
""")
|
||||
await self.conn.execute("DROP TABLE nodes")
|
||||
await self.conn.execute("ALTER TABLE nodes_new RENAME TO nodes")
|
||||
cursor = await self.conn.execute("PRAGMA table_info(nodes)")
|
||||
rows = await cursor.fetchall()
|
||||
columns = [r[1] for r in rows] if rows else []
|
||||
if "bind_data_id" not in columns:
|
||||
await self.conn.execute("ALTER TABLE nodes ADD COLUMN bind_data_id TEXT")
|
||||
await self.conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_nodes_collection_scope_type ON nodes (collection_id, scope, node_type)"
|
||||
)
|
||||
await self.conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_nodes_collection_scope_data_id ON nodes (collection_id, scope, data_id)"
|
||||
)
|
||||
|
||||
async def _ensure_bindings_schema(self) -> None:
|
||||
assert self.conn is not None
|
||||
cursor = await self.conn.execute("PRAGMA table_info(bindings)")
|
||||
rows = await cursor.fetchall()
|
||||
columns = [r[1] for r in rows] if rows else []
|
||||
|
||||
if not columns:
|
||||
await self.conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS bindings (
|
||||
scope TEXT NOT NULL DEFAULT 'global',
|
||||
node_type TEXT NOT NULL,
|
||||
local_id TEXT NOT NULL,
|
||||
remote_id TEXT,
|
||||
last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (scope, node_type, local_id)
|
||||
)
|
||||
""")
|
||||
return
|
||||
|
||||
if "scope" not in columns or ("payload" in columns and "remote_id" not in columns):
|
||||
await self.conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS bindings_new (
|
||||
scope TEXT NOT NULL DEFAULT 'global',
|
||||
node_type TEXT NOT NULL,
|
||||
local_id TEXT NOT NULL,
|
||||
remote_id TEXT,
|
||||
last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (scope, node_type, local_id)
|
||||
)
|
||||
""")
|
||||
|
||||
if "payload" in columns and "remote_id" not in columns:
|
||||
async with self.conn.execute("SELECT node_type, local_id, payload FROM bindings") as cur:
|
||||
old_rows = await cur.fetchall()
|
||||
for row in old_rows:
|
||||
try:
|
||||
payload_dict = json.loads(row[2]) if row[2] else {}
|
||||
except json.JSONDecodeError:
|
||||
payload_dict = {}
|
||||
remote_id = payload_dict.get("remote_id")
|
||||
await self.conn.execute(
|
||||
"INSERT OR REPLACE INTO bindings_new (scope, node_type, local_id, remote_id) VALUES (?, ?, ?, ?)",
|
||||
("global", row[0], row[1], remote_id),
|
||||
)
|
||||
else:
|
||||
async with self.conn.execute("SELECT node_type, local_id, remote_id FROM bindings") as cur:
|
||||
old_rows = await cur.fetchall()
|
||||
for row in old_rows:
|
||||
await self.conn.execute(
|
||||
"INSERT OR REPLACE INTO bindings_new (scope, node_type, local_id, remote_id) VALUES (?, ?, ?, ?)",
|
||||
("global", row[0], row[1], row[2]),
|
||||
)
|
||||
|
||||
await self.conn.execute("DROP TABLE bindings")
|
||||
await self.conn.execute("ALTER TABLE bindings_new RENAME TO bindings")
|
||||
await self.conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_bindings_scope_type_remote ON bindings (scope, node_type, remote_id)"
|
||||
)
|
||||
|
||||
async def save_node(self, collection_id: str, scope: str, node: "SyncNode") -> None:
|
||||
assert self.conn is not None
|
||||
bind_data_id = None
|
||||
if isinstance(node.context, dict):
|
||||
value = node.context.get("bind_data_id")
|
||||
if value is not None and value != "":
|
||||
bind_data_id = str(value)
|
||||
await self.conn.execute("""
|
||||
INSERT OR REPLACE INTO nodes (
|
||||
collection_id, scope, node_id, node_type, data_id, bind_data_id,
|
||||
data, origin_data, action, status, binding_status, error, sync_log, context
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
collection_id,
|
||||
scope,
|
||||
node.node_id,
|
||||
node.node_type,
|
||||
node.data_id,
|
||||
bind_data_id,
|
||||
json.dumps(node.data) if node.data else None,
|
||||
json.dumps(node.origin_data) if node.origin_data else None,
|
||||
node.action.value,
|
||||
node.status.value,
|
||||
node.binding_status.value,
|
||||
node.error,
|
||||
node.sync_log,
|
||||
json.dumps(node.context) if node.context else None,
|
||||
))
|
||||
await self.conn.commit()
|
||||
|
||||
async def save_nodes_bulk(self, collection_id: str, scope: str, nodes: List["SyncNode"]) -> None:
|
||||
assert self.conn is not None
|
||||
if not nodes:
|
||||
return
|
||||
rows = []
|
||||
for node in nodes:
|
||||
bind_data_id = None
|
||||
if isinstance(node.context, dict):
|
||||
value = node.context.get("bind_data_id")
|
||||
if value is not None and value != "":
|
||||
bind_data_id = str(value)
|
||||
rows.append((
|
||||
collection_id,
|
||||
scope,
|
||||
node.node_id,
|
||||
node.node_type,
|
||||
node.data_id,
|
||||
bind_data_id,
|
||||
json.dumps(node.data) if node.data else None,
|
||||
json.dumps(node.origin_data) if node.origin_data else None,
|
||||
node.action.value,
|
||||
node.status.value,
|
||||
node.binding_status.value,
|
||||
node.error,
|
||||
node.sync_log,
|
||||
json.dumps(node.context) if node.context else None,
|
||||
))
|
||||
|
||||
await self.conn.executemany("""
|
||||
INSERT OR REPLACE INTO nodes (
|
||||
collection_id, scope, node_id, node_type, data_id, bind_data_id,
|
||||
data, origin_data, action, status, binding_status, error, sync_log, context
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", rows)
|
||||
await self.conn.commit()
|
||||
|
||||
async def delete_node(self, collection_id: str, scope: str, node_id: str) -> None:
|
||||
assert self.conn is not None
|
||||
await self.conn.execute(
|
||||
"DELETE FROM nodes WHERE collection_id = ? AND scope = ? AND node_id = ?",
|
||||
(collection_id, scope, node_id),
|
||||
)
|
||||
await self.conn.commit()
|
||||
|
||||
async def delete_nodes_bulk(self, collection_id: str, scope: str, node_ids: List[str]) -> None:
|
||||
assert self.conn is not None
|
||||
if not node_ids:
|
||||
return
|
||||
rows = [(collection_id, scope, node_id) for node_id in node_ids]
|
||||
await self.conn.executemany(
|
||||
"DELETE FROM nodes WHERE collection_id = ? AND scope = ? AND node_id = ?",
|
||||
rows,
|
||||
)
|
||||
await self.conn.commit()
|
||||
|
||||
async def load_nodes(
|
||||
self,
|
||||
collection_id: str,
|
||||
scope: str,
|
||||
*,
|
||||
exclude_node_types: Optional[List[str]] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
assert self.conn is not None
|
||||
sql = "SELECT * FROM nodes WHERE collection_id = ? AND scope = ?"
|
||||
params: List[Any] = [collection_id, scope]
|
||||
filtered_types = [node_type for node_type in (exclude_node_types or []) if node_type]
|
||||
if filtered_types:
|
||||
placeholders = ", ".join("?" for _ in filtered_types)
|
||||
sql += f" AND node_type NOT IN ({placeholders})"
|
||||
params.extend(filtered_types)
|
||||
|
||||
async with self.conn.execute(sql, tuple(params)) as cursor:
|
||||
rows = await cursor.fetchall()
|
||||
results = []
|
||||
for row in rows:
|
||||
item = {
|
||||
"node_id": row["node_id"],
|
||||
"node_type": row["node_type"],
|
||||
"data_id": row["data_id"],
|
||||
"bind_data_id": row["bind_data_id"],
|
||||
"depend_ids": [],
|
||||
"data": json.loads(row["data"]) if row["data"] else None,
|
||||
"origin_data": json.loads(row["origin_data"]) if row["origin_data"] else None,
|
||||
"action": row["action"],
|
||||
"status": row["status"],
|
||||
"binding_status": row["binding_status"],
|
||||
"error": row["error"],
|
||||
"sync_log": row["sync_log"],
|
||||
"context": json.loads(row["context"]) if row["context"] else {},
|
||||
}
|
||||
if item["bind_data_id"] and isinstance(item["context"], dict):
|
||||
item["context"]["bind_data_id"] = item["bind_data_id"]
|
||||
results.append(item)
|
||||
return results
|
||||
|
||||
async def save_binding(self, scope: str, node_type: str, local_id: str, payload_dict: Dict[str, Any]) -> None:
|
||||
assert self.conn is not None
|
||||
remote_id = payload_dict.get("remote_id") if payload_dict else None
|
||||
await self.conn.execute("""
|
||||
INSERT OR REPLACE INTO bindings (scope, node_type, local_id, remote_id)
|
||||
VALUES (?, ?, ?, ?)
|
||||
""", (scope, node_type, local_id, remote_id))
|
||||
await self.conn.commit()
|
||||
|
||||
async def save_bindings_bulk(self, scope: str, node_type: str, bindings: Dict[str, Optional[str]]) -> None:
|
||||
assert self.conn is not None
|
||||
if not bindings:
|
||||
return
|
||||
rows = [(scope, node_type, local_id, remote_id) for local_id, remote_id in bindings.items()]
|
||||
await self.conn.executemany(
|
||||
"INSERT OR REPLACE INTO bindings (scope, node_type, local_id, remote_id) VALUES (?, ?, ?, ?)",
|
||||
rows,
|
||||
)
|
||||
await self.conn.commit()
|
||||
|
||||
async def delete_binding(self, scope: str, node_type: str, local_id: str) -> None:
|
||||
assert self.conn is not None
|
||||
await self.conn.execute(
|
||||
"DELETE FROM bindings WHERE scope = ? AND node_type = ? AND local_id = ?",
|
||||
(scope, node_type, local_id),
|
||||
)
|
||||
await self.conn.commit()
|
||||
|
||||
async def delete_bindings_bulk(self, scope: str, node_type: str, local_ids: List[str]) -> None:
|
||||
assert self.conn is not None
|
||||
if not local_ids:
|
||||
return
|
||||
rows = [(scope, node_type, local_id) for local_id in local_ids]
|
||||
await self.conn.executemany(
|
||||
"DELETE FROM bindings WHERE scope = ? AND node_type = ? AND local_id = ?",
|
||||
rows,
|
||||
)
|
||||
await self.conn.commit()
|
||||
|
||||
async def load_bindings(self, scope: str, node_type: str) -> List[Dict[str, Any]]:
|
||||
assert self.conn is not None
|
||||
async with self.conn.execute(
|
||||
"SELECT local_id, remote_id FROM bindings WHERE scope = ? AND node_type = ?",
|
||||
(scope, node_type),
|
||||
) as cursor:
|
||||
rows = await cursor.fetchall()
|
||||
return [{"local_id": r["local_id"], "payload": {"remote_id": r["remote_id"]}} for r in rows]
|
||||
|
||||
async def load_bind_data_id_overrides(self, node_type: str, scope: str) -> Dict[str, Optional[str]]:
|
||||
assert self.conn is not None
|
||||
sql = """
|
||||
SELECT b.local_id AS src_node_id, rn.data_id AS bind_data_id
|
||||
FROM bindings b
|
||||
LEFT JOIN nodes rn
|
||||
ON rn.node_id = b.remote_id
|
||||
AND rn.node_type = b.node_type
|
||||
AND rn.scope = b.scope
|
||||
WHERE b.node_type = ?
|
||||
AND b.scope = ?
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT b.remote_id AS src_node_id, ln.data_id AS bind_data_id
|
||||
FROM bindings b
|
||||
LEFT JOIN nodes ln
|
||||
ON ln.node_id = b.local_id
|
||||
AND ln.node_type = b.node_type
|
||||
AND ln.scope = b.scope
|
||||
WHERE b.node_type = ?
|
||||
AND b.scope = ?
|
||||
AND b.remote_id IS NOT NULL
|
||||
"""
|
||||
async with self.conn.execute(sql, (node_type, scope, node_type, scope)) as cursor:
|
||||
rows = await cursor.fetchall()
|
||||
result: Dict[str, Optional[str]] = {}
|
||||
for row in rows:
|
||||
src_node_id = row["src_node_id"]
|
||||
if not src_node_id:
|
||||
continue
|
||||
result[str(src_node_id)] = row["bind_data_id"]
|
||||
return result
|
||||
|
||||
async def delete_scope_node_types(self, *, scope: str, node_types: List[str]) -> None:
|
||||
assert self.conn is not None
|
||||
filtered_types = [node_type for node_type in node_types if node_type]
|
||||
if not filtered_types:
|
||||
return
|
||||
|
||||
placeholders = ", ".join("?" for _ in filtered_types)
|
||||
await self.conn.execute(
|
||||
f"DELETE FROM nodes WHERE scope = ? AND node_type IN ({placeholders})",
|
||||
(scope, *filtered_types),
|
||||
)
|
||||
await self.conn.execute(
|
||||
f"DELETE FROM bindings WHERE scope = ? AND node_type IN ({placeholders})",
|
||||
(scope, *filtered_types),
|
||||
)
|
||||
await self.conn.commit()
|
||||
|
||||
async def copy_nodes_between_scopes(
|
||||
self,
|
||||
*,
|
||||
collection_id: str,
|
||||
source_scope: str,
|
||||
target_scope: str,
|
||||
node_types: List[str],
|
||||
) -> None:
|
||||
assert self.conn is not None
|
||||
if source_scope == target_scope or not node_types:
|
||||
return
|
||||
placeholders = ", ".join("?" for _ in node_types)
|
||||
sql = f"""
|
||||
INSERT OR REPLACE INTO nodes (
|
||||
collection_id, scope, node_id, node_type, data_id, bind_data_id,
|
||||
data, origin_data, action, status, binding_status, error, sync_log, context
|
||||
)
|
||||
SELECT
|
||||
collection_id, ?, node_id, node_type, data_id, bind_data_id,
|
||||
data, origin_data, action, status, binding_status, error, sync_log, context
|
||||
FROM nodes
|
||||
WHERE collection_id = ?
|
||||
AND scope = ?
|
||||
AND node_type IN ({placeholders})
|
||||
"""
|
||||
await self.conn.execute(sql, (target_scope, collection_id, source_scope, *node_types))
|
||||
await self.conn.commit()
|
||||
|
||||
async def copy_bindings_between_scopes(
|
||||
self,
|
||||
*,
|
||||
source_scope: str,
|
||||
target_scope: str,
|
||||
node_types: List[str],
|
||||
) -> None:
|
||||
assert self.conn is not None
|
||||
if source_scope == target_scope or not node_types:
|
||||
return
|
||||
placeholders = ", ".join("?" for _ in node_types)
|
||||
sql = f"""
|
||||
INSERT OR REPLACE INTO bindings (scope, node_type, local_id, remote_id)
|
||||
SELECT ?, node_type, local_id, remote_id
|
||||
FROM bindings
|
||||
WHERE scope = ?
|
||||
AND node_type IN ({placeholders})
|
||||
"""
|
||||
await self.conn.execute(sql, (target_scope, source_scope, *node_types))
|
||||
await self.conn.commit()
|
||||
|
||||
async def dump_to_runtime(self, filename: str) -> None:
|
||||
assert self.conn is not None
|
||||
runtime_path = os.path.join("_runtime", filename)
|
||||
if os.path.exists(runtime_path):
|
||||
os.remove(runtime_path)
|
||||
await self.conn.execute("VACUUM INTO ?", (runtime_path,))
|
||||
|
||||
async def close(self) -> None:
|
||||
if self.conn:
|
||||
try:
|
||||
await self.conn.close()
|
||||
except Exception as exc:
|
||||
print(f"⚠️ Error closing persistence: {exc}")
|
||||
finally:
|
||||
self.conn = None
|
||||
Reference in New Issue
Block a user