strepsiades
559 lines
22 KiB
Python
559 lines
22 KiB
Python
import aiosqlite
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
from typing import Optional, Dict, List, Any, TYPE_CHECKING
|
|
|
|
if TYPE_CHECKING:
|
|
from .sync_node import SyncNode
|
|
|
|
class PersistenceBackend:
|
|
"""
|
|
Persistence 负责同步系统运行时数据的持久化存储（aiosqlite 实现）。
|
|
主要作为 DataCollection 和 BindingManager 的后端存储。
|
|
"""
|
|
def __init__(self, db_path: str = ":memory:", backend: str = "sqlite"):
|
|
self.db_path = db_path
|
|
self.backend = backend
|
|
self.conn: Optional[aiosqlite.Connection] = None
|
|
|
|
@staticmethod
|
|
def _is_read_only_sql(sql: str) -> bool:
|
|
stmt = (sql or "").strip().lower()
|
|
return stmt.startswith("select") or stmt.startswith("pragma") or stmt.startswith("with")
|
|
|
|
@classmethod
|
|
def query_records(
|
|
cls,
|
|
*,
|
|
backend: str,
|
|
db_path: str,
|
|
sql: str,
|
|
limit: int = 200,
|
|
) -> tuple[List[str], List[Dict[str, Any]]]:
|
|
if backend != "sqlite":
|
|
raise NotImplementedError(f"records query backend not supported yet: {backend}")
|
|
if not cls._is_read_only_sql(sql):
|
|
raise ValueError("Only read-only SQL is allowed")
|
|
|
|
path = db_path
|
|
if not os.path.exists(path):
|
|
raise FileNotFoundError(f"DB not found: {path}")
|
|
|
|
with sqlite3.connect(path) as conn:
|
|
conn.row_factory = sqlite3.Row
|
|
cursor = conn.execute(sql)
|
|
rows = cursor.fetchmany(max(1, min(limit, 2000)))
|
|
columns = [item[0] for item in (cursor.description or [])]
|
|
data = [{k: row[k] for k in columns} for row in rows]
|
|
return columns, data
|
|
|
|
@classmethod
|
|
def records_summary(
|
|
cls,
|
|
*,
|
|
backend: str,
|
|
db_path: str,
|
|
) -> Dict[str, Any]:
|
|
summary: Dict[str, Any] = {
|
|
"backend": backend,
|
|
"db_path": db_path,
|
|
"db_exists": os.path.exists(db_path),
|
|
"tables": [],
|
|
"sample_bindings": [],
|
|
}
|
|
if not os.path.exists(db_path):
|
|
return summary
|
|
if backend != "sqlite":
|
|
summary["error"] = f"records summary backend not supported yet: {backend}"
|
|
return summary
|
|
|
|
with sqlite3.connect(db_path) as conn:
|
|
conn.row_factory = sqlite3.Row
|
|
tables = conn.execute(
|
|
"SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name"
|
|
).fetchall()
|
|
table_names = [str(r["name"]) for r in tables]
|
|
|
|
for name in table_names:
|
|
count = conn.execute(f"SELECT COUNT(1) AS c FROM {name}").fetchone()["c"]
|
|
summary["tables"].append({"name": name, "count": int(count)})
|
|
|
|
if "bindings" in table_names:
|
|
sample = conn.execute(
|
|
"SELECT node_type, local_id, remote_id FROM bindings ORDER BY last_updated DESC LIMIT 50"
|
|
).fetchall()
|
|
summary["sample_bindings"] = [
|
|
{
|
|
"node_type": row["node_type"],
|
|
"local_id": row["local_id"],
|
|
"remote_id": row["remote_id"],
|
|
}
|
|
for row in sample
|
|
]
|
|
return summary
|
|
|
|
async def initialize(self):
|
|
"""异步初始化数据库连接和表结构"""
|
|
if self.backend != "sqlite":
|
|
raise NotImplementedError(f"Persistence backend not supported yet: {self.backend}")
|
|
|
|
# 确保目录存在
|
|
if self.db_path != ":memory:":
|
|
dir_path = os.path.dirname(self.db_path)
|
|
if dir_path:
|
|
os.makedirs(dir_path, exist_ok=True)
|
|
|
|
self.conn = await aiosqlite.connect(self.db_path)
|
|
self.conn.row_factory = aiosqlite.Row
|
|
await self._init_db()
|
|
|
|
async def _init_db(self):
|
|
# 1. 节点表: 仅用于持久化存储 DataCollection 中的 SyncNode
|
|
# 注意: depend_ids不持久化，每次从data实时计算
|
|
await self.conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS nodes (
|
|
collection_id TEXT NOT NULL,
|
|
scope TEXT NOT NULL DEFAULT 'global',
|
|
node_id TEXT NOT NULL,
|
|
node_type TEXT NOT NULL,
|
|
data_id TEXT,
|
|
bind_data_id TEXT,
|
|
data TEXT,
|
|
origin_data TEXT,
|
|
action TEXT,
|
|
status TEXT,
|
|
binding_status TEXT,
|
|
error TEXT,
|
|
sync_log TEXT,
|
|
context TEXT,
|
|
PRIMARY KEY (collection_id, scope, node_id)
|
|
)
|
|
""")
|
|
await self._ensure_nodes_schema()
|
|
|
|
# 2. 绑定表: 存储本地到远程的映射关系
|
|
# 只保留 local_id / remote_id 两字段（remote_id 可空）
|
|
await self._ensure_bindings_schema()
|
|
await self.conn.commit()
|
|
|
|
async def _ensure_nodes_schema(self) -> None:
|
|
cursor = await self.conn.execute("PRAGMA table_info(nodes)")
|
|
rows = await cursor.fetchall()
|
|
columns = [r[1] for r in rows] if rows else []
|
|
if "scope" not in columns:
|
|
await self.conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS nodes_new (
|
|
collection_id TEXT NOT NULL,
|
|
scope TEXT NOT NULL DEFAULT 'global',
|
|
node_id TEXT NOT NULL,
|
|
node_type TEXT NOT NULL,
|
|
data_id TEXT,
|
|
bind_data_id TEXT,
|
|
data TEXT,
|
|
origin_data TEXT,
|
|
action TEXT,
|
|
status TEXT,
|
|
binding_status TEXT,
|
|
error TEXT,
|
|
sync_log TEXT,
|
|
context TEXT,
|
|
PRIMARY KEY (collection_id, scope, node_id)
|
|
)
|
|
""")
|
|
await self.conn.execute("""
|
|
INSERT INTO nodes_new (
|
|
collection_id, scope, node_id, node_type, data_id, bind_data_id,
|
|
data, origin_data, action, status, binding_status, error, sync_log, context
|
|
)
|
|
SELECT
|
|
collection_id, 'global', node_id, node_type, data_id, bind_data_id,
|
|
data, origin_data, action, status, binding_status, error, sync_log, context
|
|
FROM nodes
|
|
""")
|
|
await self.conn.execute("DROP TABLE nodes")
|
|
await self.conn.execute("ALTER TABLE nodes_new RENAME TO nodes")
|
|
cursor = await self.conn.execute("PRAGMA table_info(nodes)")
|
|
rows = await cursor.fetchall()
|
|
columns = [r[1] for r in rows] if rows else []
|
|
if "bind_data_id" not in columns:
|
|
await self.conn.execute("ALTER TABLE nodes ADD COLUMN bind_data_id TEXT")
|
|
await self.conn.execute(
|
|
"CREATE INDEX IF NOT EXISTS idx_nodes_collection_scope_type ON nodes (collection_id, scope, node_type)"
|
|
)
|
|
await self.conn.execute(
|
|
"CREATE INDEX IF NOT EXISTS idx_nodes_collection_scope_data_id ON nodes (collection_id, scope, data_id)"
|
|
)
|
|
|
|
async def _ensure_bindings_schema(self) -> None:
|
|
"""确保 bindings 表为 (node_type, local_id, remote_id) 结构。
|
|
|
|
若检测到旧表含 payload 列，则进行迁移。
|
|
"""
|
|
cursor = await self.conn.execute("PRAGMA table_info(bindings)")
|
|
rows = await cursor.fetchall()
|
|
columns = [r[1] for r in rows] if rows else []
|
|
|
|
if not columns:
|
|
await self.conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS bindings (
|
|
scope TEXT NOT NULL DEFAULT 'global',
|
|
node_type TEXT NOT NULL,
|
|
local_id TEXT NOT NULL,
|
|
remote_id TEXT,
|
|
last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
PRIMARY KEY (scope, node_type, local_id)
|
|
)
|
|
""")
|
|
return
|
|
|
|
if "scope" not in columns or ("payload" in columns and "remote_id" not in columns):
|
|
# 迁移旧结构到 scope + remote_id 新结构
|
|
await self.conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS bindings_new (
|
|
scope TEXT NOT NULL DEFAULT 'global',
|
|
node_type TEXT NOT NULL,
|
|
local_id TEXT NOT NULL,
|
|
remote_id TEXT,
|
|
last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
PRIMARY KEY (scope, node_type, local_id)
|
|
)
|
|
""")
|
|
|
|
if "payload" in columns and "remote_id" not in columns:
|
|
async with self.conn.execute("SELECT node_type, local_id, payload FROM bindings") as cur:
|
|
old_rows = await cur.fetchall()
|
|
for row in old_rows:
|
|
try:
|
|
payload_dict = json.loads(row[2]) if row[2] else {}
|
|
except json.JSONDecodeError:
|
|
payload_dict = {}
|
|
remote_id = payload_dict.get("remote_id")
|
|
await self.conn.execute(
|
|
"INSERT OR REPLACE INTO bindings_new (scope, node_type, local_id, remote_id) VALUES (?, ?, ?, ?)",
|
|
("global", row[0], row[1], remote_id),
|
|
)
|
|
else:
|
|
async with self.conn.execute("SELECT node_type, local_id, remote_id FROM bindings") as cur:
|
|
old_rows = await cur.fetchall()
|
|
for row in old_rows:
|
|
await self.conn.execute(
|
|
"INSERT OR REPLACE INTO bindings_new (scope, node_type, local_id, remote_id) VALUES (?, ?, ?, ?)",
|
|
("global", row[0], row[1], row[2]),
|
|
)
|
|
|
|
await self.conn.execute("DROP TABLE bindings")
|
|
await self.conn.execute("ALTER TABLE bindings_new RENAME TO bindings")
|
|
await self.conn.execute(
|
|
"CREATE INDEX IF NOT EXISTS idx_bindings_scope_type_remote ON bindings (scope, node_type, remote_id)"
|
|
)
|
|
|
|
# --- DataCollection 支持 ---
|
|
|
|
async def save_node(self, collection_id: str, scope: str, node: "SyncNode"):
|
|
"""保存或更新单个节点（depend_ids不持久化，运行时从data实时计算）"""
|
|
bind_data_id = None
|
|
if isinstance(node.context, dict):
|
|
value = node.context.get("bind_data_id")
|
|
if value is not None and value != "":
|
|
bind_data_id = str(value)
|
|
await self.conn.execute("""
|
|
INSERT OR REPLACE INTO nodes (
|
|
collection_id, scope, node_id, node_type, data_id, bind_data_id,
|
|
data, origin_data, action, status, binding_status, error, sync_log, context
|
|
)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
""", (
|
|
collection_id,
|
|
scope,
|
|
node.node_id,
|
|
node.node_type,
|
|
node.data_id,
|
|
bind_data_id,
|
|
json.dumps(node.data) if node.data else None,
|
|
json.dumps(node.origin_data) if node.origin_data else None,
|
|
node.action.value,
|
|
node.status.value,
|
|
node.binding_status.value,
|
|
node.error,
|
|
node.sync_log,
|
|
json.dumps(node.context) if node.context else None
|
|
))
|
|
await self.conn.commit()
|
|
|
|
async def save_nodes_bulk(self, collection_id: str, scope: str, nodes: List["SyncNode"]):
|
|
"""批量保存或更新节点 (单次提交，depend_ids不持久化)"""
|
|
if not nodes:
|
|
return
|
|
rows = []
|
|
for node in nodes:
|
|
bind_data_id = None
|
|
if isinstance(node.context, dict):
|
|
value = node.context.get("bind_data_id")
|
|
if value is not None and value != "":
|
|
bind_data_id = str(value)
|
|
rows.append((
|
|
collection_id,
|
|
scope,
|
|
node.node_id,
|
|
node.node_type,
|
|
node.data_id,
|
|
bind_data_id,
|
|
json.dumps(node.data) if node.data else None,
|
|
json.dumps(node.origin_data) if node.origin_data else None,
|
|
node.action.value,
|
|
node.status.value,
|
|
node.binding_status.value,
|
|
node.error,
|
|
node.sync_log,
|
|
json.dumps(node.context) if node.context else None
|
|
))
|
|
|
|
await self.conn.executemany("""
|
|
INSERT OR REPLACE INTO nodes (
|
|
collection_id, scope, node_id, node_type, data_id, bind_data_id,
|
|
data, origin_data, action, status, binding_status, error, sync_log, context
|
|
)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
""", rows)
|
|
await self.conn.commit()
|
|
|
|
async def delete_node(self, collection_id: str, scope: str, node_id: str):
|
|
"""删除单个节点"""
|
|
await self.conn.execute(
|
|
"DELETE FROM nodes WHERE collection_id = ? AND scope = ? AND node_id = ?",
|
|
(collection_id, scope, node_id),
|
|
)
|
|
await self.conn.commit()
|
|
|
|
async def delete_nodes_bulk(self, collection_id: str, scope: str, node_ids: List[str]):
|
|
"""批量删除节点 (单次提交)"""
|
|
if not node_ids:
|
|
return
|
|
rows = [(collection_id, scope, node_id) for node_id in node_ids]
|
|
await self.conn.executemany(
|
|
"DELETE FROM nodes WHERE collection_id = ? AND scope = ? AND node_id = ?",
|
|
rows,
|
|
)
|
|
await self.conn.commit()
|
|
|
|
async def load_nodes(
|
|
self,
|
|
collection_id: str,
|
|
scope: str,
|
|
*,
|
|
exclude_node_types: Optional[List[str]] = None,
|
|
) -> List[Dict[str, Any]]:
|
|
"""从特定 collection 中加载所有节点（depend_ids初始化为空，稍后从data实时计算）"""
|
|
sql = "SELECT * FROM nodes WHERE collection_id = ? AND scope = ?"
|
|
params: List[Any] = [collection_id, scope]
|
|
filtered_types = [node_type for node_type in (exclude_node_types or []) if node_type]
|
|
if filtered_types:
|
|
placeholders = ", ".join("?" for _ in filtered_types)
|
|
sql += f" AND node_type NOT IN ({placeholders})"
|
|
params.extend(filtered_types)
|
|
|
|
async with self.conn.execute(sql, tuple(params)) as cursor:
|
|
rows = await cursor.fetchall()
|
|
results = []
|
|
for row in rows:
|
|
results.append({
|
|
"node_id": row["node_id"],
|
|
"node_type": row["node_type"],
|
|
"data_id": row["data_id"],
|
|
"bind_data_id": row["bind_data_id"],
|
|
"depend_ids": [], # 不从数据库加载，运行时从data实时计算
|
|
"data": json.loads(row["data"]) if row["data"] else None,
|
|
"origin_data": json.loads(row["origin_data"]) if row["origin_data"] else None,
|
|
"action": row["action"],
|
|
"status": row["status"],
|
|
"binding_status": row["binding_status"],
|
|
"error": row["error"],
|
|
"sync_log": row["sync_log"],
|
|
"context": json.loads(row["context"]) if row["context"] else {}
|
|
})
|
|
if results[-1]["bind_data_id"] and isinstance(results[-1]["context"], dict):
|
|
results[-1]["context"]["bind_data_id"] = results[-1]["bind_data_id"]
|
|
return results
|
|
|
|
# --- BindingManager 支持 ---
|
|
|
|
async def save_binding(self, scope: str, node_type: str, local_id: str, payload_dict: Dict[str, Any]):
|
|
"""保存或更新绑定关系及其元数据"""
|
|
remote_id = payload_dict.get("remote_id") if payload_dict else None
|
|
await self.conn.execute("""
|
|
INSERT OR REPLACE INTO bindings (scope, node_type, local_id, remote_id)
|
|
VALUES (?, ?, ?, ?)
|
|
""", (scope, node_type, local_id, remote_id))
|
|
await self.conn.commit()
|
|
|
|
async def save_bindings_bulk(self, scope: str, node_type: str, bindings: Dict[str, Optional[str]]):
|
|
"""批量保存或更新绑定关系 (单次提交)"""
|
|
if not bindings:
|
|
return
|
|
rows = []
|
|
for local_id, remote_id in bindings.items():
|
|
rows.append((scope, node_type, local_id, remote_id))
|
|
await self.conn.executemany(
|
|
"INSERT OR REPLACE INTO bindings (scope, node_type, local_id, remote_id) VALUES (?, ?, ?, ?)",
|
|
rows,
|
|
)
|
|
await self.conn.commit()
|
|
|
|
async def delete_binding(self, scope: str, node_type: str, local_id: str):
|
|
"""删除绑定关系"""
|
|
await self.conn.execute(
|
|
"DELETE FROM bindings WHERE scope = ? AND node_type = ? AND local_id = ?",
|
|
(scope, node_type, local_id)
|
|
)
|
|
await self.conn.commit()
|
|
|
|
async def delete_bindings_bulk(self, scope: str, node_type: str, local_ids: List[str]):
|
|
"""批量删除绑定关系 (单次提交)"""
|
|
if not local_ids:
|
|
return
|
|
rows = [(scope, node_type, local_id) for local_id in local_ids]
|
|
await self.conn.executemany(
|
|
"DELETE FROM bindings WHERE scope = ? AND node_type = ? AND local_id = ?",
|
|
rows,
|
|
)
|
|
await self.conn.commit()
|
|
|
|
async def load_bindings(self, scope: str, node_type: str) -> List[Dict[str, Any]]:
|
|
"""回写该类型下的所有绑定关系到内存"""
|
|
async with self.conn.execute(
|
|
"SELECT local_id, remote_id FROM bindings WHERE scope = ? AND node_type = ?",
|
|
(scope, node_type)
|
|
) as cursor:
|
|
rows = await cursor.fetchall()
|
|
return [
|
|
{"local_id": r["local_id"], "payload": {"remote_id": r["remote_id"]}}
|
|
for r in rows
|
|
]
|
|
|
|
async def load_bind_data_id_overrides(self, node_type: str, scope: str) -> Dict[str, Optional[str]]:
|
|
"""基于 bindings 表计算 node_id -> bind_data_id 覆盖映射。
|
|
|
|
规则：
|
|
- local 节点的 bind_data_id 取其绑定 remote 节点的 data_id
|
|
- remote 节点的 bind_data_id 取其绑定 local 节点的 data_id
|
|
"""
|
|
sql = """
|
|
SELECT b.local_id AS src_node_id, rn.data_id AS bind_data_id
|
|
FROM bindings b
|
|
LEFT JOIN nodes rn
|
|
ON rn.node_id = b.remote_id
|
|
AND rn.node_type = b.node_type
|
|
AND rn.scope = b.scope
|
|
WHERE b.node_type = ?
|
|
AND b.scope = ?
|
|
|
|
UNION ALL
|
|
|
|
SELECT b.remote_id AS src_node_id, ln.data_id AS bind_data_id
|
|
FROM bindings b
|
|
LEFT JOIN nodes ln
|
|
ON ln.node_id = b.local_id
|
|
AND ln.node_type = b.node_type
|
|
AND ln.scope = b.scope
|
|
WHERE b.node_type = ?
|
|
AND b.scope = ?
|
|
AND b.remote_id IS NOT NULL
|
|
"""
|
|
async with self.conn.execute(sql, (node_type, scope, node_type, scope)) as cursor:
|
|
rows = await cursor.fetchall()
|
|
result: Dict[str, Optional[str]] = {}
|
|
for row in rows:
|
|
src_node_id = row["src_node_id"]
|
|
if not src_node_id:
|
|
continue
|
|
result[str(src_node_id)] = row["bind_data_id"]
|
|
return result
|
|
|
|
async def delete_scope_node_types(
|
|
self,
|
|
*,
|
|
scope: str,
|
|
node_types: List[str],
|
|
) -> None:
|
|
filtered_types = [node_type for node_type in node_types if node_type]
|
|
if not filtered_types:
|
|
return
|
|
|
|
placeholders = ", ".join("?" for _ in filtered_types)
|
|
await self.conn.execute(
|
|
f"DELETE FROM nodes WHERE scope = ? AND node_type IN ({placeholders})",
|
|
(scope, *filtered_types),
|
|
)
|
|
await self.conn.execute(
|
|
f"DELETE FROM bindings WHERE scope = ? AND node_type IN ({placeholders})",
|
|
(scope, *filtered_types),
|
|
)
|
|
await self.conn.commit()
|
|
|
|
async def copy_nodes_between_scopes(
|
|
self,
|
|
*,
|
|
collection_id: str,
|
|
source_scope: str,
|
|
target_scope: str,
|
|
node_types: List[str],
|
|
) -> None:
|
|
if source_scope == target_scope or not node_types:
|
|
return
|
|
placeholders = ", ".join("?" for _ in node_types)
|
|
sql = f"""
|
|
INSERT OR REPLACE INTO nodes (
|
|
collection_id, scope, node_id, node_type, data_id, bind_data_id,
|
|
data, origin_data, action, status, binding_status, error, sync_log, context
|
|
)
|
|
SELECT
|
|
collection_id, ?, node_id, node_type, data_id, bind_data_id,
|
|
data, origin_data, action, status, binding_status, error, sync_log, context
|
|
FROM nodes
|
|
WHERE collection_id = ?
|
|
AND scope = ?
|
|
AND node_type IN ({placeholders})
|
|
"""
|
|
await self.conn.execute(sql, (target_scope, collection_id, source_scope, *node_types))
|
|
await self.conn.commit()
|
|
|
|
async def copy_bindings_between_scopes(
|
|
self,
|
|
*,
|
|
source_scope: str,
|
|
target_scope: str,
|
|
node_types: List[str],
|
|
) -> None:
|
|
if source_scope == target_scope or not node_types:
|
|
return
|
|
placeholders = ", ".join("?" for _ in node_types)
|
|
sql = f"""
|
|
INSERT OR REPLACE INTO bindings (scope, node_type, local_id, remote_id)
|
|
SELECT ?, node_type, local_id, remote_id
|
|
FROM bindings
|
|
WHERE scope = ?
|
|
AND node_type IN ({placeholders})
|
|
"""
|
|
await self.conn.execute(sql, (target_scope, source_scope, *node_types))
|
|
await self.conn.commit()
|
|
|
|
async def dump_to_runtime(self, filename: str):
|
|
"""将当前状态转储到 _runtime 文件夹 (使用 VACUUM INTO)"""
|
|
runtime_path = os.path.join("_runtime", filename)
|
|
if os.path.exists(runtime_path):
|
|
os.remove(runtime_path)
|
|
|
|
# VACUUM INTO 是备份 SQLite 的推荐方式 (SQLite 3.27+)
|
|
await self.conn.execute("VACUUM INTO ?", (runtime_path,))
|
|
|
|
async def close(self):
|
|
"""关闭数据库连接（包含异常处理）"""
|
|
if self.conn:
|
|
try:
|
|
await self.conn.close()
|
|
except Exception as e:
|
|
print(f"⚠️ Error closing persistence: {e}")
|
|
finally:
|
|
self.conn = None
|