import aiosqlite import json import os import sqlite3 from typing import Optional, Dict, List, Any, TYPE_CHECKING if TYPE_CHECKING: from .sync_node import SyncNode class PersistenceBackend: """ Persistence 负责同步系统运行时数据的持久化存储(aiosqlite 实现)。 主要作为 DataCollection 和 BindingManager 的后端存储。 """ def __init__(self, db_path: str = ":memory:", backend: str = "sqlite"): self.db_path = db_path self.backend = backend self.conn: Optional[aiosqlite.Connection] = None @staticmethod def _is_read_only_sql(sql: str) -> bool: stmt = (sql or "").strip().lower() return stmt.startswith("select") or stmt.startswith("pragma") or stmt.startswith("with") @classmethod def query_records( cls, *, backend: str, db_path: str, sql: str, limit: int = 200, ) -> tuple[List[str], List[Dict[str, Any]]]: if backend != "sqlite": raise NotImplementedError(f"records query backend not supported yet: {backend}") if not cls._is_read_only_sql(sql): raise ValueError("Only read-only SQL is allowed") path = db_path if not os.path.exists(path): raise FileNotFoundError(f"DB not found: {path}") with sqlite3.connect(path) as conn: conn.row_factory = sqlite3.Row cursor = conn.execute(sql) rows = cursor.fetchmany(max(1, min(limit, 2000))) columns = [item[0] for item in (cursor.description or [])] data = [{k: row[k] for k in columns} for row in rows] return columns, data @classmethod def records_summary( cls, *, backend: str, db_path: str, ) -> Dict[str, Any]: summary: Dict[str, Any] = { "backend": backend, "db_path": db_path, "db_exists": os.path.exists(db_path), "tables": [], "sample_bindings": [], } if not os.path.exists(db_path): return summary if backend != "sqlite": summary["error"] = f"records summary backend not supported yet: {backend}" return summary with sqlite3.connect(db_path) as conn: conn.row_factory = sqlite3.Row tables = conn.execute( "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name" ).fetchall() table_names = [str(r["name"]) for r in tables] for name in table_names: count = conn.execute(f"SELECT COUNT(1) AS c FROM {name}").fetchone()["c"] summary["tables"].append({"name": name, "count": int(count)}) if "bindings" in table_names: sample = conn.execute( "SELECT node_type, local_id, remote_id FROM bindings ORDER BY last_updated DESC LIMIT 50" ).fetchall() summary["sample_bindings"] = [ { "node_type": row["node_type"], "local_id": row["local_id"], "remote_id": row["remote_id"], } for row in sample ] return summary async def initialize(self): """异步初始化数据库连接和表结构""" if self.backend != "sqlite": raise NotImplementedError(f"Persistence backend not supported yet: {self.backend}") # 确保目录存在 if self.db_path != ":memory:": dir_path = os.path.dirname(self.db_path) if dir_path: os.makedirs(dir_path, exist_ok=True) self.conn = await aiosqlite.connect(self.db_path) self.conn.row_factory = aiosqlite.Row await self._init_db() async def _init_db(self): # 1. 节点表: 仅用于持久化存储 DataCollection 中的 SyncNode # 注意: depend_ids不持久化,每次从data实时计算 await self.conn.execute(""" CREATE TABLE IF NOT EXISTS nodes ( collection_id TEXT NOT NULL, node_id TEXT NOT NULL, node_type TEXT NOT NULL, data_id TEXT, bind_data_id TEXT, data TEXT, origin_data TEXT, action TEXT, status TEXT, binding_status TEXT, error TEXT, sync_log TEXT, context TEXT, PRIMARY KEY (collection_id, node_id) ) """) await self._ensure_nodes_schema() # 2. 绑定表: 存储本地到远程的映射关系 # 只保留 local_id / remote_id 两字段(remote_id 可空) await self._ensure_bindings_schema() await self.conn.commit() async def _ensure_nodes_schema(self) -> None: cursor = await self.conn.execute("PRAGMA table_info(nodes)") rows = await cursor.fetchall() columns = [r[1] for r in rows] if rows else [] if "bind_data_id" not in columns: await self.conn.execute("ALTER TABLE nodes ADD COLUMN bind_data_id TEXT") async def _ensure_bindings_schema(self) -> None: """确保 bindings 表为 (node_type, local_id, remote_id) 结构。 若检测到旧表含 payload 列,则进行迁移。 """ cursor = await self.conn.execute("PRAGMA table_info(bindings)") rows = await cursor.fetchall() columns = [r[1] for r in rows] if rows else [] if not columns: await self.conn.execute(""" CREATE TABLE IF NOT EXISTS bindings ( node_type TEXT NOT NULL, local_id TEXT NOT NULL, remote_id TEXT, last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (node_type, local_id) ) """) return if "payload" in columns and "remote_id" not in columns: # 迁移旧结构 payload -> remote_id await self.conn.execute(""" CREATE TABLE IF NOT EXISTS bindings_new ( node_type TEXT NOT NULL, local_id TEXT NOT NULL, remote_id TEXT, last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (node_type, local_id) ) """) async with self.conn.execute("SELECT node_type, local_id, payload FROM bindings") as cur: old_rows = await cur.fetchall() for row in old_rows: try: payload_dict = json.loads(row[2]) if row[2] else {} except json.JSONDecodeError: payload_dict = {} remote_id = payload_dict.get("remote_id") await self.conn.execute( "INSERT OR REPLACE INTO bindings_new (node_type, local_id, remote_id) VALUES (?, ?, ?)", (row[0], row[1], remote_id), ) await self.conn.execute("DROP TABLE bindings") await self.conn.execute("ALTER TABLE bindings_new RENAME TO bindings") # --- DataCollection 支持 --- async def save_node(self, collection_id: str, node: "SyncNode"): """保存或更新单个节点(depend_ids不持久化,运行时从data实时计算)""" bind_data_id = None if isinstance(node.context, dict): value = node.context.get("bind_data_id") if value is not None and value != "": bind_data_id = str(value) await self.conn.execute(""" INSERT OR REPLACE INTO nodes ( collection_id, node_id, node_type, data_id, bind_data_id, data, origin_data, action, status, binding_status, error, sync_log, context ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( collection_id, node.node_id, node.node_type, node.data_id, bind_data_id, json.dumps(node.data) if node.data else None, json.dumps(node.origin_data) if node.origin_data else None, node.action.value, node.status.value, node.binding_status.value, node.error, node.sync_log, json.dumps(node.context) if node.context else None )) await self.conn.commit() async def save_nodes_bulk(self, collection_id: str, nodes: List["SyncNode"]): """批量保存或更新节点 (单次提交,depend_ids不持久化)""" if not nodes: return rows = [] for node in nodes: bind_data_id = None if isinstance(node.context, dict): value = node.context.get("bind_data_id") if value is not None and value != "": bind_data_id = str(value) rows.append(( collection_id, node.node_id, node.node_type, node.data_id, bind_data_id, json.dumps(node.data) if node.data else None, json.dumps(node.origin_data) if node.origin_data else None, node.action.value, node.status.value, node.binding_status.value, node.error, node.sync_log, json.dumps(node.context) if node.context else None )) await self.conn.executemany(""" INSERT OR REPLACE INTO nodes ( collection_id, node_id, node_type, data_id, bind_data_id, data, origin_data, action, status, binding_status, error, sync_log, context ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, rows) await self.conn.commit() async def delete_node(self, collection_id: str, node_id: str): """删除单个节点""" await self.conn.execute("DELETE FROM nodes WHERE collection_id = ? AND node_id = ?", (collection_id, node_id)) await self.conn.commit() async def delete_nodes_bulk(self, collection_id: str, node_ids: List[str]): """批量删除节点 (单次提交)""" if not node_ids: return rows = [(collection_id, node_id) for node_id in node_ids] await self.conn.executemany( "DELETE FROM nodes WHERE collection_id = ? AND node_id = ?", rows, ) await self.conn.commit() async def load_nodes(self, collection_id: str) -> List[Dict[str, Any]]: """从特定 collection 中加载所有节点(depend_ids初始化为空,稍后从data实时计算)""" async with self.conn.execute("SELECT * FROM nodes WHERE collection_id = ?", (collection_id,)) as cursor: rows = await cursor.fetchall() results = [] for row in rows: results.append({ "node_id": row["node_id"], "node_type": row["node_type"], "data_id": row["data_id"], "bind_data_id": row["bind_data_id"], "depend_ids": [], # 不从数据库加载,运行时从data实时计算 "data": json.loads(row["data"]) if row["data"] else None, "origin_data": json.loads(row["origin_data"]) if row["origin_data"] else None, "action": row["action"], "status": row["status"], "binding_status": row["binding_status"], "error": row["error"], "sync_log": row["sync_log"], "context": json.loads(row["context"]) if row["context"] else {} }) if results[-1]["bind_data_id"] and isinstance(results[-1]["context"], dict): results[-1]["context"]["bind_data_id"] = results[-1]["bind_data_id"] return results # --- BindingManager 支持 --- async def save_binding(self, node_type: str, local_id: str, payload_dict: Dict[str, Any]): """保存或更新绑定关系及其元数据""" remote_id = payload_dict.get("remote_id") if payload_dict else None await self.conn.execute(""" INSERT OR REPLACE INTO bindings (node_type, local_id, remote_id) VALUES (?, ?, ?) """, (node_type, local_id, remote_id)) await self.conn.commit() async def save_bindings_bulk(self, node_type: str, bindings: Dict[str, Optional[str]]): """批量保存或更新绑定关系 (单次提交)""" if not bindings: return rows = [] for local_id, remote_id in bindings.items(): rows.append((node_type, local_id, remote_id)) await self.conn.executemany( "INSERT OR REPLACE INTO bindings (node_type, local_id, remote_id) VALUES (?, ?, ?)", rows, ) await self.conn.commit() async def delete_binding(self, node_type: str, local_id: str): """删除绑定关系""" await self.conn.execute( "DELETE FROM bindings WHERE node_type = ? AND local_id = ?", (node_type, local_id) ) await self.conn.commit() async def delete_bindings_bulk(self, node_type: str, local_ids: List[str]): """批量删除绑定关系 (单次提交)""" if not local_ids: return rows = [(node_type, local_id) for local_id in local_ids] await self.conn.executemany( "DELETE FROM bindings WHERE node_type = ? AND local_id = ?", rows, ) await self.conn.commit() async def load_bindings(self, node_type: str) -> List[Dict[str, Any]]: """回写该类型下的所有绑定关系到内存""" async with self.conn.execute( "SELECT local_id, remote_id FROM bindings WHERE node_type = ?", (node_type,) ) as cursor: rows = await cursor.fetchall() return [ {"local_id": r["local_id"], "payload": {"remote_id": r["remote_id"]}} for r in rows ] async def load_bind_data_id_overrides(self, node_type: str) -> Dict[str, Optional[str]]: """基于 bindings 表计算 node_id -> bind_data_id 覆盖映射。 规则: - local 节点的 bind_data_id 取其绑定 remote 节点的 data_id - remote 节点的 bind_data_id 取其绑定 local 节点的 data_id """ sql = """ SELECT b.local_id AS src_node_id, rn.data_id AS bind_data_id FROM bindings b LEFT JOIN nodes rn ON rn.node_id = b.remote_id AND rn.node_type = b.node_type WHERE b.node_type = ? UNION ALL SELECT b.remote_id AS src_node_id, ln.data_id AS bind_data_id FROM bindings b LEFT JOIN nodes ln ON ln.node_id = b.local_id AND ln.node_type = b.node_type WHERE b.node_type = ? AND b.remote_id IS NOT NULL """ async with self.conn.execute(sql, (node_type, node_type)) as cursor: rows = await cursor.fetchall() result: Dict[str, Optional[str]] = {} for row in rows: src_node_id = row["src_node_id"] if not src_node_id: continue result[str(src_node_id)] = row["bind_data_id"] return result async def dump_to_runtime(self, filename: str): """将当前状态转储到 _runtime 文件夹 (使用 VACUUM INTO)""" runtime_path = os.path.join("_runtime", filename) if os.path.exists(runtime_path): os.remove(runtime_path) # VACUUM INTO 是备份 SQLite 的推荐方式 (SQLite 3.27+) await self.conn.execute("VACUUM INTO ?", (runtime_path,)) async def close(self): """关闭数据库连接(包含异常处理)""" if self.conn: try: await self.conn.close() except Exception as e: print(f"⚠️ Error closing persistence: {e}") finally: self.conn = None