将persist和后端分离,方便替换为其他数据库
This commit is contained in:
@@ -28,6 +28,8 @@ remote_datasource:
|
||||
jsonl_dir: ${PROJECT_ROOT}/remote_datasource/datasource
|
||||
persist:
|
||||
db_path: ${PROJECT_ROOT}/test_results/sync_state_machine/demo.db
|
||||
backend: sqlite
|
||||
backend_options: {}
|
||||
strategies:
|
||||
project:
|
||||
config:
|
||||
@@ -109,6 +111,45 @@ logging:
|
||||
- 每个业务类型的行为策略(依赖、自动绑定、创建方向、更新方向、跳过等)。
|
||||
- `persist`
|
||||
- 持久化开关与数据库位置。
|
||||
- `persist.backend`
|
||||
- 持久化后端注册表里的名称,例如 `sqlite`。
|
||||
- backend 实现会自行解释 `persist` 的其余字段。
|
||||
- `persist.backend_options`
|
||||
- 后端私有配置,后端自己决定如何解释。
|
||||
|
||||
### 6. 持久化后端注册
|
||||
|
||||
如果你希望在 backend 初始化阶段替换持久化后端,可以先在启动代码里按名称注册,再通过配置选择,不需要改调用方。
|
||||
|
||||
#### 6.1 内置 SQLite
|
||||
|
||||
```yaml
|
||||
persist:
|
||||
backend: sqlite
|
||||
db_path: ${PROJECT_ROOT}/test_results/sync_state_machine/demo.db
|
||||
backend_options: {}
|
||||
```
|
||||
|
||||
#### 6.2 注册自定义后端
|
||||
|
||||
```python
|
||||
from sync_state_machine.common.persistence import register_persistence_backend
|
||||
|
||||
register_persistence_backend("mysql", lambda config: MySQLPersistenceBackend(config))
|
||||
```
|
||||
|
||||
然后在配置里选择这个名字:
|
||||
|
||||
```yaml
|
||||
persist:
|
||||
backend: mysql
|
||||
db_path: ${PROJECT_ROOT}/test_results/sync_state_machine/demo.db
|
||||
backend_options:
|
||||
url: mysql+aiomysql://user:pass@127.0.0.1/depm_sync?charset=utf8mb4
|
||||
pool_size: 5
|
||||
```
|
||||
|
||||
后端实现会收到整份 `PersistConfig`,自行决定是读取 `db_path`、`backend_options`,还是额外扩展别的字段。
|
||||
- `logging`
|
||||
- 运行日志输出位置与级别。
|
||||
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
from .types import SyncAction, SyncStatus, BindingStatus
|
||||
from .sync_node import SyncNode
|
||||
from .persistence import PersistenceBackend
|
||||
from .persistence import (
|
||||
PersistenceBackend,
|
||||
create_persistence_backend,
|
||||
get_registered_persistence_backends,
|
||||
register_persistence_backend,
|
||||
)
|
||||
from .persistence_backends import SQLitePersistenceBackend
|
||||
from .collection import DataCollection
|
||||
from .binding import BindingManager
|
||||
|
||||
@@ -10,6 +16,10 @@ __all__ = [
|
||||
"BindingStatus",
|
||||
"SyncNode",
|
||||
"PersistenceBackend",
|
||||
"SQLitePersistenceBackend",
|
||||
"create_persistence_backend",
|
||||
"register_persistence_backend",
|
||||
"get_registered_persistence_backends",
|
||||
"DataCollection",
|
||||
"BindingManager",
|
||||
]
|
||||
|
||||
@@ -1,21 +1,20 @@
|
||||
import aiosqlite
|
||||
import json
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sqlite3
|
||||
from typing import Optional, Dict, List, Any, TYPE_CHECKING
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any, Callable, ClassVar, Dict, List, Optional, TYPE_CHECKING
|
||||
|
||||
from ..config.persist_config import PersistConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .sync_node import SyncNode
|
||||
|
||||
class PersistenceBackend:
|
||||
"""
|
||||
Persistence 负责同步系统运行时数据的持久化存储(aiosqlite 实现)。
|
||||
主要作为 DataCollection 和 BindingManager 的后端存储。
|
||||
"""
|
||||
def __init__(self, db_path: str = ":memory:", backend: str = "sqlite"):
|
||||
self.db_path = db_path
|
||||
self.backend = backend
|
||||
self.conn: Optional[aiosqlite.Connection] = None
|
||||
|
||||
class PersistenceBackend(ABC):
|
||||
"""同步系统持久化契约。"""
|
||||
|
||||
backend_name: ClassVar[str] = "abstract"
|
||||
|
||||
@staticmethod
|
||||
def _is_read_only_sql(sql: str) -> bool:
|
||||
@@ -93,250 +92,27 @@ class PersistenceBackend:
|
||||
]
|
||||
return summary
|
||||
|
||||
async def initialize(self):
|
||||
"""异步初始化数据库连接和表结构"""
|
||||
if self.backend != "sqlite":
|
||||
raise NotImplementedError(f"Persistence backend not supported yet: {self.backend}")
|
||||
@abstractmethod
|
||||
async def initialize(self) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
# 确保目录存在
|
||||
if self.db_path != ":memory:":
|
||||
dir_path = os.path.dirname(self.db_path)
|
||||
if dir_path:
|
||||
os.makedirs(dir_path, exist_ok=True)
|
||||
|
||||
self.conn = await aiosqlite.connect(self.db_path)
|
||||
self.conn.row_factory = aiosqlite.Row
|
||||
await self._init_db()
|
||||
@abstractmethod
|
||||
async def save_node(self, collection_id: str, scope: str, node: "SyncNode") -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def _init_db(self):
|
||||
# 1. 节点表: 仅用于持久化存储 DataCollection 中的 SyncNode
|
||||
# 注意: depend_ids不持久化,每次从data实时计算
|
||||
await self.conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS nodes (
|
||||
collection_id TEXT NOT NULL,
|
||||
scope TEXT NOT NULL DEFAULT 'global',
|
||||
node_id TEXT NOT NULL,
|
||||
node_type TEXT NOT NULL,
|
||||
data_id TEXT,
|
||||
bind_data_id TEXT,
|
||||
data TEXT,
|
||||
origin_data TEXT,
|
||||
action TEXT,
|
||||
status TEXT,
|
||||
binding_status TEXT,
|
||||
error TEXT,
|
||||
sync_log TEXT,
|
||||
context TEXT,
|
||||
PRIMARY KEY (collection_id, scope, node_id)
|
||||
)
|
||||
""")
|
||||
await self._ensure_nodes_schema()
|
||||
|
||||
# 2. 绑定表: 存储本地到远程的映射关系
|
||||
# 只保留 local_id / remote_id 两字段(remote_id 可空)
|
||||
await self._ensure_bindings_schema()
|
||||
await self.conn.commit()
|
||||
@abstractmethod
|
||||
async def save_nodes_bulk(self, collection_id: str, scope: str, nodes: List["SyncNode"]) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def _ensure_nodes_schema(self) -> None:
|
||||
cursor = await self.conn.execute("PRAGMA table_info(nodes)")
|
||||
rows = await cursor.fetchall()
|
||||
columns = [r[1] for r in rows] if rows else []
|
||||
if "scope" not in columns:
|
||||
await self.conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS nodes_new (
|
||||
collection_id TEXT NOT NULL,
|
||||
scope TEXT NOT NULL DEFAULT 'global',
|
||||
node_id TEXT NOT NULL,
|
||||
node_type TEXT NOT NULL,
|
||||
data_id TEXT,
|
||||
bind_data_id TEXT,
|
||||
data TEXT,
|
||||
origin_data TEXT,
|
||||
action TEXT,
|
||||
status TEXT,
|
||||
binding_status TEXT,
|
||||
error TEXT,
|
||||
sync_log TEXT,
|
||||
context TEXT,
|
||||
PRIMARY KEY (collection_id, scope, node_id)
|
||||
)
|
||||
""")
|
||||
await self.conn.execute("""
|
||||
INSERT INTO nodes_new (
|
||||
collection_id, scope, node_id, node_type, data_id, bind_data_id,
|
||||
data, origin_data, action, status, binding_status, error, sync_log, context
|
||||
)
|
||||
SELECT
|
||||
collection_id, 'global', node_id, node_type, data_id, bind_data_id,
|
||||
data, origin_data, action, status, binding_status, error, sync_log, context
|
||||
FROM nodes
|
||||
""")
|
||||
await self.conn.execute("DROP TABLE nodes")
|
||||
await self.conn.execute("ALTER TABLE nodes_new RENAME TO nodes")
|
||||
cursor = await self.conn.execute("PRAGMA table_info(nodes)")
|
||||
rows = await cursor.fetchall()
|
||||
columns = [r[1] for r in rows] if rows else []
|
||||
if "bind_data_id" not in columns:
|
||||
await self.conn.execute("ALTER TABLE nodes ADD COLUMN bind_data_id TEXT")
|
||||
await self.conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_nodes_collection_scope_type ON nodes (collection_id, scope, node_type)"
|
||||
)
|
||||
await self.conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_nodes_collection_scope_data_id ON nodes (collection_id, scope, data_id)"
|
||||
)
|
||||
@abstractmethod
|
||||
async def delete_node(self, collection_id: str, scope: str, node_id: str) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def _ensure_bindings_schema(self) -> None:
|
||||
"""确保 bindings 表为 (node_type, local_id, remote_id) 结构。
|
||||
|
||||
若检测到旧表含 payload 列,则进行迁移。
|
||||
"""
|
||||
cursor = await self.conn.execute("PRAGMA table_info(bindings)")
|
||||
rows = await cursor.fetchall()
|
||||
columns = [r[1] for r in rows] if rows else []
|
||||
|
||||
if not columns:
|
||||
await self.conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS bindings (
|
||||
scope TEXT NOT NULL DEFAULT 'global',
|
||||
node_type TEXT NOT NULL,
|
||||
local_id TEXT NOT NULL,
|
||||
remote_id TEXT,
|
||||
last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (scope, node_type, local_id)
|
||||
)
|
||||
""")
|
||||
return
|
||||
|
||||
if "scope" not in columns or ("payload" in columns and "remote_id" not in columns):
|
||||
# 迁移旧结构到 scope + remote_id 新结构
|
||||
await self.conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS bindings_new (
|
||||
scope TEXT NOT NULL DEFAULT 'global',
|
||||
node_type TEXT NOT NULL,
|
||||
local_id TEXT NOT NULL,
|
||||
remote_id TEXT,
|
||||
last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (scope, node_type, local_id)
|
||||
)
|
||||
""")
|
||||
|
||||
if "payload" in columns and "remote_id" not in columns:
|
||||
async with self.conn.execute("SELECT node_type, local_id, payload FROM bindings") as cur:
|
||||
old_rows = await cur.fetchall()
|
||||
for row in old_rows:
|
||||
try:
|
||||
payload_dict = json.loads(row[2]) if row[2] else {}
|
||||
except json.JSONDecodeError:
|
||||
payload_dict = {}
|
||||
remote_id = payload_dict.get("remote_id")
|
||||
await self.conn.execute(
|
||||
"INSERT OR REPLACE INTO bindings_new (scope, node_type, local_id, remote_id) VALUES (?, ?, ?, ?)",
|
||||
("global", row[0], row[1], remote_id),
|
||||
)
|
||||
else:
|
||||
async with self.conn.execute("SELECT node_type, local_id, remote_id FROM bindings") as cur:
|
||||
old_rows = await cur.fetchall()
|
||||
for row in old_rows:
|
||||
await self.conn.execute(
|
||||
"INSERT OR REPLACE INTO bindings_new (scope, node_type, local_id, remote_id) VALUES (?, ?, ?, ?)",
|
||||
("global", row[0], row[1], row[2]),
|
||||
)
|
||||
|
||||
await self.conn.execute("DROP TABLE bindings")
|
||||
await self.conn.execute("ALTER TABLE bindings_new RENAME TO bindings")
|
||||
await self.conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_bindings_scope_type_remote ON bindings (scope, node_type, remote_id)"
|
||||
)
|
||||
|
||||
# --- DataCollection 支持 ---
|
||||
|
||||
async def save_node(self, collection_id: str, scope: str, node: "SyncNode"):
|
||||
"""保存或更新单个节点(depend_ids不持久化,运行时从data实时计算)"""
|
||||
bind_data_id = None
|
||||
if isinstance(node.context, dict):
|
||||
value = node.context.get("bind_data_id")
|
||||
if value is not None and value != "":
|
||||
bind_data_id = str(value)
|
||||
await self.conn.execute("""
|
||||
INSERT OR REPLACE INTO nodes (
|
||||
collection_id, scope, node_id, node_type, data_id, bind_data_id,
|
||||
data, origin_data, action, status, binding_status, error, sync_log, context
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
collection_id,
|
||||
scope,
|
||||
node.node_id,
|
||||
node.node_type,
|
||||
node.data_id,
|
||||
bind_data_id,
|
||||
json.dumps(node.data) if node.data else None,
|
||||
json.dumps(node.origin_data) if node.origin_data else None,
|
||||
node.action.value,
|
||||
node.status.value,
|
||||
node.binding_status.value,
|
||||
node.error,
|
||||
node.sync_log,
|
||||
json.dumps(node.context) if node.context else None
|
||||
))
|
||||
await self.conn.commit()
|
||||
|
||||
async def save_nodes_bulk(self, collection_id: str, scope: str, nodes: List["SyncNode"]):
|
||||
"""批量保存或更新节点 (单次提交,depend_ids不持久化)"""
|
||||
if not nodes:
|
||||
return
|
||||
rows = []
|
||||
for node in nodes:
|
||||
bind_data_id = None
|
||||
if isinstance(node.context, dict):
|
||||
value = node.context.get("bind_data_id")
|
||||
if value is not None and value != "":
|
||||
bind_data_id = str(value)
|
||||
rows.append((
|
||||
collection_id,
|
||||
scope,
|
||||
node.node_id,
|
||||
node.node_type,
|
||||
node.data_id,
|
||||
bind_data_id,
|
||||
json.dumps(node.data) if node.data else None,
|
||||
json.dumps(node.origin_data) if node.origin_data else None,
|
||||
node.action.value,
|
||||
node.status.value,
|
||||
node.binding_status.value,
|
||||
node.error,
|
||||
node.sync_log,
|
||||
json.dumps(node.context) if node.context else None
|
||||
))
|
||||
|
||||
await self.conn.executemany("""
|
||||
INSERT OR REPLACE INTO nodes (
|
||||
collection_id, scope, node_id, node_type, data_id, bind_data_id,
|
||||
data, origin_data, action, status, binding_status, error, sync_log, context
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", rows)
|
||||
await self.conn.commit()
|
||||
|
||||
async def delete_node(self, collection_id: str, scope: str, node_id: str):
|
||||
"""删除单个节点"""
|
||||
await self.conn.execute(
|
||||
"DELETE FROM nodes WHERE collection_id = ? AND scope = ? AND node_id = ?",
|
||||
(collection_id, scope, node_id),
|
||||
)
|
||||
await self.conn.commit()
|
||||
|
||||
async def delete_nodes_bulk(self, collection_id: str, scope: str, node_ids: List[str]):
|
||||
"""批量删除节点 (单次提交)"""
|
||||
if not node_ids:
|
||||
return
|
||||
rows = [(collection_id, scope, node_id) for node_id in node_ids]
|
||||
await self.conn.executemany(
|
||||
"DELETE FROM nodes WHERE collection_id = ? AND scope = ? AND node_id = ?",
|
||||
rows,
|
||||
)
|
||||
await self.conn.commit()
|
||||
@abstractmethod
|
||||
async def delete_nodes_bulk(self, collection_id: str, scope: str, node_ids: List[str]) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def load_nodes(
|
||||
self,
|
||||
collection_id: str,
|
||||
@@ -344,153 +120,37 @@ class PersistenceBackend:
|
||||
*,
|
||||
exclude_node_types: Optional[List[str]] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""从特定 collection 中加载所有节点(depend_ids初始化为空,稍后从data实时计算)"""
|
||||
sql = "SELECT * FROM nodes WHERE collection_id = ? AND scope = ?"
|
||||
params: List[Any] = [collection_id, scope]
|
||||
filtered_types = [node_type for node_type in (exclude_node_types or []) if node_type]
|
||||
if filtered_types:
|
||||
placeholders = ", ".join("?" for _ in filtered_types)
|
||||
sql += f" AND node_type NOT IN ({placeholders})"
|
||||
params.extend(filtered_types)
|
||||
raise NotImplementedError
|
||||
|
||||
async with self.conn.execute(sql, tuple(params)) as cursor:
|
||||
rows = await cursor.fetchall()
|
||||
results = []
|
||||
for row in rows:
|
||||
results.append({
|
||||
"node_id": row["node_id"],
|
||||
"node_type": row["node_type"],
|
||||
"data_id": row["data_id"],
|
||||
"bind_data_id": row["bind_data_id"],
|
||||
"depend_ids": [], # 不从数据库加载,运行时从data实时计算
|
||||
"data": json.loads(row["data"]) if row["data"] else None,
|
||||
"origin_data": json.loads(row["origin_data"]) if row["origin_data"] else None,
|
||||
"action": row["action"],
|
||||
"status": row["status"],
|
||||
"binding_status": row["binding_status"],
|
||||
"error": row["error"],
|
||||
"sync_log": row["sync_log"],
|
||||
"context": json.loads(row["context"]) if row["context"] else {}
|
||||
})
|
||||
if results[-1]["bind_data_id"] and isinstance(results[-1]["context"], dict):
|
||||
results[-1]["context"]["bind_data_id"] = results[-1]["bind_data_id"]
|
||||
return results
|
||||
@abstractmethod
|
||||
async def save_binding(self, scope: str, node_type: str, local_id: str, payload_dict: Dict[str, Any]) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
# --- BindingManager 支持 ---
|
||||
@abstractmethod
|
||||
async def save_bindings_bulk(self, scope: str, node_type: str, bindings: Dict[str, Optional[str]]) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def save_binding(self, scope: str, node_type: str, local_id: str, payload_dict: Dict[str, Any]):
|
||||
"""保存或更新绑定关系及其元数据"""
|
||||
remote_id = payload_dict.get("remote_id") if payload_dict else None
|
||||
await self.conn.execute("""
|
||||
INSERT OR REPLACE INTO bindings (scope, node_type, local_id, remote_id)
|
||||
VALUES (?, ?, ?, ?)
|
||||
""", (scope, node_type, local_id, remote_id))
|
||||
await self.conn.commit()
|
||||
@abstractmethod
|
||||
async def delete_binding(self, scope: str, node_type: str, local_id: str) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def save_bindings_bulk(self, scope: str, node_type: str, bindings: Dict[str, Optional[str]]):
|
||||
"""批量保存或更新绑定关系 (单次提交)"""
|
||||
if not bindings:
|
||||
return
|
||||
rows = []
|
||||
for local_id, remote_id in bindings.items():
|
||||
rows.append((scope, node_type, local_id, remote_id))
|
||||
await self.conn.executemany(
|
||||
"INSERT OR REPLACE INTO bindings (scope, node_type, local_id, remote_id) VALUES (?, ?, ?, ?)",
|
||||
rows,
|
||||
)
|
||||
await self.conn.commit()
|
||||
|
||||
async def delete_binding(self, scope: str, node_type: str, local_id: str):
|
||||
"""删除绑定关系"""
|
||||
await self.conn.execute(
|
||||
"DELETE FROM bindings WHERE scope = ? AND node_type = ? AND local_id = ?",
|
||||
(scope, node_type, local_id)
|
||||
)
|
||||
await self.conn.commit()
|
||||
|
||||
async def delete_bindings_bulk(self, scope: str, node_type: str, local_ids: List[str]):
|
||||
"""批量删除绑定关系 (单次提交)"""
|
||||
if not local_ids:
|
||||
return
|
||||
rows = [(scope, node_type, local_id) for local_id in local_ids]
|
||||
await self.conn.executemany(
|
||||
"DELETE FROM bindings WHERE scope = ? AND node_type = ? AND local_id = ?",
|
||||
rows,
|
||||
)
|
||||
await self.conn.commit()
|
||||
@abstractmethod
|
||||
async def delete_bindings_bulk(self, scope: str, node_type: str, local_ids: List[str]) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def load_bindings(self, scope: str, node_type: str) -> List[Dict[str, Any]]:
|
||||
"""回写该类型下的所有绑定关系到内存"""
|
||||
async with self.conn.execute(
|
||||
"SELECT local_id, remote_id FROM bindings WHERE scope = ? AND node_type = ?",
|
||||
(scope, node_type)
|
||||
) as cursor:
|
||||
rows = await cursor.fetchall()
|
||||
return [
|
||||
{"local_id": r["local_id"], "payload": {"remote_id": r["remote_id"]}}
|
||||
for r in rows
|
||||
]
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def load_bind_data_id_overrides(self, node_type: str, scope: str) -> Dict[str, Optional[str]]:
|
||||
"""基于 bindings 表计算 node_id -> bind_data_id 覆盖映射。
|
||||
raise NotImplementedError
|
||||
|
||||
规则:
|
||||
- local 节点的 bind_data_id 取其绑定 remote 节点的 data_id
|
||||
- remote 节点的 bind_data_id 取其绑定 local 节点的 data_id
|
||||
"""
|
||||
sql = """
|
||||
SELECT b.local_id AS src_node_id, rn.data_id AS bind_data_id
|
||||
FROM bindings b
|
||||
LEFT JOIN nodes rn
|
||||
ON rn.node_id = b.remote_id
|
||||
AND rn.node_type = b.node_type
|
||||
AND rn.scope = b.scope
|
||||
WHERE b.node_type = ?
|
||||
AND b.scope = ?
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT b.remote_id AS src_node_id, ln.data_id AS bind_data_id
|
||||
FROM bindings b
|
||||
LEFT JOIN nodes ln
|
||||
ON ln.node_id = b.local_id
|
||||
AND ln.node_type = b.node_type
|
||||
AND ln.scope = b.scope
|
||||
WHERE b.node_type = ?
|
||||
AND b.scope = ?
|
||||
AND b.remote_id IS NOT NULL
|
||||
"""
|
||||
async with self.conn.execute(sql, (node_type, scope, node_type, scope)) as cursor:
|
||||
rows = await cursor.fetchall()
|
||||
result: Dict[str, Optional[str]] = {}
|
||||
for row in rows:
|
||||
src_node_id = row["src_node_id"]
|
||||
if not src_node_id:
|
||||
continue
|
||||
result[str(src_node_id)] = row["bind_data_id"]
|
||||
return result
|
||||
|
||||
async def delete_scope_node_types(
|
||||
self,
|
||||
*,
|
||||
scope: str,
|
||||
node_types: List[str],
|
||||
) -> None:
|
||||
filtered_types = [node_type for node_type in node_types if node_type]
|
||||
if not filtered_types:
|
||||
return
|
||||
|
||||
placeholders = ", ".join("?" for _ in filtered_types)
|
||||
await self.conn.execute(
|
||||
f"DELETE FROM nodes WHERE scope = ? AND node_type IN ({placeholders})",
|
||||
(scope, *filtered_types),
|
||||
)
|
||||
await self.conn.execute(
|
||||
f"DELETE FROM bindings WHERE scope = ? AND node_type IN ({placeholders})",
|
||||
(scope, *filtered_types),
|
||||
)
|
||||
await self.conn.commit()
|
||||
@abstractmethod
|
||||
async def delete_scope_node_types(self, *, scope: str, node_types: List[str]) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def copy_nodes_between_scopes(
|
||||
self,
|
||||
*,
|
||||
@@ -499,25 +159,9 @@ class PersistenceBackend:
|
||||
target_scope: str,
|
||||
node_types: List[str],
|
||||
) -> None:
|
||||
if source_scope == target_scope or not node_types:
|
||||
return
|
||||
placeholders = ", ".join("?" for _ in node_types)
|
||||
sql = f"""
|
||||
INSERT OR REPLACE INTO nodes (
|
||||
collection_id, scope, node_id, node_type, data_id, bind_data_id,
|
||||
data, origin_data, action, status, binding_status, error, sync_log, context
|
||||
)
|
||||
SELECT
|
||||
collection_id, ?, node_id, node_type, data_id, bind_data_id,
|
||||
data, origin_data, action, status, binding_status, error, sync_log, context
|
||||
FROM nodes
|
||||
WHERE collection_id = ?
|
||||
AND scope = ?
|
||||
AND node_type IN ({placeholders})
|
||||
"""
|
||||
await self.conn.execute(sql, (target_scope, collection_id, source_scope, *node_types))
|
||||
await self.conn.commit()
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def copy_bindings_between_scopes(
|
||||
self,
|
||||
*,
|
||||
@@ -525,34 +169,58 @@ class PersistenceBackend:
|
||||
target_scope: str,
|
||||
node_types: List[str],
|
||||
) -> None:
|
||||
if source_scope == target_scope or not node_types:
|
||||
return
|
||||
placeholders = ", ".join("?" for _ in node_types)
|
||||
sql = f"""
|
||||
INSERT OR REPLACE INTO bindings (scope, node_type, local_id, remote_id)
|
||||
SELECT ?, node_type, local_id, remote_id
|
||||
FROM bindings
|
||||
WHERE scope = ?
|
||||
AND node_type IN ({placeholders})
|
||||
"""
|
||||
await self.conn.execute(sql, (target_scope, source_scope, *node_types))
|
||||
await self.conn.commit()
|
||||
raise NotImplementedError
|
||||
|
||||
async def dump_to_runtime(self, filename: str):
|
||||
"""将当前状态转储到 _runtime 文件夹 (使用 VACUUM INTO)"""
|
||||
runtime_path = os.path.join("_runtime", filename)
|
||||
if os.path.exists(runtime_path):
|
||||
os.remove(runtime_path)
|
||||
|
||||
# VACUUM INTO 是备份 SQLite 的推荐方式 (SQLite 3.27+)
|
||||
await self.conn.execute("VACUUM INTO ?", (runtime_path,))
|
||||
@abstractmethod
|
||||
async def dump_to_runtime(self, filename: str) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def close(self):
|
||||
"""关闭数据库连接(包含异常处理)"""
|
||||
if self.conn:
|
||||
try:
|
||||
await self.conn.close()
|
||||
except Exception as e:
|
||||
print(f"⚠️ Error closing persistence: {e}")
|
||||
finally:
|
||||
self.conn = None
|
||||
@abstractmethod
|
||||
async def close(self) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
PersistenceBackendFactory = Callable[[PersistConfig], PersistenceBackend]
|
||||
_PERSISTENCE_BACKEND_REGISTRY: dict[str, PersistenceBackendFactory] = {}
|
||||
|
||||
|
||||
def register_persistence_backend(name: str, factory: PersistenceBackendFactory, *, override: bool = False) -> None:
|
||||
backend_name = name.strip()
|
||||
if not backend_name:
|
||||
raise ValueError("backend name cannot be empty")
|
||||
if not override and backend_name in _PERSISTENCE_BACKEND_REGISTRY:
|
||||
raise ValueError(f"persistence backend already registered: {backend_name}")
|
||||
_PERSISTENCE_BACKEND_REGISTRY[backend_name] = factory
|
||||
|
||||
|
||||
def get_registered_persistence_backends() -> Dict[str, PersistenceBackendFactory]:
|
||||
return dict(_PERSISTENCE_BACKEND_REGISTRY)
|
||||
|
||||
|
||||
def create_persistence_backend(
|
||||
persist_config: PersistConfig,
|
||||
) -> PersistenceBackend:
|
||||
backend_name = (persist_config.backend or "").strip()
|
||||
if not backend_name:
|
||||
raise ValueError("persist.backend cannot be empty")
|
||||
|
||||
if not _PERSISTENCE_BACKEND_REGISTRY:
|
||||
register_persistence_backend(
|
||||
"sqlite",
|
||||
lambda config: SQLitePersistenceBackend(config),
|
||||
)
|
||||
|
||||
factory = _PERSISTENCE_BACKEND_REGISTRY.get(backend_name)
|
||||
if factory is None:
|
||||
available = ", ".join(sorted(_PERSISTENCE_BACKEND_REGISTRY)) or "none"
|
||||
raise NotImplementedError(f"Persistence backend not supported yet: {backend_name}. Available: {available}")
|
||||
return factory(persist_config)
|
||||
|
||||
|
||||
def _register_builtin_persistence_backends() -> None:
|
||||
from .persistence_backends.sqlite_backend import SQLitePersistenceBackend
|
||||
|
||||
register_persistence_backend("sqlite", lambda config: SQLitePersistenceBackend(config), override=True)
|
||||
|
||||
|
||||
_register_builtin_persistence_backends()
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
from .sqlite_backend import SQLitePersistenceBackend
|
||||
|
||||
__all__ = ["SQLitePersistenceBackend"]
|
||||
@@ -0,0 +1,464 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import aiosqlite
|
||||
import json
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional, TYPE_CHECKING
|
||||
|
||||
from ...config.persist_config import PersistConfig
|
||||
from ..persistence import PersistenceBackend
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..sync_node import SyncNode
|
||||
|
||||
|
||||
class SQLitePersistenceBackend(PersistenceBackend):
|
||||
"""SQLite implementation of the persistence contract."""
|
||||
|
||||
backend_name = "sqlite"
|
||||
|
||||
def __init__(self, persist_config: PersistConfig | str | None = None, **_: Any) -> None:
|
||||
if isinstance(persist_config, PersistConfig):
|
||||
self.persist_config = persist_config
|
||||
self.db_path = persist_config.db_path
|
||||
elif isinstance(persist_config, str):
|
||||
self.persist_config = None
|
||||
self.db_path = persist_config
|
||||
else:
|
||||
self.persist_config = None
|
||||
self.db_path = ":memory:"
|
||||
self.conn: Optional[aiosqlite.Connection] = None
|
||||
|
||||
async def initialize(self) -> None:
|
||||
if self.db_path != ":memory:":
|
||||
dir_path = os.path.dirname(self.db_path)
|
||||
if dir_path:
|
||||
os.makedirs(dir_path, exist_ok=True)
|
||||
|
||||
self.conn = await aiosqlite.connect(self.db_path)
|
||||
self.conn.row_factory = aiosqlite.Row
|
||||
await self._init_db()
|
||||
|
||||
async def _init_db(self) -> None:
|
||||
assert self.conn is not None
|
||||
await self.conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS nodes (
|
||||
collection_id TEXT NOT NULL,
|
||||
scope TEXT NOT NULL DEFAULT 'global',
|
||||
node_id TEXT NOT NULL,
|
||||
node_type TEXT NOT NULL,
|
||||
data_id TEXT,
|
||||
bind_data_id TEXT,
|
||||
data TEXT,
|
||||
origin_data TEXT,
|
||||
action TEXT,
|
||||
status TEXT,
|
||||
binding_status TEXT,
|
||||
error TEXT,
|
||||
sync_log TEXT,
|
||||
context TEXT,
|
||||
PRIMARY KEY (collection_id, scope, node_id)
|
||||
)
|
||||
""")
|
||||
await self._ensure_nodes_schema()
|
||||
await self._ensure_bindings_schema()
|
||||
await self.conn.commit()
|
||||
|
||||
async def _ensure_nodes_schema(self) -> None:
|
||||
assert self.conn is not None
|
||||
cursor = await self.conn.execute("PRAGMA table_info(nodes)")
|
||||
rows = await cursor.fetchall()
|
||||
columns = [r[1] for r in rows] if rows else []
|
||||
if "scope" not in columns:
|
||||
await self.conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS nodes_new (
|
||||
collection_id TEXT NOT NULL,
|
||||
scope TEXT NOT NULL DEFAULT 'global',
|
||||
node_id TEXT NOT NULL,
|
||||
node_type TEXT NOT NULL,
|
||||
data_id TEXT,
|
||||
bind_data_id TEXT,
|
||||
data TEXT,
|
||||
origin_data TEXT,
|
||||
action TEXT,
|
||||
status TEXT,
|
||||
binding_status TEXT,
|
||||
error TEXT,
|
||||
sync_log TEXT,
|
||||
context TEXT,
|
||||
PRIMARY KEY (collection_id, scope, node_id)
|
||||
)
|
||||
""")
|
||||
await self.conn.execute("""
|
||||
INSERT INTO nodes_new (
|
||||
collection_id, scope, node_id, node_type, data_id, bind_data_id,
|
||||
data, origin_data, action, status, binding_status, error, sync_log, context
|
||||
)
|
||||
SELECT
|
||||
collection_id, 'global', node_id, node_type, data_id, bind_data_id,
|
||||
data, origin_data, action, status, binding_status, error, sync_log, context
|
||||
FROM nodes
|
||||
""")
|
||||
await self.conn.execute("DROP TABLE nodes")
|
||||
await self.conn.execute("ALTER TABLE nodes_new RENAME TO nodes")
|
||||
cursor = await self.conn.execute("PRAGMA table_info(nodes)")
|
||||
rows = await cursor.fetchall()
|
||||
columns = [r[1] for r in rows] if rows else []
|
||||
if "bind_data_id" not in columns:
|
||||
await self.conn.execute("ALTER TABLE nodes ADD COLUMN bind_data_id TEXT")
|
||||
await self.conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_nodes_collection_scope_type ON nodes (collection_id, scope, node_type)"
|
||||
)
|
||||
await self.conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_nodes_collection_scope_data_id ON nodes (collection_id, scope, data_id)"
|
||||
)
|
||||
|
||||
async def _ensure_bindings_schema(self) -> None:
|
||||
assert self.conn is not None
|
||||
cursor = await self.conn.execute("PRAGMA table_info(bindings)")
|
||||
rows = await cursor.fetchall()
|
||||
columns = [r[1] for r in rows] if rows else []
|
||||
|
||||
if not columns:
|
||||
await self.conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS bindings (
|
||||
scope TEXT NOT NULL DEFAULT 'global',
|
||||
node_type TEXT NOT NULL,
|
||||
local_id TEXT NOT NULL,
|
||||
remote_id TEXT,
|
||||
last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (scope, node_type, local_id)
|
||||
)
|
||||
""")
|
||||
return
|
||||
|
||||
if "scope" not in columns or ("payload" in columns and "remote_id" not in columns):
|
||||
await self.conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS bindings_new (
|
||||
scope TEXT NOT NULL DEFAULT 'global',
|
||||
node_type TEXT NOT NULL,
|
||||
local_id TEXT NOT NULL,
|
||||
remote_id TEXT,
|
||||
last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (scope, node_type, local_id)
|
||||
)
|
||||
""")
|
||||
|
||||
if "payload" in columns and "remote_id" not in columns:
|
||||
async with self.conn.execute("SELECT node_type, local_id, payload FROM bindings") as cur:
|
||||
old_rows = await cur.fetchall()
|
||||
for row in old_rows:
|
||||
try:
|
||||
payload_dict = json.loads(row[2]) if row[2] else {}
|
||||
except json.JSONDecodeError:
|
||||
payload_dict = {}
|
||||
remote_id = payload_dict.get("remote_id")
|
||||
await self.conn.execute(
|
||||
"INSERT OR REPLACE INTO bindings_new (scope, node_type, local_id, remote_id) VALUES (?, ?, ?, ?)",
|
||||
("global", row[0], row[1], remote_id),
|
||||
)
|
||||
else:
|
||||
async with self.conn.execute("SELECT node_type, local_id, remote_id FROM bindings") as cur:
|
||||
old_rows = await cur.fetchall()
|
||||
for row in old_rows:
|
||||
await self.conn.execute(
|
||||
"INSERT OR REPLACE INTO bindings_new (scope, node_type, local_id, remote_id) VALUES (?, ?, ?, ?)",
|
||||
("global", row[0], row[1], row[2]),
|
||||
)
|
||||
|
||||
await self.conn.execute("DROP TABLE bindings")
|
||||
await self.conn.execute("ALTER TABLE bindings_new RENAME TO bindings")
|
||||
await self.conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_bindings_scope_type_remote ON bindings (scope, node_type, remote_id)"
|
||||
)
|
||||
|
||||
async def save_node(self, collection_id: str, scope: str, node: "SyncNode") -> None:
|
||||
assert self.conn is not None
|
||||
bind_data_id = None
|
||||
if isinstance(node.context, dict):
|
||||
value = node.context.get("bind_data_id")
|
||||
if value is not None and value != "":
|
||||
bind_data_id = str(value)
|
||||
await self.conn.execute("""
|
||||
INSERT OR REPLACE INTO nodes (
|
||||
collection_id, scope, node_id, node_type, data_id, bind_data_id,
|
||||
data, origin_data, action, status, binding_status, error, sync_log, context
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
collection_id,
|
||||
scope,
|
||||
node.node_id,
|
||||
node.node_type,
|
||||
node.data_id,
|
||||
bind_data_id,
|
||||
json.dumps(node.data) if node.data else None,
|
||||
json.dumps(node.origin_data) if node.origin_data else None,
|
||||
node.action.value,
|
||||
node.status.value,
|
||||
node.binding_status.value,
|
||||
node.error,
|
||||
node.sync_log,
|
||||
json.dumps(node.context) if node.context else None,
|
||||
))
|
||||
await self.conn.commit()
|
||||
|
||||
async def save_nodes_bulk(self, collection_id: str, scope: str, nodes: List["SyncNode"]) -> None:
|
||||
assert self.conn is not None
|
||||
if not nodes:
|
||||
return
|
||||
rows = []
|
||||
for node in nodes:
|
||||
bind_data_id = None
|
||||
if isinstance(node.context, dict):
|
||||
value = node.context.get("bind_data_id")
|
||||
if value is not None and value != "":
|
||||
bind_data_id = str(value)
|
||||
rows.append((
|
||||
collection_id,
|
||||
scope,
|
||||
node.node_id,
|
||||
node.node_type,
|
||||
node.data_id,
|
||||
bind_data_id,
|
||||
json.dumps(node.data) if node.data else None,
|
||||
json.dumps(node.origin_data) if node.origin_data else None,
|
||||
node.action.value,
|
||||
node.status.value,
|
||||
node.binding_status.value,
|
||||
node.error,
|
||||
node.sync_log,
|
||||
json.dumps(node.context) if node.context else None,
|
||||
))
|
||||
|
||||
await self.conn.executemany("""
|
||||
INSERT OR REPLACE INTO nodes (
|
||||
collection_id, scope, node_id, node_type, data_id, bind_data_id,
|
||||
data, origin_data, action, status, binding_status, error, sync_log, context
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", rows)
|
||||
await self.conn.commit()
|
||||
|
||||
async def delete_node(self, collection_id: str, scope: str, node_id: str) -> None:
|
||||
assert self.conn is not None
|
||||
await self.conn.execute(
|
||||
"DELETE FROM nodes WHERE collection_id = ? AND scope = ? AND node_id = ?",
|
||||
(collection_id, scope, node_id),
|
||||
)
|
||||
await self.conn.commit()
|
||||
|
||||
async def delete_nodes_bulk(self, collection_id: str, scope: str, node_ids: List[str]) -> None:
|
||||
assert self.conn is not None
|
||||
if not node_ids:
|
||||
return
|
||||
rows = [(collection_id, scope, node_id) for node_id in node_ids]
|
||||
await self.conn.executemany(
|
||||
"DELETE FROM nodes WHERE collection_id = ? AND scope = ? AND node_id = ?",
|
||||
rows,
|
||||
)
|
||||
await self.conn.commit()
|
||||
|
||||
async def load_nodes(
|
||||
self,
|
||||
collection_id: str,
|
||||
scope: str,
|
||||
*,
|
||||
exclude_node_types: Optional[List[str]] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
assert self.conn is not None
|
||||
sql = "SELECT * FROM nodes WHERE collection_id = ? AND scope = ?"
|
||||
params: List[Any] = [collection_id, scope]
|
||||
filtered_types = [node_type for node_type in (exclude_node_types or []) if node_type]
|
||||
if filtered_types:
|
||||
placeholders = ", ".join("?" for _ in filtered_types)
|
||||
sql += f" AND node_type NOT IN ({placeholders})"
|
||||
params.extend(filtered_types)
|
||||
|
||||
async with self.conn.execute(sql, tuple(params)) as cursor:
|
||||
rows = await cursor.fetchall()
|
||||
results = []
|
||||
for row in rows:
|
||||
item = {
|
||||
"node_id": row["node_id"],
|
||||
"node_type": row["node_type"],
|
||||
"data_id": row["data_id"],
|
||||
"bind_data_id": row["bind_data_id"],
|
||||
"depend_ids": [],
|
||||
"data": json.loads(row["data"]) if row["data"] else None,
|
||||
"origin_data": json.loads(row["origin_data"]) if row["origin_data"] else None,
|
||||
"action": row["action"],
|
||||
"status": row["status"],
|
||||
"binding_status": row["binding_status"],
|
||||
"error": row["error"],
|
||||
"sync_log": row["sync_log"],
|
||||
"context": json.loads(row["context"]) if row["context"] else {},
|
||||
}
|
||||
if item["bind_data_id"] and isinstance(item["context"], dict):
|
||||
item["context"]["bind_data_id"] = item["bind_data_id"]
|
||||
results.append(item)
|
||||
return results
|
||||
|
||||
async def save_binding(self, scope: str, node_type: str, local_id: str, payload_dict: Dict[str, Any]) -> None:
|
||||
assert self.conn is not None
|
||||
remote_id = payload_dict.get("remote_id") if payload_dict else None
|
||||
await self.conn.execute("""
|
||||
INSERT OR REPLACE INTO bindings (scope, node_type, local_id, remote_id)
|
||||
VALUES (?, ?, ?, ?)
|
||||
""", (scope, node_type, local_id, remote_id))
|
||||
await self.conn.commit()
|
||||
|
||||
async def save_bindings_bulk(self, scope: str, node_type: str, bindings: Dict[str, Optional[str]]) -> None:
|
||||
assert self.conn is not None
|
||||
if not bindings:
|
||||
return
|
||||
rows = [(scope, node_type, local_id, remote_id) for local_id, remote_id in bindings.items()]
|
||||
await self.conn.executemany(
|
||||
"INSERT OR REPLACE INTO bindings (scope, node_type, local_id, remote_id) VALUES (?, ?, ?, ?)",
|
||||
rows,
|
||||
)
|
||||
await self.conn.commit()
|
||||
|
||||
async def delete_binding(self, scope: str, node_type: str, local_id: str) -> None:
|
||||
assert self.conn is not None
|
||||
await self.conn.execute(
|
||||
"DELETE FROM bindings WHERE scope = ? AND node_type = ? AND local_id = ?",
|
||||
(scope, node_type, local_id),
|
||||
)
|
||||
await self.conn.commit()
|
||||
|
||||
async def delete_bindings_bulk(self, scope: str, node_type: str, local_ids: List[str]) -> None:
|
||||
assert self.conn is not None
|
||||
if not local_ids:
|
||||
return
|
||||
rows = [(scope, node_type, local_id) for local_id in local_ids]
|
||||
await self.conn.executemany(
|
||||
"DELETE FROM bindings WHERE scope = ? AND node_type = ? AND local_id = ?",
|
||||
rows,
|
||||
)
|
||||
await self.conn.commit()
|
||||
|
||||
async def load_bindings(self, scope: str, node_type: str) -> List[Dict[str, Any]]:
|
||||
assert self.conn is not None
|
||||
async with self.conn.execute(
|
||||
"SELECT local_id, remote_id FROM bindings WHERE scope = ? AND node_type = ?",
|
||||
(scope, node_type),
|
||||
) as cursor:
|
||||
rows = await cursor.fetchall()
|
||||
return [{"local_id": r["local_id"], "payload": {"remote_id": r["remote_id"]}} for r in rows]
|
||||
|
||||
async def load_bind_data_id_overrides(self, node_type: str, scope: str) -> Dict[str, Optional[str]]:
|
||||
assert self.conn is not None
|
||||
sql = """
|
||||
SELECT b.local_id AS src_node_id, rn.data_id AS bind_data_id
|
||||
FROM bindings b
|
||||
LEFT JOIN nodes rn
|
||||
ON rn.node_id = b.remote_id
|
||||
AND rn.node_type = b.node_type
|
||||
AND rn.scope = b.scope
|
||||
WHERE b.node_type = ?
|
||||
AND b.scope = ?
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT b.remote_id AS src_node_id, ln.data_id AS bind_data_id
|
||||
FROM bindings b
|
||||
LEFT JOIN nodes ln
|
||||
ON ln.node_id = b.local_id
|
||||
AND ln.node_type = b.node_type
|
||||
AND ln.scope = b.scope
|
||||
WHERE b.node_type = ?
|
||||
AND b.scope = ?
|
||||
AND b.remote_id IS NOT NULL
|
||||
"""
|
||||
async with self.conn.execute(sql, (node_type, scope, node_type, scope)) as cursor:
|
||||
rows = await cursor.fetchall()
|
||||
result: Dict[str, Optional[str]] = {}
|
||||
for row in rows:
|
||||
src_node_id = row["src_node_id"]
|
||||
if not src_node_id:
|
||||
continue
|
||||
result[str(src_node_id)] = row["bind_data_id"]
|
||||
return result
|
||||
|
||||
async def delete_scope_node_types(self, *, scope: str, node_types: List[str]) -> None:
|
||||
assert self.conn is not None
|
||||
filtered_types = [node_type for node_type in node_types if node_type]
|
||||
if not filtered_types:
|
||||
return
|
||||
|
||||
placeholders = ", ".join("?" for _ in filtered_types)
|
||||
await self.conn.execute(
|
||||
f"DELETE FROM nodes WHERE scope = ? AND node_type IN ({placeholders})",
|
||||
(scope, *filtered_types),
|
||||
)
|
||||
await self.conn.execute(
|
||||
f"DELETE FROM bindings WHERE scope = ? AND node_type IN ({placeholders})",
|
||||
(scope, *filtered_types),
|
||||
)
|
||||
await self.conn.commit()
|
||||
|
||||
async def copy_nodes_between_scopes(
|
||||
self,
|
||||
*,
|
||||
collection_id: str,
|
||||
source_scope: str,
|
||||
target_scope: str,
|
||||
node_types: List[str],
|
||||
) -> None:
|
||||
assert self.conn is not None
|
||||
if source_scope == target_scope or not node_types:
|
||||
return
|
||||
placeholders = ", ".join("?" for _ in node_types)
|
||||
sql = f"""
|
||||
INSERT OR REPLACE INTO nodes (
|
||||
collection_id, scope, node_id, node_type, data_id, bind_data_id,
|
||||
data, origin_data, action, status, binding_status, error, sync_log, context
|
||||
)
|
||||
SELECT
|
||||
collection_id, ?, node_id, node_type, data_id, bind_data_id,
|
||||
data, origin_data, action, status, binding_status, error, sync_log, context
|
||||
FROM nodes
|
||||
WHERE collection_id = ?
|
||||
AND scope = ?
|
||||
AND node_type IN ({placeholders})
|
||||
"""
|
||||
await self.conn.execute(sql, (target_scope, collection_id, source_scope, *node_types))
|
||||
await self.conn.commit()
|
||||
|
||||
async def copy_bindings_between_scopes(
|
||||
self,
|
||||
*,
|
||||
source_scope: str,
|
||||
target_scope: str,
|
||||
node_types: List[str],
|
||||
) -> None:
|
||||
assert self.conn is not None
|
||||
if source_scope == target_scope or not node_types:
|
||||
return
|
||||
placeholders = ", ".join("?" for _ in node_types)
|
||||
sql = f"""
|
||||
INSERT OR REPLACE INTO bindings (scope, node_type, local_id, remote_id)
|
||||
SELECT ?, node_type, local_id, remote_id
|
||||
FROM bindings
|
||||
WHERE scope = ?
|
||||
AND node_type IN ({placeholders})
|
||||
"""
|
||||
await self.conn.execute(sql, (target_scope, source_scope, *node_types))
|
||||
await self.conn.commit()
|
||||
|
||||
async def dump_to_runtime(self, filename: str) -> None:
|
||||
assert self.conn is not None
|
||||
runtime_path = os.path.join("_runtime", filename)
|
||||
if os.path.exists(runtime_path):
|
||||
os.remove(runtime_path)
|
||||
await self.conn.execute("VACUUM INTO ?", (runtime_path,))
|
||||
|
||||
async def close(self) -> None:
|
||||
if self.conn:
|
||||
try:
|
||||
await self.conn.close()
|
||||
except Exception as exc:
|
||||
print(f"⚠️ Error closing persistence: {exc}")
|
||||
finally:
|
||||
self.conn = None
|
||||
@@ -0,0 +1,3 @@
|
||||
from .persistence_backends.sqlite_backend import SQLitePersistenceBackend
|
||||
|
||||
__all__ = ["SQLitePersistenceBackend"]
|
||||
@@ -1,4 +1,4 @@
|
||||
from typing import Any, Dict, Literal
|
||||
from typing import Any, Dict
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
@@ -6,7 +6,7 @@ from pydantic import BaseModel, ConfigDict, Field
|
||||
class PersistConfig(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", validate_assignment=True)
|
||||
|
||||
backend: Literal["sqlite", "sqlalchemy", "custom"] = "sqlite"
|
||||
backend: str = "sqlite"
|
||||
db_path: str
|
||||
backend_options: Dict[str, Any] = Field(default_factory=dict)
|
||||
enable: bool = True
|
||||
|
||||
@@ -21,7 +21,7 @@ from typing import Any, Dict, List, Optional
|
||||
|
||||
from ..common.binding import BindingManager
|
||||
from ..common.collection import DataCollection
|
||||
from ..common.persistence import PersistenceBackend
|
||||
from ..common.persistence import PersistenceBackend, create_persistence_backend
|
||||
from ..common.registry import DomainRegistry
|
||||
from ..datasource import ensure_builtin_datasource_types_registered
|
||||
from ..datasource.type_registry import get_datasource_factory
|
||||
@@ -292,9 +292,6 @@ async def create_pipeline_from_config(
|
||||
all_types = list(config.node_types)
|
||||
strategy_map = _resolve_strategy_map(config, all_types)
|
||||
|
||||
if config.persist.backend != "sqlite":
|
||||
raise NotImplementedError(f"Persistence backend not supported yet: {config.persist.backend}")
|
||||
|
||||
db_path = Path(config.persist.db_path)
|
||||
if config.persist.wipe_on_start and db_path.exists():
|
||||
db_path.unlink()
|
||||
@@ -303,7 +300,7 @@ async def create_pipeline_from_config(
|
||||
local_ds = None
|
||||
remote_ds = None
|
||||
try:
|
||||
persistence = PersistenceBackend(str(config.persist.db_path), backend=config.persist.backend)
|
||||
persistence = create_persistence_backend(config.persist)
|
||||
await persistence.initialize()
|
||||
|
||||
local_ds = await _create_datasource(
|
||||
|
||||
@@ -11,6 +11,7 @@ import pytest_asyncio
|
||||
from sync_state_machine.common.binding import BindingManager
|
||||
from sync_state_machine.common.collection import DataCollection
|
||||
from sync_state_machine.common.persistence import PersistenceBackend
|
||||
from sync_state_machine.common.sqlite_backend import SQLitePersistenceBackend
|
||||
from sync_state_machine.common.types import BindingStatus, SyncAction, SyncStatus
|
||||
from sync_state_machine.pipeline.full_sync_pipeline import FullSyncPipeline
|
||||
from sync_state_machine.sync_system.strategy_ops.compare_ops import (
|
||||
@@ -39,7 +40,7 @@ async def pipeline_bundle():
|
||||
register_test_domain()
|
||||
|
||||
db = Path(tempfile.mkdtemp(prefix="sync_state_machine_pipeline_") ) / "state.db"
|
||||
persistence = PersistenceBackend(str(db))
|
||||
persistence = SQLitePersistenceBackend(str(db))
|
||||
await persistence.initialize()
|
||||
|
||||
local_collection = DataCollection("local", persistence=persistence, auto_persist=False)
|
||||
@@ -304,7 +305,7 @@ async def test_load_overrides_bind_data_id_from_bindings(pipeline_bundle):
|
||||
|
||||
existing_persistence = local_collection.persistence
|
||||
assert existing_persistence is not None
|
||||
persistence = PersistenceBackend(existing_persistence.db_path)
|
||||
persistence = SQLitePersistenceBackend(existing_persistence.db_path)
|
||||
await persistence.initialize()
|
||||
|
||||
await persistence.conn.execute(
|
||||
@@ -443,7 +444,7 @@ async def test_pipeline_create_then_update_completes_in_single_run_with_reload()
|
||||
register_test_domain()
|
||||
|
||||
db = Path(tempfile.mkdtemp(prefix="sync_state_machine_pipeline_reload_") ) / "state.db"
|
||||
persistence = PersistenceBackend(str(db))
|
||||
persistence = SQLitePersistenceBackend(str(db))
|
||||
await persistence.initialize()
|
||||
|
||||
local_collection = DataCollection("local", persistence=persistence, auto_persist=False)
|
||||
|
||||
@@ -8,6 +8,7 @@ import pytest
|
||||
from sync_state_machine.common.binding import BindingManager
|
||||
from sync_state_machine.common.collection import DataCollection
|
||||
from sync_state_machine.common.persistence import PersistenceBackend
|
||||
from sync_state_machine.common.sqlite_backend import SQLitePersistenceBackend
|
||||
from sync_state_machine.pipeline.full_sync_pipeline import FullSyncPipeline
|
||||
from sync_state_machine.sync_system.config import OrphanAction, UpdateDirection
|
||||
from tests.fixtures.mock_domain import (
|
||||
@@ -36,7 +37,7 @@ async def test_full_pipeline_multinode_mock_persisted_db_snapshot() -> None:
|
||||
if db_path.exists():
|
||||
db_path.unlink()
|
||||
|
||||
persistence = PersistenceBackend(str(db_path))
|
||||
persistence = SQLitePersistenceBackend(str(db_path))
|
||||
await persistence.initialize()
|
||||
|
||||
local_collection = DataCollection("local", persistence=persistence, auto_persist=False)
|
||||
|
||||
@@ -7,6 +7,7 @@ import pytest
|
||||
|
||||
from sync_state_machine.common.collection import DataCollection
|
||||
from sync_state_machine.common.persistence import PersistenceBackend
|
||||
from sync_state_machine.common.sqlite_backend import SQLitePersistenceBackend
|
||||
from sync_state_machine.common.types import BindingStatus, SyncAction, SyncStatus
|
||||
from sync_state_machine.engine.dispatcher import StateMachineRuntime
|
||||
from sync_state_machine.engine.model import StateMachineConfig
|
||||
@@ -21,7 +22,7 @@ from tests.mocks.mock_datasource import (
|
||||
@pytest.fixture
|
||||
def datasource_bundle():
|
||||
db = Path(tempfile.mkdtemp(prefix="sync_state_machine_mock_ds_")) / "state.db"
|
||||
persistence = PersistenceBackend(str(db))
|
||||
persistence = SQLitePersistenceBackend(str(db))
|
||||
cfg_path = Path(__file__).resolve().parents[2] / "sync_state_machine" / "config" / "node_state_machine.yaml"
|
||||
runtime = StateMachineRuntime(StateMachineConfig.load(cfg_path))
|
||||
collection = DataCollection("mock", persistence=None, auto_persist=False)
|
||||
@@ -49,7 +50,7 @@ async def test_create_success_to_s11(datasource_bundle):
|
||||
node = _new_node("n1", SyncAction.CREATE)
|
||||
await collection.add(node)
|
||||
|
||||
datasource.register_handler(ScriptedMockHandler(HandlerScript(create="success")))
|
||||
datasource.add_handler(ScriptedMockHandler(HandlerScript(create="success")))
|
||||
datasource.set_collection(collection)
|
||||
await datasource.sync_all(data_type="mock")
|
||||
|
||||
@@ -62,7 +63,7 @@ async def test_create_skipped_is_rejected(datasource_bundle):
|
||||
node = _new_node("n2", SyncAction.CREATE)
|
||||
await collection.add(node)
|
||||
|
||||
datasource.register_handler(ScriptedMockHandler(HandlerScript(create="skipped")))
|
||||
datasource.add_handler(ScriptedMockHandler(HandlerScript(create="skipped")))
|
||||
datasource.set_collection(collection)
|
||||
await datasource.sync_all(data_type="mock")
|
||||
|
||||
@@ -76,7 +77,7 @@ async def test_update_skip_downgrade_to_s14(datasource_bundle):
|
||||
node = _new_node("n3", SyncAction.UPDATE)
|
||||
await collection.add(node)
|
||||
|
||||
datasource.register_handler(
|
||||
datasource.add_handler(
|
||||
ScriptedMockHandler(HandlerScript(update="skipped"))
|
||||
)
|
||||
datasource.set_collection(collection)
|
||||
@@ -94,7 +95,7 @@ async def test_update_success_writes_back_origin_data(datasource_bundle):
|
||||
node.set_data({"id": "n3b", "name": "after"})
|
||||
await collection.add(node)
|
||||
|
||||
datasource.register_handler(ScriptedMockHandler(HandlerScript(update="success")))
|
||||
datasource.add_handler(ScriptedMockHandler(HandlerScript(update="success")))
|
||||
datasource.set_collection(collection)
|
||||
await datasource.sync_all(data_type="mock")
|
||||
|
||||
@@ -109,7 +110,7 @@ async def test_in_progress_poll_success_to_s11(datasource_bundle):
|
||||
node = _new_node("n4", SyncAction.CREATE)
|
||||
await collection.add(node)
|
||||
|
||||
datasource.register_handler(
|
||||
datasource.add_handler(
|
||||
ScriptedMockHandler(HandlerScript(create="in_progress", poll_sequence=["success"]))
|
||||
)
|
||||
datasource.set_collection(collection)
|
||||
@@ -124,7 +125,7 @@ async def test_in_progress_poll_timeout_to_s12(datasource_bundle):
|
||||
node = _new_node("n5", SyncAction.CREATE)
|
||||
await collection.add(node)
|
||||
|
||||
datasource.register_handler(
|
||||
datasource.add_handler(
|
||||
ScriptedMockHandler(HandlerScript(create="in_progress", poll_sequence=["in_progress", "in_progress"]))
|
||||
)
|
||||
datasource.set_collection(collection)
|
||||
|
||||
@@ -10,6 +10,7 @@ import pytest
|
||||
from sync_state_machine.common.collection import DataCollection
|
||||
from sync_state_machine.common.binding import BindingManager
|
||||
from sync_state_machine.common.persistence import PersistenceBackend
|
||||
from sync_state_machine.common.sqlite_backend import SQLitePersistenceBackend
|
||||
from sync_state_machine.common.types import BindingStatus, SyncAction, SyncStatus
|
||||
from sync_state_machine.engine import StateMachineConfig, StateMachineRuntime
|
||||
from sync_state_machine.engine.events import (
|
||||
@@ -464,7 +465,7 @@ async def test_sync_log_cleared_on_initial_load(tmp_path: Path) -> None:
|
||||
REPORT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
db = tmp_path / "sync_log_clear_state.db"
|
||||
|
||||
persistence = PersistenceBackend(str(db))
|
||||
persistence = SQLitePersistenceBackend(str(db))
|
||||
await persistence.initialize()
|
||||
collection = DataCollection("project-log-clear", persistence=persistence, auto_persist=False)
|
||||
|
||||
@@ -531,7 +532,7 @@ async def test_load_preserves_state_then_e01_normalizes_to_s00(tmp_path: Path) -
|
||||
REPORT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
db = tmp_path / "load_preserve_then_e01_normalize.db"
|
||||
|
||||
persistence = PersistenceBackend(str(db))
|
||||
persistence = SQLitePersistenceBackend(str(db))
|
||||
await persistence.initialize()
|
||||
|
||||
collection = DataCollection("project-e01-normalize", persistence=persistence, auto_persist=False)
|
||||
@@ -583,7 +584,7 @@ async def test_invalid_persisted_enum_is_bootstrap_blocked_and_skipped_in_reset(
|
||||
REPORT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
db = tmp_path / "invalid_enum_bootstrap_blocked.db"
|
||||
|
||||
persistence = PersistenceBackend(str(db))
|
||||
persistence = SQLitePersistenceBackend(str(db))
|
||||
await persistence.initialize()
|
||||
|
||||
collection_id = "project-invalid-enum"
|
||||
|
||||
@@ -9,6 +9,7 @@ import pytest_asyncio
|
||||
from sync_state_machine.common.binding import BindingManager
|
||||
from sync_state_machine.common.collection import DataCollection
|
||||
from sync_state_machine.common.persistence import PersistenceBackend
|
||||
from sync_state_machine.common.sqlite_backend import SQLitePersistenceBackend
|
||||
from sync_state_machine.common.types import BindingStatus, SyncAction, SyncStatus
|
||||
from sync_state_machine.sync_system.resolve_ids import IDResolver
|
||||
from sync_state_machine.sync_system.config import OrphanAction, UpdateDirection
|
||||
@@ -32,7 +33,7 @@ from tests.fixtures.mock_domain import (
|
||||
async def setup_env():
|
||||
register_test_domain()
|
||||
db = Path(tempfile.mkdtemp(prefix="sync_state_machine_biz_") ) / "state.db"
|
||||
persistence = PersistenceBackend(str(db))
|
||||
persistence = SQLitePersistenceBackend(str(db))
|
||||
await persistence.initialize()
|
||||
|
||||
local = DataCollection("local", persistence=persistence, auto_persist=False)
|
||||
|
||||
@@ -9,7 +9,7 @@ from pydantic import BaseModel
|
||||
from sync_state_machine.config.multi_project_config import build_multi_project_config_from_file
|
||||
from sync_state_machine.common.binding import BindingManager
|
||||
from sync_state_machine.common.collection import DataCollection
|
||||
from sync_state_machine.common.persistence import PersistenceBackend
|
||||
from sync_state_machine.common.sqlite_backend import SQLitePersistenceBackend
|
||||
from sync_state_machine.common.sync_node import SyncNode
|
||||
from sync_state_machine.pipeline.multi_project_pipeline import (
|
||||
_project_scope_overrides,
|
||||
@@ -88,7 +88,7 @@ class _ProjectNode(SyncNode[dict]):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_binding_manager_prunes_stale_bindings_after_project_scope_cleanup() -> None:
|
||||
persistence = PersistenceBackend(":memory:")
|
||||
persistence = SQLitePersistenceBackend(":memory:")
|
||||
await persistence.initialize()
|
||||
try:
|
||||
local_collection = DataCollection("local", persistence=persistence, auto_persist=False)
|
||||
|
||||
@@ -7,6 +7,7 @@ import pytest
|
||||
from sync_state_machine.common.binding import BindingManager
|
||||
from sync_state_machine.common.collection import DataCollection
|
||||
from sync_state_machine.common.persistence import PersistenceBackend
|
||||
from sync_state_machine.common.sqlite_backend import SQLitePersistenceBackend
|
||||
from sync_state_machine.common.types import BindingStatus, SyncAction, SyncStatus
|
||||
from sync_state_machine.datasource.datasource import BaseDataSource
|
||||
from sync_state_machine.engine.state_apply import apply_state_to_node
|
||||
@@ -38,7 +39,7 @@ def _set_state(pipeline: FullSyncPipeline, node: MockSyncNode, state_id: str, *,
|
||||
|
||||
async def _new_pipeline(tmp_path: Path) -> tuple[FullSyncPipeline, DataCollection, DataCollection, BindingManager, PersistenceBackend]:
|
||||
db_path = tmp_path / "state.db"
|
||||
persistence = PersistenceBackend(str(db_path))
|
||||
persistence = SQLitePersistenceBackend(str(db_path))
|
||||
await persistence.initialize()
|
||||
|
||||
local_collection = DataCollection("local", persistence=persistence, auto_persist=False)
|
||||
|
||||
@@ -7,6 +7,7 @@ import pytest
|
||||
from sync_state_machine.common.binding import BindingManager
|
||||
from sync_state_machine.common.collection import DataCollection
|
||||
from sync_state_machine.common.persistence import PersistenceBackend
|
||||
from sync_state_machine.common.sqlite_backend import SQLitePersistenceBackend
|
||||
from sync_state_machine.datasource.datasource import BaseDataSource
|
||||
from sync_state_machine.pipeline.full_sync_pipeline import FullSyncPipeline
|
||||
from sync_state_machine.sync_system.strategy import BaseSyncStrategy
|
||||
@@ -53,7 +54,7 @@ class _OkStrategy(BaseSyncStrategy):
|
||||
@pytest.mark.asyncio
|
||||
async def test_phase_sync_by_node_type_continues_after_strategy_error(tmp_path: Path) -> None:
|
||||
db_path = tmp_path / "state.db"
|
||||
persistence = PersistenceBackend(str(db_path))
|
||||
persistence = SQLitePersistenceBackend(str(db_path))
|
||||
await persistence.initialize()
|
||||
|
||||
local_collection = DataCollection("local", persistence=persistence, auto_persist=False)
|
||||
@@ -101,7 +102,7 @@ async def test_phase_sync_by_node_type_continues_after_strategy_error(tmp_path:
|
||||
@pytest.mark.asyncio
|
||||
async def test_phase_sync_by_node_type_skip_sync_keeps_bind_but_skips_writes(tmp_path: Path) -> None:
|
||||
db_path = tmp_path / "state.db"
|
||||
persistence = PersistenceBackend(str(db_path))
|
||||
persistence = SQLitePersistenceBackend(str(db_path))
|
||||
await persistence.initialize()
|
||||
|
||||
local_collection = DataCollection("local", persistence=persistence, auto_persist=False)
|
||||
|
||||
@@ -8,6 +8,7 @@ import pytest
|
||||
from sync_state_machine.common.binding import BindingManager
|
||||
from sync_state_machine.common.collection import DataCollection
|
||||
from sync_state_machine.common.persistence import PersistenceBackend
|
||||
from sync_state_machine.common.sqlite_backend import SQLitePersistenceBackend
|
||||
from sync_state_machine.common.types import BindingStatus, SyncAction, SyncStatus
|
||||
from sync_state_machine.domain.construction.sync_node import ConstructionTaskSyncNode
|
||||
from sync_state_machine.domain.construction.sync_strategy import ConstructionTaskSyncStrategy
|
||||
@@ -19,7 +20,7 @@ from sync_state_machine.sync_system.strategy_ops.post_check_ops import evaluate_
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_check_uses_same_id_resolver_as_create_update_for_depend_ids(tmp_path: Path) -> None:
|
||||
db_path = tmp_path / "state.db"
|
||||
persistence = PersistenceBackend(str(db_path))
|
||||
persistence = SQLitePersistenceBackend(str(db_path))
|
||||
await persistence.initialize()
|
||||
|
||||
local_collection = DataCollection("local", persistence=persistence, auto_persist=False)
|
||||
@@ -167,7 +168,7 @@ async def test_post_check_uses_same_id_resolver_as_create_update_for_depend_ids(
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_check_ignores_only_explicit_configured_fields(tmp_path: Path) -> None:
|
||||
db_path = tmp_path / "state.db"
|
||||
persistence = PersistenceBackend(str(db_path))
|
||||
persistence = SQLitePersistenceBackend(str(db_path))
|
||||
await persistence.initialize()
|
||||
|
||||
local_collection = DataCollection("local", persistence=persistence, auto_persist=False)
|
||||
|
||||
@@ -7,6 +7,7 @@ import pytest
|
||||
from sync_state_machine.common.binding import BindingManager
|
||||
from sync_state_machine.common.collection import DataCollection
|
||||
from sync_state_machine.common.persistence import PersistenceBackend
|
||||
from sync_state_machine.common.sqlite_backend import SQLitePersistenceBackend
|
||||
from tests.fixtures.mock_domain import build_project_node, register_test_domain
|
||||
|
||||
|
||||
@@ -15,7 +16,7 @@ async def test_scope_partitioning_and_copy_between_scopes(tmp_path: Path) -> Non
|
||||
register_test_domain()
|
||||
|
||||
db_path = tmp_path / "scope_partitioning.db"
|
||||
persistence = PersistenceBackend(str(db_path))
|
||||
persistence = SQLitePersistenceBackend(str(db_path))
|
||||
await persistence.initialize()
|
||||
|
||||
global_collection = DataCollection("local", scope="global", persistence=persistence, auto_persist=False)
|
||||
|
||||
+4
-5
@@ -89,11 +89,10 @@
|
||||
|
||||
<div class="row">
|
||||
<label>Persist Backend
|
||||
<select id="persistBackend">
|
||||
<option value="sqlite">sqlite</option>
|
||||
<option value="sqlalchemy">sqlalchemy</option>
|
||||
<option value="custom">custom</option>
|
||||
</select>
|
||||
<input id="persistBackend" list="persistBackendOptions" />
|
||||
<datalist id="persistBackendOptions">
|
||||
<option value="sqlite"></option>
|
||||
</datalist>
|
||||
</label>
|
||||
<label>持久化 DB 路径
|
||||
<input id="persistenceDb" />
|
||||
|
||||
Reference in New Issue
Block a user