Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion astrbot/core/db/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ async def insert_platform_message_history(
self,
platform_id: str,
user_id: str,
content: list[dict],
content: dict,
sender_id: str | None = None,
sender_name: str | None = None,
) -> None:
Expand Down Expand Up @@ -287,3 +287,14 @@ async def clear_preferences(self, scope: str, scope_id: str) -> None:
# async def get_llm_messages(self, cid: str) -> list[LLMMessage]:
# """Get all LLM messages for a specific conversation."""
# ...

@abc.abstractmethod
async def get_session_conversations(
self,
page: int = 1,
page_size: int = 20,
search_query: str | None = None,
platform: str | None = None,
) -> tuple[list[dict], int]:
"""Get paginated session conversations with joined conversation and persona details, support search and platform filter."""
...
12 changes: 8 additions & 4 deletions astrbot/core/db/po.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,9 @@ class Persona(SQLModel, table=True):

__tablename__ = "personas"

id: int = Field(primary_key=True, sa_column_kwargs={"autoincrement": True})
id: int | None = Field(
primary_key=True, sa_column_kwargs={"autoincrement": True}, default=None
)
persona_id: str = Field(max_length=255, nullable=False)
system_prompt: str = Field(sa_type=Text, nullable=False)
begin_dialogs: Optional[list] = Field(default=None, sa_type=JSON)
Expand Down Expand Up @@ -135,7 +137,9 @@ class PlatformMessageHistory(SQLModel, table=True):

__tablename__ = "platform_message_history"

id: int = Field(primary_key=True, sa_column_kwargs={"autoincrement": True})
id: int | None = Field(
primary_key=True, sa_column_kwargs={"autoincrement": True}, default=None
)
platform_id: str = Field(nullable=False)
user_id: str = Field(nullable=False) # An id of group, user in platform
sender_id: Optional[str] = Field(default=None) # ID of the sender in the platform
Expand All @@ -158,8 +162,8 @@ class Attachment(SQLModel, table=True):

__tablename__ = "attachments"

inner_attachment_id: int = Field(
primary_key=True, sa_column_kwargs={"autoincrement": True}
inner_attachment_id: int | None = Field(
primary_key=True, sa_column_kwargs={"autoincrement": True}, default=None
)
attachment_id: str = Field(
max_length=36,
Expand Down
178 changes: 144 additions & 34 deletions astrbot/core/db/sqlite.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,8 @@
SQLModel,
)

from sqlalchemy import select, update, delete, text
from sqlmodel import select, update, delete, text, func, or_, desc, col
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.sql import func
from sqlalchemy import or_

NOT_GIVEN = T.TypeVar("NOT_GIVEN")

Expand All @@ -42,10 +40,10 @@ async def initialize(self) -> None:

async def insert_platform_stats(
self,
platform_id: str,
platform_type: str,
count: int = 1,
timestamp: datetime = None,
platform_id,
platform_type,
count=1,
timestamp=None,
) -> None:
"""Insert a new platform statistic record."""
async with self.get_db() as session:
Expand Down Expand Up @@ -76,7 +74,9 @@ async def count_platform_stats(self) -> int:
async with self.get_db() as session:
session: AsyncSession
result = await session.execute(
select(func.count(PlatformStat.platform_id)).select_from(PlatformStat)
select(func.count(col(PlatformStat.platform_id))).select_from(
PlatformStat
)
)
count = result.scalar_one_or_none()
return count if count is not None else 0
Expand All @@ -96,7 +96,7 @@ async def get_platform_stats(self, offset_sec: int = 86400) -> T.List[PlatformSt
"""),
{"start_time": start_time},
)
return result.scalars().all()
return list(result.scalars().all())

# ====
# Conversation Management
Expand All @@ -112,7 +112,7 @@ async def get_conversations(self, user_id=None, platform_id=None):
if platform_id:
query = query.where(ConversationV2.platform_id == platform_id)
# order by
query = query.order_by(ConversationV2.created_at.desc())
query = query.order_by(desc(ConversationV2.created_at))
result = await session.execute(query)

return result.scalars().all()
Expand All @@ -130,7 +130,7 @@ async def get_all_conversations(self, page=1, page_size=20):
offset = (page - 1) * page_size
result = await session.execute(
select(ConversationV2)
.order_by(ConversationV2.created_at.desc())
.order_by(desc(ConversationV2.created_at))
.offset(offset)
.limit(page_size)
)
Expand All @@ -151,25 +151,25 @@ async def get_filtered_conversations(

if platform_ids:
base_query = base_query.where(
ConversationV2.platform_id.in_(platform_ids)
col(ConversationV2.platform_id).in_(platform_ids)
)
if search_query:
search_query = search_query.encode("unicode_escape").decode("utf-8")
base_query = base_query.where(
or_(
ConversationV2.title.ilike(f"%{search_query}%"),
ConversationV2.content.ilike(f"%{search_query}%"),
ConversationV2.user_id.ilike(f"%{search_query}%"),
col(ConversationV2.title).ilike(f"%{search_query}%"),
col(ConversationV2.content).ilike(f"%{search_query}%"),
col(ConversationV2.user_id).ilike(f"%{search_query}%"),
)
)
if "message_types" in kwargs and len(kwargs["message_types"]) > 0:
for msg_type in kwargs["message_types"]:
base_query = base_query.where(
ConversationV2.user_id.ilike(f"%:{msg_type}:%")
col(ConversationV2.user_id).ilike(f"%:{msg_type}:%")
)
if "platforms" in kwargs and len(kwargs["platforms"]) > 0:
base_query = base_query.where(
ConversationV2.platform_id.in_(kwargs["platforms"])
col(ConversationV2.platform_id).in_(kwargs["platforms"])
)

# Get total count matching the filters
Expand All @@ -180,7 +180,7 @@ async def get_filtered_conversations(
# Get paginated results
offset = (page - 1) * page_size
result_query = (
base_query.order_by(ConversationV2.created_at.desc())
base_query.order_by(desc(ConversationV2.created_at))
.offset(offset)
.limit(page_size)
)
Expand Down Expand Up @@ -226,7 +226,7 @@ async def update_conversation(self, cid, title=None, persona_id=None, content=No
session: AsyncSession
async with session.begin():
query = update(ConversationV2).where(
ConversationV2.conversation_id == cid
col(ConversationV2.conversation_id) == cid
)
values = {}
if title is not None:
Expand All @@ -246,16 +246,125 @@ async def delete_conversation(self, cid):
session: AsyncSession
async with session.begin():
await session.execute(
delete(ConversationV2).where(ConversationV2.conversation_id == cid)
delete(ConversationV2).where(
col(ConversationV2.conversation_id) == cid
)
)

async def delete_conversations_by_user_id(self, user_id: str) -> None:
async with self.get_db() as session:
session: AsyncSession
async with session.begin():
await session.execute(
delete(ConversationV2).where(ConversationV2.user_id == user_id)
delete(ConversationV2).where(col(ConversationV2.user_id) == user_id)
)

async def get_session_conversations(
self,
page=1,
page_size=20,
search_query=None,
platform=None,
) -> tuple[list[dict], int]:
"""Get paginated session conversations with joined conversation and persona details."""
async with self.get_db() as session:
session: AsyncSession
offset = (page - 1) * page_size

base_query = (
select(
col(Preference.scope_id).label("session_id"),
func.json_extract(Preference.value, "$.val").label(
"conversation_id"
), # type: ignore
col(ConversationV2.persona_id).label("persona_id"),
col(ConversationV2.title).label("title"),
col(Persona.persona_id).label("persona_name"),
)
.select_from(Preference)
.outerjoin(
ConversationV2,
func.json_extract(Preference.value, "$.val")
== ConversationV2.conversation_id,
)
.outerjoin(
Persona, col(ConversationV2.persona_id) == Persona.persona_id
)
.where(Preference.scope == "umo", Preference.key == "sel_conv_id")
)

# 搜索筛选
if search_query:
search_pattern = f"%{search_query}%"
base_query = base_query.where(
or_(
col(Preference.scope_id).ilike(search_pattern),
col(ConversationV2.title).ilike(search_pattern),
col(Persona.persona_id).ilike(search_pattern),
)
)

# 平台筛选
if platform:
platform_pattern = f"{platform}:%"
base_query = base_query.where(
col(Preference.scope_id).like(platform_pattern)
)

# 排序
base_query = base_query.order_by(Preference.scope_id)

# 分页结果
result_query = base_query.offset(offset).limit(page_size)
result = await session.execute(result_query)
rows = result.fetchall()

# 查询总数(应用相同的筛选条件)
count_base_query = (
select(func.count(col(Preference.scope_id)))
.select_from(Preference)
.outerjoin(
ConversationV2,
func.json_extract(Preference.value, "$.val")
== ConversationV2.conversation_id,
)
.outerjoin(
Persona, col(ConversationV2.persona_id) == Persona.persona_id
)
.where(Preference.scope == "umo", Preference.key == "sel_conv_id")
)

# 应用相同的搜索和平台筛选条件到计数查询
if search_query:
search_pattern = f"%{search_query}%"
count_base_query = count_base_query.where(
or_(
col(Preference.scope_id).ilike(search_pattern),
col(ConversationV2.title).ilike(search_pattern),
col(Persona.persona_id).ilike(search_pattern),
)
)

if platform:
platform_pattern = f"{platform}:%"
count_base_query = count_base_query.where(
col(Preference.scope_id).like(platform_pattern)
)

total_result = await session.execute(count_base_query)
total = total_result.scalar() or 0

sessions_data = [
{
"session_id": row.session_id,
"conversation_id": row.conversation_id,
"persona_id": row.persona_id,
"title": row.title,
"persona_name": row.persona_name,
}
for row in rows
]
return sessions_data, total

async def insert_platform_message_history(
self,
Expand Down Expand Up @@ -290,9 +399,9 @@ async def delete_platform_message_offset(
cutoff_time = now - timedelta(seconds=offset_sec)
await session.execute(
delete(PlatformMessageHistory).where(
PlatformMessageHistory.platform_id == platform_id,
PlatformMessageHistory.user_id == user_id,
PlatformMessageHistory.created_at < cutoff_time,
col(PlatformMessageHistory.platform_id) == platform_id,
col(PlatformMessageHistory.user_id) == user_id,
col(PlatformMessageHistory.created_at) < cutoff_time,
)
)

Expand All @@ -309,7 +418,7 @@ async def get_platform_message_history(
PlatformMessageHistory.platform_id == platform_id,
PlatformMessageHistory.user_id == user_id,
)
.order_by(PlatformMessageHistory.created_at.desc())
.order_by(desc(PlatformMessageHistory.created_at))
)
result = await session.execute(query.offset(offset).limit(page_size))
return result.scalars().all()
Expand All @@ -331,7 +440,7 @@ async def get_attachment_by_id(self, attachment_id):
"""Get an attachment by its ID."""
async with self.get_db() as session:
session: AsyncSession
query = select(Attachment).where(Attachment.id == attachment_id)
query = select(Attachment).where(Attachment.attachment_id == attachment_id)
result = await session.execute(query)
return result.scalar_one_or_none()

Expand Down Expand Up @@ -374,7 +483,7 @@ async def update_persona(
async with self.get_db() as session:
session: AsyncSession
async with session.begin():
query = update(Persona).where(Persona.persona_id == persona_id)
query = update(Persona).where(col(Persona.persona_id) == persona_id)
values = {}
if system_prompt is not None:
values["system_prompt"] = system_prompt
Expand All @@ -394,7 +503,7 @@ async def delete_persona(self, persona_id):
session: AsyncSession
async with session.begin():
await session.execute(
delete(Persona).where(Persona.persona_id == persona_id)
delete(Persona).where(col(Persona.persona_id) == persona_id)
)

async def insert_preference_or_update(self, scope, scope_id, key, value):
Expand Down Expand Up @@ -449,9 +558,9 @@ async def remove_preference(self, scope, scope_id, key):
async with session.begin():
await session.execute(
delete(Preference).where(
Preference.scope == scope,
Preference.scope_id == scope_id,
Preference.key == key,
col(Preference.scope) == scope,
col(Preference.scope_id) == scope_id,
col(Preference.key) == key,
)
)
await session.commit()
Expand All @@ -463,7 +572,8 @@ async def clear_preferences(self, scope, scope_id):
async with session.begin():
await session.execute(
delete(Preference).where(
Preference.scope == scope, Preference.scope_id == scope_id
col(Preference.scope) == scope,
col(Preference.scope_id) == scope_id,
)
)
await session.commit()
Expand All @@ -490,7 +600,7 @@ async def _inner():
DeprecatedPlatformStat(
name=data.platform_id,
count=data.count,
timestamp=data.timestamp.timestamp(),
timestamp=int(data.timestamp.timestamp()),
)
)
return deprecated_stats
Expand Down Expand Up @@ -548,7 +658,7 @@ async def _inner():
DeprecatedPlatformStat(
name=platform_id,
count=count,
timestamp=start_time.timestamp(),
timestamp=int(start_time.timestamp()),
)
)
return deprecated_stats
Expand Down
Loading