Add indexes on high-traffic foreign keys and sort columns

Queries throughout the codebase filter on generations.profile_id,
generations.status, generations.created_at, story_items.story_id,
story_items.generation_id, generation_versions.generation_id, and
profile_samples.profile_id with every request. Without indexes SQLite
falls back to a full table scan; as history grows (hundreds or thousands
of generations) these scans become the dominant latency.

Changes:
- Add index=True on the most-queried FK and sort columns in models.py so
  new installs get them from Base.metadata.create_all
- Add _migrate_add_indexes() called from run_migrations() so existing
  installs get the same indexes on next startup (uses CREATE INDEX IF
  NOT EXISTS — idempotent, <10 ms on any realistic dataset)
This commit is contained in:
Will Anderson
2026-10-04 00:00:41 +00:00
committed by capy-ai-staging[bot]
parent ccc092bad0
commit b97ffbfba9
2 changed files with 51 additions and 10 deletions
+41
View File
@@ -44,6 +44,7 @@ def run_migrations(engine) -> None:
_migrate_capture_settings(engine, inspector, tables) _migrate_capture_settings(engine, inspector, tables)
_migrate_mcp_bindings(engine, inspector, tables) _migrate_mcp_bindings(engine, inspector, tables)
_normalize_storage_paths(engine, tables) _normalize_storage_paths(engine, tables)
_migrate_add_indexes(engine, tables)
# -- helpers --------------------------------------------------------------- # -- helpers ---------------------------------------------------------------
@@ -292,6 +293,46 @@ def _supports_drop_column(engine) -> bool:
return tuple(int(p) for p in sqlite3.sqlite_version.split(".")[:3]) >= (3, 35, 0) return tuple(int(p) for p in sqlite3.sqlite_version.split(".")[:3]) >= (3, 35, 0)
def _migrate_add_indexes(engine, tables: set[str]) -> None:
"""Create missing indexes on high-traffic foreign keys and sort columns.
SQLite silently ignores ``CREATE INDEX IF NOT EXISTS``, so this is
safe to run on every startup regardless of whether the index already
exists. New installs get the indexes from ``Base.metadata.create_all``
(via the ``index=True`` column flags); this migration brings existing
databases into parity without dropping or recreating any data.
"""
indexes = [
# generations — filtered by profile, ordered/filtered by date, filtered by status
("ix_generations_profile_id", "generations", "profile_id"),
("ix_generations_created_at", "generations", "created_at"),
("ix_generations_status", "generations", "status"),
# story_items — every story lookup filters by story_id; join on generation_id
("ix_story_items_story_id", "story_items", "story_id"),
("ix_story_items_generation_id", "story_items", "generation_id"),
# generation_versions — always filtered/joined on generation_id
("ix_generation_versions_generation_id", "generation_versions", "generation_id"),
# profile_samples — loaded per-profile on every voice prompt build
("ix_profile_samples_profile_id", "profile_samples", "profile_id"),
# captures — ordered by date in list view
("ix_captures_created_at", "captures", "created_at"),
# channel_device_mappings — looked up per channel
("ix_channel_device_mappings_channel_id", "channel_device_mappings", "channel_id"),
]
with engine.connect() as conn:
for index_name, table, column in indexes:
if table not in tables:
continue
conn.execute(
text(
f"CREATE INDEX IF NOT EXISTS {index_name}"
f" ON {table} ({column})"
)
)
conn.commit()
def _normalize_storage_paths(engine, tables: set[str]) -> None: def _normalize_storage_paths(engine, tables: set[str]) -> None:
"""Normalize stored file paths to be relative to the configured data dir.""" """Normalize stored file paths to be relative to the configured data dir."""
from pathlib import Path from pathlib import Path
+10 -10
View File
@@ -3,7 +3,7 @@
from datetime import datetime from datetime import datetime
import uuid import uuid
from sqlalchemy import Column, String, Integer, Float, DateTime, Text, ForeignKey, Boolean, JSON from sqlalchemy import Column, Index, String, Integer, Float, DateTime, Text, ForeignKey, Boolean, JSON
from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.ext.declarative import declarative_base
from ..utils.capture_chords import ( from ..utils.capture_chords import (
@@ -54,7 +54,7 @@ class ProfileSample(Base):
__tablename__ = "profile_samples" __tablename__ = "profile_samples"
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4())) id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
profile_id = Column(String, ForeignKey("profiles.id"), nullable=False) profile_id = Column(String, ForeignKey("profiles.id"), nullable=False, index=True)
audio_path = Column(String, nullable=False) audio_path = Column(String, nullable=False)
reference_text = Column(Text, nullable=False) reference_text = Column(Text, nullable=False)
@@ -65,7 +65,7 @@ class Generation(Base):
__tablename__ = "generations" __tablename__ = "generations"
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4())) id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
profile_id = Column(String, ForeignKey("profiles.id"), nullable=False) profile_id = Column(String, ForeignKey("profiles.id"), nullable=False, index=True)
text = Column(Text, nullable=False) text = Column(Text, nullable=False)
language = Column(String, default="en") language = Column(String, default="en")
audio_path = Column(String, nullable=True) audio_path = Column(String, nullable=True)
@@ -74,7 +74,7 @@ class Generation(Base):
instruct = Column(Text) instruct = Column(Text)
engine = Column(String, default="qwen") engine = Column(String, default="qwen")
model_size = Column(String, nullable=True) model_size = Column(String, nullable=True)
status = Column(String, default="completed") status = Column(String, default="completed", index=True)
error = Column(Text, nullable=True) error = Column(Text, nullable=True)
is_favorited = Column(Boolean, default=False) is_favorited = Column(Boolean, default=False)
# Origin of this generation — "manual" for plain /generate calls, # Origin of this generation — "manual" for plain /generate calls,
@@ -82,7 +82,7 @@ class Generation(Base):
# profile's personality LLM before TTS. Future sources (bulk import, # profile's personality LLM before TTS. Future sources (bulk import,
# agent replies, etc.) can extend this. # agent replies, etc.) can extend this.
source = Column(String, nullable=False, default="manual") source = Column(String, nullable=False, default="manual")
created_at = Column(DateTime, default=datetime.utcnow) created_at = Column(DateTime, default=datetime.utcnow, index=True)
class Story(Base): class Story(Base):
@@ -103,8 +103,8 @@ class StoryItem(Base):
__tablename__ = "story_items" __tablename__ = "story_items"
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4())) id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
story_id = Column(String, ForeignKey("stories.id"), nullable=False) story_id = Column(String, ForeignKey("stories.id"), nullable=False, index=True)
generation_id = Column(String, ForeignKey("generations.id"), nullable=False) generation_id = Column(String, ForeignKey("generations.id"), nullable=False, index=True)
version_id = Column(String, ForeignKey("generation_versions.id"), nullable=True) version_id = Column(String, ForeignKey("generation_versions.id"), nullable=True)
start_time_ms = Column(Integer, nullable=False, default=0) start_time_ms = Column(Integer, nullable=False, default=0)
track = Column(Integer, nullable=False, default=0) track = Column(Integer, nullable=False, default=0)
@@ -132,7 +132,7 @@ class GenerationVersion(Base):
__tablename__ = "generation_versions" __tablename__ = "generation_versions"
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4())) id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
generation_id = Column(String, ForeignKey("generations.id"), nullable=False) generation_id = Column(String, ForeignKey("generations.id"), nullable=False, index=True)
label = Column(String, nullable=False) label = Column(String, nullable=False)
audio_path = Column(String, nullable=False) audio_path = Column(String, nullable=False)
effects_chain = Column(Text, nullable=True) effects_chain = Column(Text, nullable=True)
@@ -172,7 +172,7 @@ class ChannelDeviceMapping(Base):
__tablename__ = "channel_device_mappings" __tablename__ = "channel_device_mappings"
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4())) id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
channel_id = Column(String, ForeignKey("audio_channels.id"), nullable=False) channel_id = Column(String, ForeignKey("audio_channels.id"), nullable=False, index=True)
device_id = Column(String, nullable=False) device_id = Column(String, nullable=False)
@@ -300,4 +300,4 @@ class Capture(Base):
stt_model = Column(String, nullable=True) stt_model = Column(String, nullable=True)
llm_model = Column(String, nullable=True) llm_model = Column(String, nullable=True)
refinement_flags = Column(Text, nullable=True) # JSON blob refinement_flags = Column(Text, nullable=True) # JSON blob
created_at = Column(DateTime, default=datetime.utcnow) created_at = Column(DateTime, default=datetime.utcnow, index=True)