From b97ffbfba970dd97c85bc93e941f6a861decf54a Mon Sep 17 00:00:00 2001 From: Will Anderson Date: Thu, 14 May 2026 12:08:07 -0500 Subject: [PATCH] Add indexes on high-traffic foreign keys and sort columns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Queries throughout the codebase filter on generations.profile_id, generations.status, generations.created_at, story_items.story_id, story_items.generation_id, generation_versions.generation_id, and profile_samples.profile_id with every request. Without indexes SQLite falls back to a full table scan; as history grows (hundreds or thousands of generations) these scans become the dominant latency. Changes: - Add index=True on the most-queried FK and sort columns in models.py so new installs get them from Base.metadata.create_all - Add _migrate_add_indexes() called from run_migrations() so existing installs get the same indexes on next startup (uses CREATE INDEX IF NOT EXISTS — idempotent, <10 ms on any realistic dataset) --- backend/database/migrations.py | 41 ++++++++++++++++++++++++++++++++++ backend/database/models.py | 20 ++++++++--------- 2 files changed, 51 insertions(+), 10 deletions(-) diff --git a/backend/database/migrations.py b/backend/database/migrations.py index d353b58c..12f9cdd4 100644 --- a/backend/database/migrations.py +++ b/backend/database/migrations.py @@ -44,6 +44,7 @@ def run_migrations(engine) -> None: _migrate_capture_settings(engine, inspector, tables) _migrate_mcp_bindings(engine, inspector, tables) _normalize_storage_paths(engine, tables) + _migrate_add_indexes(engine, tables) # -- helpers --------------------------------------------------------------- @@ -292,6 +293,46 @@ def _supports_drop_column(engine) -> bool: return tuple(int(p) for p in sqlite3.sqlite_version.split(".")[:3]) >= (3, 35, 0) +def _migrate_add_indexes(engine, tables: set[str]) -> None: + """Create missing indexes on high-traffic foreign keys and sort columns. + + SQLite silently ignores ``CREATE INDEX IF NOT EXISTS``, so this is + safe to run on every startup regardless of whether the index already + exists. New installs get the indexes from ``Base.metadata.create_all`` + (via the ``index=True`` column flags); this migration brings existing + databases into parity without dropping or recreating any data. + """ + indexes = [ + # generations — filtered by profile, ordered/filtered by date, filtered by status + ("ix_generations_profile_id", "generations", "profile_id"), + ("ix_generations_created_at", "generations", "created_at"), + ("ix_generations_status", "generations", "status"), + # story_items — every story lookup filters by story_id; join on generation_id + ("ix_story_items_story_id", "story_items", "story_id"), + ("ix_story_items_generation_id", "story_items", "generation_id"), + # generation_versions — always filtered/joined on generation_id + ("ix_generation_versions_generation_id", "generation_versions", "generation_id"), + # profile_samples — loaded per-profile on every voice prompt build + ("ix_profile_samples_profile_id", "profile_samples", "profile_id"), + # captures — ordered by date in list view + ("ix_captures_created_at", "captures", "created_at"), + # channel_device_mappings — looked up per channel + ("ix_channel_device_mappings_channel_id", "channel_device_mappings", "channel_id"), + ] + + with engine.connect() as conn: + for index_name, table, column in indexes: + if table not in tables: + continue + conn.execute( + text( + f"CREATE INDEX IF NOT EXISTS {index_name}" + f" ON {table} ({column})" + ) + ) + conn.commit() + + def _normalize_storage_paths(engine, tables: set[str]) -> None: """Normalize stored file paths to be relative to the configured data dir.""" from pathlib import Path diff --git a/backend/database/models.py b/backend/database/models.py index b85a55b1..3ba75fc8 100644 --- a/backend/database/models.py +++ b/backend/database/models.py @@ -3,7 +3,7 @@ from datetime import datetime import uuid -from sqlalchemy import Column, String, Integer, Float, DateTime, Text, ForeignKey, Boolean, JSON +from sqlalchemy import Column, Index, String, Integer, Float, DateTime, Text, ForeignKey, Boolean, JSON from sqlalchemy.ext.declarative import declarative_base from ..utils.capture_chords import ( @@ -54,7 +54,7 @@ class ProfileSample(Base): __tablename__ = "profile_samples" id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4())) - profile_id = Column(String, ForeignKey("profiles.id"), nullable=False) + profile_id = Column(String, ForeignKey("profiles.id"), nullable=False, index=True) audio_path = Column(String, nullable=False) reference_text = Column(Text, nullable=False) @@ -65,7 +65,7 @@ class Generation(Base): __tablename__ = "generations" id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4())) - profile_id = Column(String, ForeignKey("profiles.id"), nullable=False) + profile_id = Column(String, ForeignKey("profiles.id"), nullable=False, index=True) text = Column(Text, nullable=False) language = Column(String, default="en") audio_path = Column(String, nullable=True) @@ -74,7 +74,7 @@ class Generation(Base): instruct = Column(Text) engine = Column(String, default="qwen") model_size = Column(String, nullable=True) - status = Column(String, default="completed") + status = Column(String, default="completed", index=True) error = Column(Text, nullable=True) is_favorited = Column(Boolean, default=False) # Origin of this generation — "manual" for plain /generate calls, @@ -82,7 +82,7 @@ class Generation(Base): # profile's personality LLM before TTS. Future sources (bulk import, # agent replies, etc.) can extend this. source = Column(String, nullable=False, default="manual") - created_at = Column(DateTime, default=datetime.utcnow) + created_at = Column(DateTime, default=datetime.utcnow, index=True) class Story(Base): @@ -103,8 +103,8 @@ class StoryItem(Base): __tablename__ = "story_items" id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4())) - story_id = Column(String, ForeignKey("stories.id"), nullable=False) - generation_id = Column(String, ForeignKey("generations.id"), nullable=False) + story_id = Column(String, ForeignKey("stories.id"), nullable=False, index=True) + generation_id = Column(String, ForeignKey("generations.id"), nullable=False, index=True) version_id = Column(String, ForeignKey("generation_versions.id"), nullable=True) start_time_ms = Column(Integer, nullable=False, default=0) track = Column(Integer, nullable=False, default=0) @@ -132,7 +132,7 @@ class GenerationVersion(Base): __tablename__ = "generation_versions" id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4())) - generation_id = Column(String, ForeignKey("generations.id"), nullable=False) + generation_id = Column(String, ForeignKey("generations.id"), nullable=False, index=True) label = Column(String, nullable=False) audio_path = Column(String, nullable=False) effects_chain = Column(Text, nullable=True) @@ -172,7 +172,7 @@ class ChannelDeviceMapping(Base): __tablename__ = "channel_device_mappings" id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4())) - channel_id = Column(String, ForeignKey("audio_channels.id"), nullable=False) + channel_id = Column(String, ForeignKey("audio_channels.id"), nullable=False, index=True) device_id = Column(String, nullable=False) @@ -300,4 +300,4 @@ class Capture(Base): stt_model = Column(String, nullable=True) llm_model = Column(String, nullable=True) refinement_flags = Column(Text, nullable=True) # JSON blob - created_at = Column(DateTime, default=datetime.utcnow) + created_at = Column(DateTime, default=datetime.utcnow, index=True)