feat(stories): import external audio into the timeline (drag-drop + picker)

You can now drop a music file onto the story content area or pick one through the new "Import audio" button in the add-clip popover. Both call POST /generate/import which writes the file to data/generations/<id>.<ext>, probes duration via librosa, and inserts a Generation row pointing at a singleton "Imported Audio" profile (created lazily on first import). The existing addStoryItem flow takes over from there — the timeline doesn't care that the row didn't come out of TTS.

Engine field on the row is "import"; it's surfaced on StoryItemDetail so the chat list shows a music icon instead of the (missing) profile avatar and both the dropdown and the track-editor toolbar hide the Regenerate action — there's nothing to regenerate. Accepted formats: wav/mp3/flac/ogg/m4a/aac/webm, capped at 200 MB. Translation keys added across en/ja/zh-CN/zh-TW.
This commit is contained in:
Jamie Pine
2026-04-25 01:55:49 -07:00
parent 2a937983c5
commit c43f2d45cc
12 changed files with 245 additions and 22 deletions
+1
View File
@@ -597,6 +597,7 @@ class StoryItemDetail(BaseModel):
duration: float
seed: Optional[int]
instruct: Optional[str]
engine: Optional[str] = None
generation_created_at: datetime
# Versions available for this generation
versions: Optional[List["GenerationVersionResponse"]] = None
+101 -2
View File
@@ -3,22 +3,51 @@
import asyncio
import logging
import uuid
from pathlib import Path
from fastapi import APIRouter, Depends, HTTPException
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
from fastapi.responses import StreamingResponse
from sqlalchemy.orm import Session
from .. import models
from .. import config, models
from ..services import history, personality, profiles, tts
from ..database import Generation as DBGeneration, VoiceProfile as DBVoiceProfile, get_db
from ..services.generation import run_generation
from ..services.task_queue import cancel_generation as cancel_generation_job, enqueue_generation
from ..utils.audio import load_audio
from ..utils.tasks import get_task_manager
logger = logging.getLogger(__name__)
router = APIRouter()
IMPORTED_AUDIO_PROFILE_NAME = "Imported Audio"
IMPORT_AUDIO_EXTENSIONS = {".wav", ".mp3", ".flac", ".ogg", ".m4a", ".aac", ".webm"}
IMPORT_AUDIO_MAX_BYTES = 200 * 1024 * 1024 # 200 MB
def _get_or_create_import_profile(db: Session) -> DBVoiceProfile:
"""Singleton profile every imported audio clip points at — keeps the
Generation FK happy without making profile_id nullable across the schema."""
row = (
db.query(DBVoiceProfile)
.filter(DBVoiceProfile.name == IMPORTED_AUDIO_PROFILE_NAME)
.first()
)
if row is not None:
return row
row = DBVoiceProfile(
id=str(uuid.uuid4()),
name=IMPORTED_AUDIO_PROFILE_NAME,
description="External audio imported into a story timeline.",
language="en",
voice_type="import",
)
db.add(row)
db.commit()
db.refresh(row)
return row
def _resolve_generation_engine(data: models.GenerationRequest, profile) -> str:
return data.engine or getattr(profile, "default_engine", None) or getattr(profile, "preset_engine", None) or "qwen"
@@ -371,3 +400,73 @@ async def stream_speech(
media_type="audio/wav",
headers={"Content-Disposition": 'attachment; filename="speech.wav"'},
)
@router.post("/generate/import", response_model=models.GenerationResponse)
async def import_audio(
file: UploadFile = File(...),
db: Session = Depends(get_db),
):
"""Register an external audio file as a generation row.
Designed for the story timeline so users can drop in music or other
non-TTS audio. The row points at a singleton "Imported Audio" profile
so the existing generation/story plumbing keeps working unchanged."""
suffix = Path(file.filename or "").suffix.lower()
if suffix not in IMPORT_AUDIO_EXTENSIONS:
raise HTTPException(
status_code=400,
detail=f"Unsupported audio format '{suffix}'. Allowed: {sorted(IMPORT_AUDIO_EXTENSIONS)}",
)
chunks: list[bytes] = []
total = 0
while True:
chunk = await file.read(1024 * 1024)
if not chunk:
break
total += len(chunk)
if total > IMPORT_AUDIO_MAX_BYTES:
raise HTTPException(
status_code=413,
detail=f"File exceeds {IMPORT_AUDIO_MAX_BYTES // (1024 * 1024)} MB limit.",
)
chunks.append(chunk)
audio_bytes = b"".join(chunks)
if not audio_bytes:
raise HTTPException(status_code=400, detail="Empty audio file.")
generation_id = str(uuid.uuid4())
target = config.get_generations_dir() / f"{generation_id}{suffix}"
target.write_bytes(audio_bytes)
try:
audio, sr = load_audio(str(target))
duration = float(len(audio) / sr) if sr else 0.0
except Exception as decode_err:
try:
target.unlink()
except OSError:
pass
raise HTTPException(
status_code=400,
detail=f"Could not decode audio: {decode_err}",
) from decode_err
profile = _get_or_create_import_profile(db)
display_name = Path(file.filename or "Imported audio").stem or "Imported audio"
return await history.create_generation(
profile_id=profile.id,
text=display_name,
language="en",
audio_path=config.to_storage_path(target),
duration=duration,
seed=None,
db=db,
generation_id=generation_id,
status="completed",
engine="import",
model_size=None,
source="import",
)
+1
View File
@@ -69,6 +69,7 @@ def _build_item_detail(
duration=generation.duration,
seed=generation.seed,
instruct=generation.instruct,
engine=generation.engine,
generation_created_at=generation.created_at,
versions=versions,
active_version_id=active_version_id,