diff --git a/app/src/components/StoriesTab/StoryChatItem.tsx b/app/src/components/StoriesTab/StoryChatItem.tsx index dbc823f4..d3ff0833 100644 --- a/app/src/components/StoriesTab/StoryChatItem.tsx +++ b/app/src/components/StoriesTab/StoryChatItem.tsx @@ -1,6 +1,6 @@ import { useSortable } from '@dnd-kit/sortable'; import { CSS } from '@dnd-kit/utilities'; -import { GripVertical, Mic, MoreHorizontal, Play, RotateCcw, Trash2 } from 'lucide-react'; +import { GripVertical, Mic, MoreHorizontal, Music, Play, RotateCcw, Trash2 } from 'lucide-react'; import { useState } from 'react'; import { useTranslation } from 'react-i18next'; import { Button } from '@/components/ui/button'; @@ -85,7 +85,9 @@ export function StoryChatItem({ {/* Voice Avatar */}
- {!avatarError ? ( + {item.engine === 'import' ? ( + + ) : !avatarError ? ( {`${item.profile_name}(null); + const importInputRef = useRef(null); const pendingCount = useGenerationStore((s) => s.pendingGenerationIds.size); const addPendingGeneration = useGenerationStore((s) => s.addPendingGeneration); + const [isDraggingFile, setIsDraggingFile] = useState(false); + const [isImporting, setIsImporting] = useState(false); + const dragDepthRef = useRef(0); // Add generation popover state const [searchQuery, setSearchQuery] = useState(''); @@ -229,6 +233,33 @@ export function StoryContent() { ); }; + const handleImportAudio = async (file: File) => { + if (!story) return; + setIsImporting(true); + try { + const generation = await apiClient.importAudio(file); + await addStoryItem.mutateAsync({ + storyId: story.id, + data: { generation_id: generation.id }, + }); + setIsAddOpen(false); + } catch (error) { + toast({ + title: t('storyContent.toast.importFailed'), + description: error instanceof Error ? error.message : String(error), + variant: 'destructive', + }); + } finally { + setIsImporting(false); + } + }; + + const handleImportFiles = async (files: FileList | File[]) => { + for (const file of Array.from(files)) { + await handleImportAudio(file); + } + }; + const handleAddGeneration = (generationId: string) => { if (!story) return; @@ -284,7 +315,49 @@ export function StoryContent() { } return ( -
+
{ + if (!e.dataTransfer?.types.includes('Files')) return; + e.preventDefault(); + dragDepthRef.current += 1; + setIsDraggingFile(true); + }} + onDragOver={(e) => { + if (e.dataTransfer?.types.includes('Files')) e.preventDefault(); + }} + onDragLeave={(e) => { + if (!e.dataTransfer?.types.includes('Files')) return; + dragDepthRef.current = Math.max(0, dragDepthRef.current - 1); + if (dragDepthRef.current === 0) setIsDraggingFile(false); + }} + onDrop={(e) => { + if (!e.dataTransfer?.files?.length) return; + e.preventDefault(); + dragDepthRef.current = 0; + setIsDraggingFile(false); + handleImportFiles(e.dataTransfer.files); + }} + > + { + if (e.target.files?.length) handleImportFiles(e.target.files); + e.target.value = ''; + }} + /> + {isDraggingFile && ( +
+
+ + {t('storyContent.dropToImport')} +
+
+ )} {/* Scroll Mask */}
@@ -329,13 +402,23 @@ export function StoryContent() { -
+
setSearchQuery(e.target.value)} autoFocus /> +
{availableGenerations.length === 0 ? ( @@ -414,7 +497,11 @@ export function StoryContent() { storyId={story.id} index={index} onRemove={() => handleRemoveItem(item.id)} - onRegenerate={() => handleRegenerate(item.generation_id)} + onRegenerate={ + item.engine === 'import' + ? undefined + : () => handleRegenerate(item.generation_id) + } currentTimeMs={currentTimeMs} isPlaying={isPlaying && playbackStoryId === story.id} /> diff --git a/app/src/components/StoriesTab/StoryTrackEditor.tsx b/app/src/components/StoriesTab/StoryTrackEditor.tsx index 41e9cea7..ec1656db 100644 --- a/app/src/components/StoriesTab/StoryTrackEditor.tsx +++ b/app/src/components/StoriesTab/StoryTrackEditor.tsx @@ -1016,16 +1016,18 @@ export function StoryTrackEditor({ storyId, items }: StoryTrackEditorProps) { > - + {selectedItem?.engine !== 'import' && ( + + )} {hasMultipleVersions && ( <>
diff --git a/app/src/i18n/locales/en/translation.json b/app/src/i18n/locales/en/translation.json index 3137cee5..b9e0bd68 100644 --- a/app/src/i18n/locales/en/translation.json +++ b/app/src/i18n/locales/en/translation.json @@ -632,12 +632,16 @@ "regenerate": "Regenerate", "removeFromStory": "Remove from Story" }, + "importAudio": "Import audio…", + "importing": "Importing…", + "dropToImport": "Drop audio to import", "toast": { "removeFailed": "Failed to remove item", "reorderFailed": "Failed to reorder items", "exportFailed": "Failed to export audio", "addFailed": "Failed to add generation", - "regenerateFailed": "Failed to regenerate" + "regenerateFailed": "Failed to regenerate", + "importFailed": "Failed to import audio" } }, "history": { diff --git a/app/src/i18n/locales/ja/translation.json b/app/src/i18n/locales/ja/translation.json index 43da07e3..57a3c819 100644 --- a/app/src/i18n/locales/ja/translation.json +++ b/app/src/i18n/locales/ja/translation.json @@ -632,12 +632,16 @@ "regenerate": "再生成", "removeFromStory": "ストーリーから削除" }, + "importAudio": "オーディオをインポート…", + "importing": "インポート中…", + "dropToImport": "ドロップしてオーディオをインポート", "toast": { "removeFailed": "項目の削除に失敗しました", "reorderFailed": "項目の並び替えに失敗しました", "exportFailed": "オーディオのエクスポートに失敗しました", "addFailed": "生成の追加に失敗しました", - "regenerateFailed": "再生成に失敗しました" + "regenerateFailed": "再生成に失敗しました", + "importFailed": "オーディオのインポートに失敗しました" } }, "history": { diff --git a/app/src/i18n/locales/zh-CN/translation.json b/app/src/i18n/locales/zh-CN/translation.json index ec70d8cc..e1d6040b 100644 --- a/app/src/i18n/locales/zh-CN/translation.json +++ b/app/src/i18n/locales/zh-CN/translation.json @@ -632,12 +632,16 @@ "regenerate": "重新生成", "removeFromStory": "从故事中移除" }, + "importAudio": "导入音频…", + "importing": "正在导入…", + "dropToImport": "拖放以导入音频", "toast": { "removeFailed": "移除项目失败", "reorderFailed": "重新排序项目失败", "exportFailed": "导出音频失败", "addFailed": "添加生成失败", - "regenerateFailed": "重新生成失败" + "regenerateFailed": "重新生成失败", + "importFailed": "导入音频失败" } }, "history": { diff --git a/app/src/i18n/locales/zh-TW/translation.json b/app/src/i18n/locales/zh-TW/translation.json index 78fafd5a..fdb64bf2 100644 --- a/app/src/i18n/locales/zh-TW/translation.json +++ b/app/src/i18n/locales/zh-TW/translation.json @@ -632,12 +632,16 @@ "regenerate": "重新生成", "removeFromStory": "從故事中移除" }, + "importAudio": "匯入音訊…", + "importing": "匯入中…", + "dropToImport": "拖放以匯入音訊", "toast": { "removeFailed": "移除項目失敗", "reorderFailed": "重新排序項目失敗", "exportFailed": "匯出音訊失敗", "addFailed": "新增生成失敗", - "regenerateFailed": "重新生成失敗" + "regenerateFailed": "重新生成失敗", + "importFailed": "匯入音訊失敗" } }, "history": { diff --git a/app/src/lib/api/client.ts b/app/src/lib/api/client.ts index 2034d3ad..888a1e6c 100644 --- a/app/src/lib/api/client.ts +++ b/app/src/lib/api/client.ts @@ -272,6 +272,20 @@ class ApiClient { }); } + async importAudio(file: File): Promise { + const form = new FormData(); + form.append('file', file); + const res = await fetch(`${this.getBaseUrl()}/generate/import`, { + method: 'POST', + body: form, + }); + if (!res.ok) { + const detail = await res.text().catch(() => res.statusText); + throw new Error(detail || `HTTP ${res.status}`); + } + return res.json(); + } + async toggleFavorite(generationId: string): Promise<{ is_favorited: boolean }> { return this.request<{ is_favorited: boolean }>(`/history/${generationId}/favorite`, { method: 'POST', diff --git a/app/src/lib/api/types.ts b/app/src/lib/api/types.ts index 4ff25285..d88f3546 100644 --- a/app/src/lib/api/types.ts +++ b/app/src/lib/api/types.ts @@ -391,6 +391,7 @@ export interface StoryItemDetail { duration: number; seed?: number; instruct?: string; + engine?: string; generation_created_at: string; versions?: GenerationVersionResponse[]; active_version_id?: string; diff --git a/backend/models.py b/backend/models.py index 6d56f9f7..379bc6e5 100644 --- a/backend/models.py +++ b/backend/models.py @@ -597,6 +597,7 @@ class StoryItemDetail(BaseModel): duration: float seed: Optional[int] instruct: Optional[str] + engine: Optional[str] = None generation_created_at: datetime # Versions available for this generation versions: Optional[List["GenerationVersionResponse"]] = None diff --git a/backend/routes/generations.py b/backend/routes/generations.py index 936337f2..215c96cb 100644 --- a/backend/routes/generations.py +++ b/backend/routes/generations.py @@ -3,22 +3,51 @@ import asyncio import logging import uuid +from pathlib import Path -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, File, HTTPException, UploadFile from fastapi.responses import StreamingResponse from sqlalchemy.orm import Session -from .. import models +from .. import config, models from ..services import history, personality, profiles, tts from ..database import Generation as DBGeneration, VoiceProfile as DBVoiceProfile, get_db from ..services.generation import run_generation from ..services.task_queue import cancel_generation as cancel_generation_job, enqueue_generation +from ..utils.audio import load_audio from ..utils.tasks import get_task_manager logger = logging.getLogger(__name__) router = APIRouter() +IMPORTED_AUDIO_PROFILE_NAME = "Imported Audio" +IMPORT_AUDIO_EXTENSIONS = {".wav", ".mp3", ".flac", ".ogg", ".m4a", ".aac", ".webm"} +IMPORT_AUDIO_MAX_BYTES = 200 * 1024 * 1024 # 200 MB + + +def _get_or_create_import_profile(db: Session) -> DBVoiceProfile: + """Singleton profile every imported audio clip points at — keeps the + Generation FK happy without making profile_id nullable across the schema.""" + row = ( + db.query(DBVoiceProfile) + .filter(DBVoiceProfile.name == IMPORTED_AUDIO_PROFILE_NAME) + .first() + ) + if row is not None: + return row + row = DBVoiceProfile( + id=str(uuid.uuid4()), + name=IMPORTED_AUDIO_PROFILE_NAME, + description="External audio imported into a story timeline.", + language="en", + voice_type="import", + ) + db.add(row) + db.commit() + db.refresh(row) + return row + def _resolve_generation_engine(data: models.GenerationRequest, profile) -> str: return data.engine or getattr(profile, "default_engine", None) or getattr(profile, "preset_engine", None) or "qwen" @@ -371,3 +400,73 @@ async def stream_speech( media_type="audio/wav", headers={"Content-Disposition": 'attachment; filename="speech.wav"'}, ) + + +@router.post("/generate/import", response_model=models.GenerationResponse) +async def import_audio( + file: UploadFile = File(...), + db: Session = Depends(get_db), +): + """Register an external audio file as a generation row. + + Designed for the story timeline so users can drop in music or other + non-TTS audio. The row points at a singleton "Imported Audio" profile + so the existing generation/story plumbing keeps working unchanged.""" + suffix = Path(file.filename or "").suffix.lower() + if suffix not in IMPORT_AUDIO_EXTENSIONS: + raise HTTPException( + status_code=400, + detail=f"Unsupported audio format '{suffix}'. Allowed: {sorted(IMPORT_AUDIO_EXTENSIONS)}", + ) + + chunks: list[bytes] = [] + total = 0 + while True: + chunk = await file.read(1024 * 1024) + if not chunk: + break + total += len(chunk) + if total > IMPORT_AUDIO_MAX_BYTES: + raise HTTPException( + status_code=413, + detail=f"File exceeds {IMPORT_AUDIO_MAX_BYTES // (1024 * 1024)} MB limit.", + ) + chunks.append(chunk) + audio_bytes = b"".join(chunks) + if not audio_bytes: + raise HTTPException(status_code=400, detail="Empty audio file.") + + generation_id = str(uuid.uuid4()) + target = config.get_generations_dir() / f"{generation_id}{suffix}" + target.write_bytes(audio_bytes) + + try: + audio, sr = load_audio(str(target)) + duration = float(len(audio) / sr) if sr else 0.0 + except Exception as decode_err: + try: + target.unlink() + except OSError: + pass + raise HTTPException( + status_code=400, + detail=f"Could not decode audio: {decode_err}", + ) from decode_err + + profile = _get_or_create_import_profile(db) + display_name = Path(file.filename or "Imported audio").stem or "Imported audio" + + return await history.create_generation( + profile_id=profile.id, + text=display_name, + language="en", + audio_path=config.to_storage_path(target), + duration=duration, + seed=None, + db=db, + generation_id=generation_id, + status="completed", + engine="import", + model_size=None, + source="import", + ) diff --git a/backend/services/stories.py b/backend/services/stories.py index cb7a46ef..6eb8ab27 100644 --- a/backend/services/stories.py +++ b/backend/services/stories.py @@ -69,6 +69,7 @@ def _build_item_detail( duration=generation.duration, seed=generation.seed, instruct=generation.instruct, + engine=generation.engine, generation_created_at=generation.created_at, versions=versions, active_version_id=active_version_id,