diff --git a/app/src/components/StoriesTab/StoriesTab.tsx b/app/src/components/StoriesTab/StoriesTab.tsx index 70001465..2e900666 100644 --- a/app/src/components/StoriesTab/StoriesTab.tsx +++ b/app/src/components/StoriesTab/StoriesTab.tsx @@ -1,25 +1,42 @@ import { FloatingGenerateBox } from '@/components/Generation/FloatingGenerateBox'; import { StoryContent } from './StoryContent'; import { StoryList } from './StoryList'; +import { StoryTrackEditor } from './StoryTrackEditor'; import { usePlayerStore } from '@/stores/playerStore'; +import { useStoryStore } from '@/stores/storyStore'; +import { useStory } from '@/lib/hooks/useStories'; export function StoriesTab() { const audioUrl = usePlayerStore((state) => state.audioUrl); + const selectedStoryId = useStoryStore((state) => state.selectedStoryId); + const { data: story } = useStory(selectedStoryId); + + const hasTrackEditor = selectedStoryId && story && story.items.length > 0; return ( -
- {/* Left Column - Story List */} -
- +
+ {/* Main content area */} +
+ {/* Left Column - Story List */} +
+ +
+ + {/* Right Column - Story Content */} +
+ +
+ + {/* Floating Generate Box */} +
- {/* Right Column - Story Content */} -
- -
- - {/* Floating Generate Box */} - + {/* Track Editor - at bottom when a story with items is selected */} + {hasTrackEditor && ( +
+ +
+ )}
); } diff --git a/app/src/components/StoriesTab/StoryTrackEditor.tsx b/app/src/components/StoriesTab/StoryTrackEditor.tsx new file mode 100644 index 00000000..34e4d982 --- /dev/null +++ b/app/src/components/StoriesTab/StoryTrackEditor.tsx @@ -0,0 +1,388 @@ +import { GripHorizontal, Minus, Plus } from 'lucide-react'; +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { Button } from '@/components/ui/button'; +import { useToast } from '@/components/ui/use-toast'; +import { useMoveStoryItem } from '@/lib/hooks/useStories'; +import { useStoryStore } from '@/stores/storyStore'; +import type { StoryItemDetail } from '@/lib/api/types'; +import { cn } from '@/lib/utils/cn'; + +interface StoryTrackEditorProps { + storyId: string; + items: StoryItemDetail[]; +} + +const TRACK_HEIGHT = 48; +const MIN_PIXELS_PER_SECOND = 10; +const MAX_PIXELS_PER_SECOND = 200; +const DEFAULT_PIXELS_PER_SECOND = 50; +const DEFAULT_TRACKS = [1, 0, -1]; // Default 3 tracks +const MIN_EDITOR_HEIGHT = 120; +const MAX_EDITOR_HEIGHT = 500; +const DEFAULT_EDITOR_HEIGHT = 200; + +export function StoryTrackEditor({ storyId, items }: StoryTrackEditorProps) { + const [pixelsPerSecond, setPixelsPerSecond] = useState(DEFAULT_PIXELS_PER_SECOND); + const [draggingItem, setDraggingItem] = useState(null); + const [dragOffset, setDragOffset] = useState({ x: 0, y: 0 }); + const [dragPosition, setDragPosition] = useState({ x: 0, y: 0 }); + const [editorHeight, setEditorHeight] = useState(DEFAULT_EDITOR_HEIGHT); + const [isResizing, setIsResizing] = useState(false); + const containerRef = useRef(null); + const tracksRef = useRef(null); + const resizeStartY = useRef(0); + const resizeStartHeight = useRef(0); + const moveItem = useMoveStoryItem(); + const { toast } = useToast(); + + // Playback state + const currentTimeMs = useStoryStore((state) => state.currentTimeMs); + const playbackStoryId = useStoryStore((state) => state.playbackStoryId); + const seek = useStoryStore((state) => state.seek); + + const isActiveStory = playbackStoryId === storyId; + + // Calculate unique tracks from items, always showing at least 3 default tracks + const tracks = useMemo(() => { + const trackSet = new Set([...DEFAULT_TRACKS, ...items.map((item) => item.track)]); + return Array.from(trackSet).sort((a, b) => b - a); // Higher tracks on top + }, [items]); + + // Calculate total duration + const totalDurationMs = useMemo(() => { + if (items.length === 0) return 10000; // Default 10 seconds + return Math.max( + ...items.map((item) => item.start_time_ms + item.duration * 1000), + 10000 + ); + }, [items]); + + // Calculate timeline width + const timelineWidth = (totalDurationMs / 1000) * pixelsPerSecond + 200; // Extra padding + + // Generate time markers + const timeMarkers = useMemo(() => { + const markers: number[] = []; + // Determine interval based on zoom level + let intervalMs = 5000; // 5 seconds + if (pixelsPerSecond > 100) intervalMs = 1000; + else if (pixelsPerSecond > 50) intervalMs = 2000; + else if (pixelsPerSecond < 20) intervalMs = 10000; + + for (let ms = 0; ms <= totalDurationMs + intervalMs; ms += intervalMs) { + markers.push(ms); + } + return markers; + }, [totalDurationMs, pixelsPerSecond]); + + const formatTime = (ms: number): string => { + const totalSeconds = Math.floor(ms / 1000); + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + return `${minutes}:${seconds.toString().padStart(2, '0')}`; + }; + + const msToPixels = useCallback( + (ms: number) => (ms / 1000) * pixelsPerSecond, + [pixelsPerSecond] + ); + + const pixelsToMs = useCallback( + (px: number) => (px / pixelsPerSecond) * 1000, + [pixelsPerSecond] + ); + + const handleZoomIn = () => { + setPixelsPerSecond((prev) => Math.min(prev * 1.5, MAX_PIXELS_PER_SECOND)); + }; + + const handleZoomOut = () => { + setPixelsPerSecond((prev) => Math.max(prev / 1.5, MIN_PIXELS_PER_SECOND)); + }; + + // Resize handlers + const handleResizeStart = useCallback((e: React.MouseEvent) => { + e.preventDefault(); + setIsResizing(true); + resizeStartY.current = e.clientY; + resizeStartHeight.current = editorHeight; + }, [editorHeight]); + + const handleResizeMove = useCallback((e: MouseEvent) => { + if (!isResizing) return; + const deltaY = resizeStartY.current - e.clientY; + const newHeight = Math.min( + MAX_EDITOR_HEIGHT, + Math.max(MIN_EDITOR_HEIGHT, resizeStartHeight.current + deltaY) + ); + setEditorHeight(newHeight); + }, [isResizing]); + + const handleResizeEnd = useCallback(() => { + setIsResizing(false); + }, []); + + // Add global mouse listeners for resizing + useEffect(() => { + if (isResizing) { + window.addEventListener('mousemove', handleResizeMove); + window.addEventListener('mouseup', handleResizeEnd); + return () => { + window.removeEventListener('mousemove', handleResizeMove); + window.removeEventListener('mouseup', handleResizeEnd); + }; + } + }, [isResizing, handleResizeMove, handleResizeEnd]); + + const handleTimelineClick = (e: React.MouseEvent) => { + if (!tracksRef.current || draggingItem) return; + const rect = tracksRef.current.getBoundingClientRect(); + const x = e.clientX - rect.left + tracksRef.current.scrollLeft; + const timeMs = Math.max(0, pixelsToMs(x)); + seek(timeMs); + }; + + const handleDragStart = ( + e: React.MouseEvent, + item: StoryItemDetail + ) => { + e.stopPropagation(); + if (!tracksRef.current) return; + + const rect = e.currentTarget.getBoundingClientRect(); + setDragOffset({ + x: e.clientX - rect.left, + y: e.clientY - rect.top, + }); + setDragPosition({ + x: rect.left - tracksRef.current.getBoundingClientRect().left + tracksRef.current.scrollLeft, + y: rect.top - tracksRef.current.getBoundingClientRect().top, + }); + setDraggingItem(item.generation_id); + }; + + const handleDragMove = useCallback( + (e: React.MouseEvent) => { + if (!draggingItem || !tracksRef.current) return; + + const rect = tracksRef.current.getBoundingClientRect(); + const x = e.clientX - rect.left + tracksRef.current.scrollLeft - dragOffset.x; + const y = e.clientY - rect.top - dragOffset.y; + + setDragPosition({ x: Math.max(0, x), y }); + }, + [draggingItem, dragOffset] + ); + + const handleDragEnd = useCallback(() => { + if (!draggingItem || !tracksRef.current) { + setDraggingItem(null); + return; + } + + const item = items.find((i) => i.generation_id === draggingItem); + if (!item) { + setDraggingItem(null); + return; + } + + // Calculate new time from x position + const newTimeMs = Math.max(0, Math.round(pixelsToMs(dragPosition.x))); + + // Calculate new track from y position + const trackIndex = Math.floor(dragPosition.y / TRACK_HEIGHT); + const clampedTrackIndex = Math.max(0, Math.min(trackIndex, tracks.length - 1)); + const newTrack = tracks[clampedTrackIndex] ?? 0; + + // Check if position changed + if (newTimeMs !== item.start_time_ms || newTrack !== item.track) { + moveItem.mutate( + { + storyId, + generationId: item.generation_id, + data: { + start_time_ms: newTimeMs, + track: newTrack, + }, + }, + { + onError: (error) => { + toast({ + title: 'Failed to move item', + description: error.message, + variant: 'destructive', + }); + }, + } + ); + } + + setDraggingItem(null); + }, [draggingItem, dragPosition, items, tracks, pixelsToMs, storyId, moveItem, toast]); + + // Get track index for rendering + const getTrackIndex = (trackNumber: number) => tracks.indexOf(trackNumber); + + // Calculate clip position and dimensions + const getClipStyle = (item: StoryItemDetail) => { + const isDragging = draggingItem === item.generation_id; + const trackIndex = getTrackIndex(item.track); + const width = msToPixels(item.duration * 1000); + const left = isDragging ? dragPosition.x : msToPixels(item.start_time_ms); + const top = isDragging ? dragPosition.y : trackIndex * TRACK_HEIGHT; + + return { + width: `${width}px`, + left: `${left}px`, + top: `${top}px`, + height: `${TRACK_HEIGHT - 4}px`, + }; + }; + + // Playhead position + const playheadLeft = msToPixels(currentTimeMs); + + // Calculate tracks area height + const tracksAreaHeight = tracks.length * TRACK_HEIGHT; + const timelineContainerHeight = editorHeight - 40; // Subtract toolbar height + + if (items.length === 0) { + return ( +
+

Add audio clips to see the track editor

+
+ ); + } + + return ( +
+ {/* Resize handle at top */} + + + {/* Toolbar */} +
+ Zoom: + + + + {Math.round(pixelsPerSecond)}px/s + +
+ + {/* Timeline container - drag handlers are intentional for drag-and-drop UX */} + {/* biome-ignore lint/a11y/noStaticElementInteractions: Container handles drag events for child clips */} +
+ {/* Time ruler */} +
+ {timeMarkers.map((ms) => ( +
+
+ + {formatTime(ms)} + +
+ ))} +
+ + {/* Tracks area */} +
+ {/* Track backgrounds */} + {tracks.map((trackNumber, index) => ( +
+ + Track {trackNumber} + +
+ ))} + + {/* Click area for seeking - z-index lower than clips */} + + ); + })} + + {/* Playhead */} + {isActiveStory && ( +
+
+
+ )} +
+
+
+ ); +} diff --git a/app/src/lib/api/client.ts b/app/src/lib/api/client.ts index 0a77054c..5505196f 100644 --- a/app/src/lib/api/client.ts +++ b/app/src/lib/api/client.ts @@ -20,6 +20,7 @@ import type { StoryItemDetail, StoryItemBatchUpdate, StoryItemReorder, + StoryItemMove, } from './types'; class ApiClient { @@ -425,6 +426,13 @@ class ApiClient { }); } + async moveStoryItem(storyId: string, generationId: string, data: StoryItemMove): Promise { + return this.request(`/stories/${storyId}/items/${generationId}/move`, { + method: 'PUT', + body: JSON.stringify(data), + }); + } + async exportStoryAudio(storyId: string): Promise { const url = `${this.getBaseUrl()}/stories/${storyId}/export-audio`; const response = await fetch(url); diff --git a/app/src/lib/api/types.ts b/app/src/lib/api/types.ts index d6172652..d8321dfe 100644 --- a/app/src/lib/api/types.ts +++ b/app/src/lib/api/types.ts @@ -143,6 +143,7 @@ export interface StoryItemDetail { story_id: string; generation_id: string; start_time_ms: number; + track: number; created_at: string; profile_id: string; profile_name: string; @@ -167,6 +168,7 @@ export interface StoryDetailResponse { export interface StoryItemCreate { generation_id: string; start_time_ms?: number; + track?: number; } export interface StoryItemUpdateTime { @@ -181,3 +183,8 @@ export interface StoryItemBatchUpdate { export interface StoryItemReorder { generation_ids: string[]; } + +export interface StoryItemMove { + start_time_ms: number; + track: number; +} diff --git a/app/src/lib/hooks/useStories.ts b/app/src/lib/hooks/useStories.ts index 8e6f2212..a100f837 100644 --- a/app/src/lib/hooks/useStories.ts +++ b/app/src/lib/hooks/useStories.ts @@ -1,6 +1,6 @@ import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; import { apiClient } from '@/lib/api/client'; -import type { StoryCreate, StoryItemCreate, StoryItemBatchUpdate, StoryItemReorder } from '@/lib/api/types'; +import type { StoryCreate, StoryItemCreate, StoryItemBatchUpdate, StoryItemReorder, StoryItemMove } from '@/lib/api/types'; import { isTauri } from '@/lib/tauri'; export function useStories() { @@ -105,6 +105,19 @@ export function useReorderStoryItems() { }); } +export function useMoveStoryItem() { + const queryClient = useQueryClient(); + + return useMutation({ + mutationFn: ({ storyId, generationId, data }: { storyId: string; generationId: string; data: StoryItemMove }) => + apiClient.moveStoryItem(storyId, generationId, data), + onSuccess: (_, variables) => { + queryClient.invalidateQueries({ queryKey: ['stories'] }); + queryClient.invalidateQueries({ queryKey: ['stories', variables.storyId] }); + }, + }); +} + export function useExportStoryAudio() { return useMutation({ mutationFn: async ({ storyId, storyName }: { storyId: string; storyName: string }) => { diff --git a/app/src/lib/hooks/useStoryPlayback.ts b/app/src/lib/hooks/useStoryPlayback.ts index f3e65f53..80c0cdf2 100644 --- a/app/src/lib/hooks/useStoryPlayback.ts +++ b/app/src/lib/hooks/useStoryPlayback.ts @@ -1,77 +1,274 @@ import { useEffect, useRef, useCallback } from 'react'; -import { useStoryStore } from '@/stores/storyStore'; import { apiClient } from '@/lib/api/client'; import type { StoryItemDetail } from '@/lib/api/types'; +import { useStoryStore } from '@/stores/storyStore'; + +interface ActiveSource { + source: AudioBufferSourceNode; + generationId: string; + startTimeMs: number; + endTimeMs: number; +} /** - * Hook for managing timecode-based story playback. - * Uses a single audio element for reliable playback. + * Hook for managing timecode-based story playback using Web Audio API. + * Supports multiple simultaneous audio sources for overlapping clips on different tracks. + * Uses AudioContext for sample-accurate timing synchronization. */ -export function useStoryPlayback(_items: StoryItemDetail[] | undefined) { +export function useStoryPlayback(items: StoryItemDetail[] | undefined) { const isPlaying = useStoryStore((state) => state.isPlaying); const playbackItems = useStoryStore((state) => state.playbackItems); - const tick = useStoryStore((state) => state.tick); + const playbackStartContextTime = useStoryStore((state) => state.playbackStartContextTime); + const playbackStartStoryTime = useStoryStore((state) => state.playbackStartStoryTime); + const currentTimeMs = useStoryStore((state) => state.currentTimeMs); + const setPlaybackTiming = useStoryStore((state) => state.setPlaybackTiming); - // Single audio element for playback - const audioRef = useRef(null); - const currentItemIdRef = useRef(null); + // AudioContext instance (created once) + const audioContextRef = useRef(null); + // Master gain for volume control + const masterGainRef = useRef(null); + // Preloaded AudioBuffers by generation_id + const audioBuffersRef = useRef>(new Map()); + // Currently playing AudioBufferSourceNodes by generation_id + const activeSourcesRef = useRef>(new Map()); + // Animation frame for syncing visual playhead const animationFrameRef = useRef(null); - const lastTimeRef = useRef(Date.now()); - // Get or create audio element - const getAudio = useCallback(() => { - if (!audioRef.current) { - audioRef.current = new Audio(); - audioRef.current.preload = 'auto'; + // Get or create AudioContext and audio graph + const getAudioContext = useCallback(() => { + if (!audioContextRef.current) { + audioContextRef.current = new AudioContext(); + console.log( + '[StoryPlayback] Created AudioContext, sample rate:', + audioContextRef.current.sampleRate, + ); + + // Create master gain node for volume control + // Set to 0.5 to prevent distortion from overlapping audio + masterGainRef.current = audioContextRef.current.createGain(); + masterGainRef.current.gain.value = 0.05; + masterGainRef.current.connect(audioContextRef.current.destination); } - return audioRef.current; + // Resume context if suspended (browser autoplay policy) + if (audioContextRef.current.state === 'suspended') { + audioContextRef.current.resume().catch(() => { + // Ignore resume errors + }); + } + return audioContextRef.current; }, []); - // Find the item that should be playing at a given time - const findActiveItem = useCallback((timeMs: number, items: StoryItemDetail[]): StoryItemDetail | null => { + // Stop a source + const stopSource = useCallback((generationId: string) => { + const activeSource = activeSourcesRef.current.get(generationId); + if (activeSource) { + try { + activeSource.source.stop(); + } catch { + // Source may have already stopped + } + activeSourcesRef.current.delete(generationId); + } + }, []); + + // Preload audio files as AudioBuffers + useEffect(() => { + if (!items || items.length === 0) { + // Clear preloaded buffers when no items + audioBuffersRef.current.clear(); + return; + } + + const currentIds = new Set(items.map((item) => item.generation_id)); + const audioContext = getAudioContext(); + + // Remove buffers for items that no longer exist + for (const [id] of audioBuffersRef.current) { + if (!currentIds.has(id)) { + audioBuffersRef.current.delete(id); + } + } + + // Preload audio for new items + const preloadPromises: Promise[] = []; for (const item of items) { - const itemStart = item.start_time_ms; - const itemEnd = item.start_time_ms + item.duration * 1000; - if (timeMs >= itemStart && timeMs < itemEnd) { - return item; + if (!audioBuffersRef.current.has(item.generation_id)) { + const audioUrl = apiClient.getAudioUrl(item.generation_id); + console.log('[StoryPlayback] Preloading audio buffer:', item.generation_id); + + const preloadPromise = fetch(audioUrl) + .then((response) => response.arrayBuffer()) + .then((arrayBuffer) => audioContext.decodeAudioData(arrayBuffer)) + .then((audioBuffer) => { + audioBuffersRef.current.set(item.generation_id, audioBuffer); + console.log( + '[StoryPlayback] Preloaded buffer:', + item.generation_id, + 'duration:', + audioBuffer.duration, + ); + }) + .catch((err) => { + console.error('[StoryPlayback] Failed to preload audio:', item.generation_id, err); + }); + + preloadPromises.push(preloadPromise); } } - return null; - }, []); - // Find the next item after a given time - const findNextItem = useCallback((timeMs: number, items: StoryItemDetail[]): StoryItemDetail | null => { - const sorted = [...items].sort((a, b) => a.start_time_ms - b.start_time_ms); - for (const item of sorted) { - if (item.start_time_ms > timeMs) { - return item; - } - } - return null; - }, []); + Promise.all(preloadPromises).then(() => { + console.log('[StoryPlayback] Preloaded', audioBuffersRef.current.size, 'audio buffers'); + }); + }, [items, getAudioContext]); - // Cleanup + // Cleanup AudioContext on unmount useEffect(() => { return () => { - if (audioRef.current) { - audioRef.current.pause(); - audioRef.current.src = ''; + // Stop all sources + for (const [generationId] of activeSourcesRef.current) { + stopSource(generationId); } + activeSourcesRef.current.clear(); + + // Clean up audio graph + if (masterGainRef.current) { + masterGainRef.current.disconnect(); + masterGainRef.current = null; + } + if (audioContextRef.current && audioContextRef.current.state !== 'closed') { + audioContextRef.current.close().catch(() => { + // Ignore errors when closing + }); + audioContextRef.current = null; + } + if (animationFrameRef.current !== null) { cancelAnimationFrame(animationFrameRef.current); } }; - }, []); + }, [stopSource]); - // Main playback effect + // Find ALL items that should be playing at a given story time + const findActiveItems = useCallback( + (storyTimeMs: number, itemList: StoryItemDetail[]): StoryItemDetail[] => { + return itemList.filter((item) => { + const itemStart = item.start_time_ms; + const itemEnd = item.start_time_ms + item.duration * 1000; + return storyTimeMs >= itemStart && storyTimeMs < itemEnd; + }); + }, + [], + ); + + // Convert AudioContext time to story time (ms) + const contextTimeToStoryTime = useCallback( + (contextTime: number): number => { + if (playbackStartContextTime === null || playbackStartStoryTime === null) { + return 0; + } + const elapsedContextTime = contextTime - playbackStartContextTime; + return playbackStartStoryTime + elapsedContextTime * 1000; + }, + [playbackStartContextTime, playbackStartStoryTime], + ); + + // Convert story time (ms) to AudioContext time + const storyTimeToContextTime = useCallback( + (storyTimeMs: number): number => { + if (playbackStartContextTime === null || playbackStartStoryTime === null) { + return 0; + } + const elapsedStoryTime = (storyTimeMs - playbackStartStoryTime) / 1000; + return playbackStartContextTime + elapsedStoryTime; + }, + [playbackStartContextTime, playbackStartStoryTime], + ); + + // Stop all sources + const stopAllSources = useCallback(() => { + console.log('[StoryPlayback] Stopping all sources'); + for (const [generationId] of activeSourcesRef.current) { + stopSource(generationId); + } + activeSourcesRef.current.clear(); + }, [stopSource]); + + // Schedule playback for all items that should be playing + const schedulePlayback = useCallback( + (storyTimeMs: number, itemList: StoryItemDetail[]) => { + const audioContext = getAudioContext(); + const currentContextTime = audioContext.currentTime; + + // Find all items that should be playing + const shouldBePlaying = findActiveItems(storyTimeMs, itemList); + const shouldBePlayingIds = new Set(shouldBePlaying.map((item) => item.generation_id)); + + // Stop sources that shouldn't be playing anymore + for (const [generationId] of activeSourcesRef.current) { + if (!shouldBePlayingIds.has(generationId)) { + stopSource(generationId); + } + } + + // Schedule new sources for items that should be playing + for (const item of shouldBePlaying) { + if (!activeSourcesRef.current.has(item.generation_id)) { + const buffer = audioBuffersRef.current.get(item.generation_id); + if (!buffer) { + console.warn('[StoryPlayback] Buffer not loaded for:', item.generation_id); + continue; + } + + // Calculate when this item should start in AudioContext time + const itemStartContextTime = storyTimeToContextTime(item.start_time_ms); + const itemEndStoryTime = item.start_time_ms + item.duration * 1000; + + // Calculate offset into the buffer (if seeking mid-way) + const offsetIntoBuffer = Math.max(0, (storyTimeMs - item.start_time_ms) / 1000); + const duration = item.duration - offsetIntoBuffer; + + // If the item should have already started, schedule it to start immediately + const startAtContextTime = Math.max(currentContextTime, itemStartContextTime); + + console.log('[StoryPlayback] Scheduling source:', { + generationId: item.generation_id, + storyTimeMs, + itemStart: item.start_time_ms, + offsetIntoBuffer, + startAtContextTime, + duration, + }); + + const source = audioContext.createBufferSource(); + source.buffer = buffer; + source.connect(masterGainRef.current || audioContext.destination); + + const activeSource: ActiveSource = { + source, + generationId: item.generation_id, + startTimeMs: item.start_time_ms, + endTimeMs: itemEndStoryTime, + }; + + activeSourcesRef.current.set(item.generation_id, activeSource); + + // Schedule playback + source.start(startAtContextTime, offsetIntoBuffer, duration); + + // Clean up when source ends + source.onended = () => { + console.log('[StoryPlayback] Source ended:', item.generation_id); + activeSourcesRef.current.delete(item.generation_id); + }; + } + } + }, + [getAudioContext, findActiveItems, storyTimeToContextTime, stopSource], + ); + + // Sync visual playhead from AudioContext time useEffect(() => { - const audio = getAudio(); - - if (!isPlaying || !playbackItems || playbackItems.length === 0) { - console.log('[StoryPlayback] Stopping playback'); - audio.pause(); - currentItemIdRef.current = null; - + if (!isPlaying || playbackStartContextTime === null || playbackStartStoryTime === null) { if (animationFrameRef.current !== null) { cancelAnimationFrame(animationFrameRef.current); animationFrameRef.current = null; @@ -79,115 +276,97 @@ export function useStoryPlayback(_items: StoryItemDetail[] | undefined) { return; } - const items = playbackItems; // Capture for closure - console.log('[StoryPlayback] Starting playback'); + const audioContext = getAudioContext(); + const itemList = playbackItems || []; - const playItem = (item: StoryItemDetail, offsetMs: number = 0) => { - console.log('[StoryPlayback] Playing item:', item.generation_id, 'offset:', offsetMs); - currentItemIdRef.current = item.generation_id; - - const audioUrl = apiClient.getAudioUrl(item.generation_id); - audio.src = audioUrl; - - audio.onloadedmetadata = () => { - const offsetSeconds = Math.max(0, offsetMs / 1000); - audio.currentTime = offsetSeconds; - audio.play().catch(err => { - console.error('[StoryPlayback] Play failed:', err); - }); - }; + const syncPlayhead = () => { + if (!useStoryStore.getState().isPlaying) { + return; + } - audio.onerror = (e) => { - console.error('[StoryPlayback] Audio error:', e); - }; - - // When this audio ends, advance the clock and check for next item - audio.onended = () => { - console.log('[StoryPlayback] Audio ended'); - const state = useStoryStore.getState(); - if (!state.isPlaying || !state.playbackItems) return; - - // Find what's next - const nextItem = findNextItem(state.currentTimeMs, state.playbackItems); - if (nextItem) { - // Jump to next item's start - useStoryStore.setState({ currentTimeMs: nextItem.start_time_ms }); - playItem(nextItem, 0); - } else { - // No more items - console.log('[StoryPlayback] Story complete'); - useStoryStore.getState().stop(); - } - }; - }; - - // Animation frame for updating the clock - const updateClock = () => { - if (!useStoryStore.getState().isPlaying) return; - - const now = Date.now(); - const deltaMs = now - lastTimeRef.current; - lastTimeRef.current = now; - - // Update master clock - tick(deltaMs); - - const currentTime = useStoryStore.getState().currentTimeMs; + const currentContextTime = audioContext.currentTime; + const currentStoryTime = contextTimeToStoryTime(currentContextTime); const totalDuration = useStoryStore.getState().totalDurationMs; - // Check if we need to start playing a different item - const activeItem = findActiveItem(currentTime, items); - - if (activeItem && currentItemIdRef.current !== activeItem.generation_id) { - // Need to switch to a different item - const offset = currentTime - activeItem.start_time_ms; - playItem(activeItem, offset); - } else if (!activeItem && currentItemIdRef.current) { - // We're in a gap between items, pause audio - audio.pause(); - currentItemIdRef.current = null; - - // Check if there's a next item to wait for - const nextItem = findNextItem(currentTime, items); - if (!nextItem && currentTime >= totalDuration) { + // Update store with current story time + useStoryStore.setState({ currentTimeMs: Math.min(currentStoryTime, totalDuration) }); + + // Schedule any items that should be playing + schedulePlayback(currentStoryTime, itemList); + + // Check if we've reached the end + if (currentStoryTime >= totalDuration) { + // Check if all sources have ended + if (activeSourcesRef.current.size === 0) { console.log('[StoryPlayback] Reached end'); useStoryStore.getState().stop(); return; } } - // Continue loop - animationFrameRef.current = requestAnimationFrame(updateClock); + // Continue sync loop + animationFrameRef.current = requestAnimationFrame(syncPlayhead); }; - // Start with the first item - const currentTime = useStoryStore.getState().currentTimeMs; - const activeItem = findActiveItem(currentTime, items); - - if (activeItem) { - const offset = currentTime - activeItem.start_time_ms; - playItem(activeItem, offset); - } else { - // Maybe we're before all items start, find the first one - const firstItem = [...items].sort((a, b) => a.start_time_ms - b.start_time_ms)[0]; - if (firstItem && currentTime < firstItem.start_time_ms) { - // Wait for the first item - console.log('[StoryPlayback] Waiting for first item at', firstItem.start_time_ms); - } - } + // Initial sync + const currentContextTime = audioContext.currentTime; + const currentStoryTime = contextTimeToStoryTime(currentContextTime); + schedulePlayback(currentStoryTime, itemList); - // Start clock - lastTimeRef.current = Date.now(); - animationFrameRef.current = requestAnimationFrame(updateClock); + // Start sync loop + animationFrameRef.current = requestAnimationFrame(syncPlayhead); return () => { - audio.onended = null; - audio.onloadedmetadata = null; - audio.onerror = null; if (animationFrameRef.current !== null) { cancelAnimationFrame(animationFrameRef.current); animationFrameRef.current = null; } }; - }, [isPlaying, playbackItems, getAudio, findActiveItem, findNextItem, tick]); + }, [ + isPlaying, + playbackItems, + playbackStartContextTime, + playbackStartStoryTime, + getAudioContext, + contextTimeToStoryTime, + schedulePlayback, + ]); + + // Handle play/pause/seek changes - set timing anchors and schedule playback + useEffect(() => { + if (!isPlaying || !playbackItems || playbackItems.length === 0) { + console.log('[StoryPlayback] Stopping playback'); + stopAllSources(); + return; + } + + const audioContext = getAudioContext(); + const currentContextTime = audioContext.currentTime; + const currentStoryTime = currentTimeMs; + + // If timing anchors are not set (or were reset by seek), set them now + if (playbackStartContextTime === null || playbackStartStoryTime === null) { + console.log('[StoryPlayback] Setting timing anchors:', { + contextTime: currentContextTime, + storyTime: currentStoryTime, + }); + setPlaybackTiming(currentContextTime, currentStoryTime); + } + + // Stop all existing sources + stopAllSources(); + + // Schedule playback from current position + schedulePlayback(currentStoryTime, playbackItems); + }, [ + isPlaying, + playbackItems, + currentTimeMs, + playbackStartContextTime, + playbackStartStoryTime, + getAudioContext, + stopAllSources, + schedulePlayback, + setPlaybackTiming, + ]); } diff --git a/app/src/stores/storyStore.ts b/app/src/stores/storyStore.ts index be28585a..e01b2d2a 100644 --- a/app/src/stores/storyStore.ts +++ b/app/src/stores/storyStore.ts @@ -12,13 +12,16 @@ interface StoryPlaybackState { totalDurationMs: number; playbackStoryId: string | null; playbackItems: StoryItemDetail[] | null; + // Web Audio API timing (null when not playing) + playbackStartContextTime: number | null; // AudioContext.currentTime when playback started + playbackStartStoryTime: number | null; // Story time (ms) when playback started // Actions play: (storyId: string, items: StoryItemDetail[]) => void; pause: () => void; stop: () => void; seek: (timeMs: number) => void; - tick: (deltaMs: number) => void; // Called by animation frame + setPlaybackTiming: (contextTime: number, storyTime: number) => void; // Set timing anchors for Web Audio API } export const useStoryStore = create((set, get) => ({ @@ -32,6 +35,8 @@ export const useStoryStore = create((set, get) => ({ totalDurationMs: 0, playbackStoryId: null, playbackItems: null, + playbackStartContextTime: null, + playbackStartStoryTime: null, // Actions play: (storyId, items) => { @@ -72,7 +77,10 @@ export const useStoryStore = create((set, get) => ({ }, pause: () => { - set({ isPlaying: false }); + set({ + isPlaying: false, + // Keep timing anchors so we can resume from same position + }); }, stop: () => { @@ -82,32 +90,26 @@ export const useStoryStore = create((set, get) => ({ playbackStoryId: null, playbackItems: null, totalDurationMs: 0, + playbackStartContextTime: null, + playbackStartStoryTime: null, }); }, seek: (timeMs) => { const state = get(); const clampedTime = Math.max(0, Math.min(timeMs, state.totalDurationMs)); - set({ currentTimeMs: clampedTime }); + set({ + currentTimeMs: clampedTime, + // Reset timing anchors - will be set by hook when playback resumes + playbackStartContextTime: null, + playbackStartStoryTime: null, + }); }, - tick: (deltaMs) => { - const state = get(); - if (!state.isPlaying || !state.playbackItems) { - return; - } - - const newTime = state.currentTimeMs + deltaMs; - const clampedTime = Math.min(newTime, state.totalDurationMs); - - // Auto-stop when reaching the end - if (clampedTime >= state.totalDurationMs) { - set({ - currentTimeMs: state.totalDurationMs, - isPlaying: false, - }); - } else { - set({ currentTimeMs: clampedTime }); - } + setPlaybackTiming: (contextTime, storyTime) => { + set({ + playbackStartContextTime: contextTime, + playbackStartStoryTime: storyTime, + }); }, })); diff --git a/backend/database.py b/backend/database.py index 0852c040..9cb5de9e 100644 --- a/backend/database.py +++ b/backend/database.py @@ -70,6 +70,7 @@ class StoryItem(Base): story_id = Column(String, ForeignKey("stories.id"), nullable=False) generation_id = Column(String, ForeignKey("generations.id"), nullable=False) start_time_ms = Column(Integer, nullable=False, default=0) # Milliseconds from story start + track = Column(Integer, nullable=False, default=0) # Track number (0 = main track) created_at = Column(DateTime, default=datetime.utcnow) @@ -245,6 +246,16 @@ def _run_migrations(engine): conn.commit() print("Migrated story_items table to use start_time_ms (removed position column)") + + # Migration: Add track column if it doesn't exist + # Re-check columns after potential position migration + columns = {col['name'] for col in inspector.get_columns('story_items')} + if 'track' not in columns: + print("Migrating story_items: adding track column") + with engine.connect() as conn: + conn.execute(text("ALTER TABLE story_items ADD COLUMN track INTEGER NOT NULL DEFAULT 0")) + conn.commit() + print("Added track column to story_items") def get_db(): diff --git a/backend/main.py b/backend/main.py index b695234d..74a88a5c 100644 --- a/backend/main.py +++ b/backend/main.py @@ -809,6 +809,20 @@ async def reorder_story_items( return items +@app.put("/stories/{story_id}/items/{generation_id}/move", response_model=models.StoryItemDetail) +async def move_story_item( + story_id: str, + generation_id: str, + data: models.StoryItemMove, + db: Session = Depends(get_db), +): + """Move a story item (update position and/or track).""" + item = await stories.move_story_item(story_id, generation_id, data, db) + if item is None: + raise HTTPException(status_code=404, detail="Story item not found") + return item + + @app.get("/stories/{story_id}/export-audio") async def export_story_audio( story_id: str, diff --git a/backend/models.py b/backend/models.py index 235f66d6..8c8a6658 100644 --- a/backend/models.py +++ b/backend/models.py @@ -220,6 +220,7 @@ class StoryItemDetail(BaseModel): story_id: str generation_id: str start_time_ms: int + track: int = 0 created_at: datetime # Generation details profile_id: str @@ -253,6 +254,7 @@ class StoryItemCreate(BaseModel): """Request model for adding a generation to a story.""" generation_id: str start_time_ms: Optional[int] = None # If not provided, will be calculated automatically + track: Optional[int] = 0 # Track number (0 = main track) class StoryItemUpdateTime(BaseModel): @@ -269,3 +271,9 @@ class StoryItemBatchUpdate(BaseModel): class StoryItemReorder(BaseModel): """Request model for reordering story items.""" generation_ids: List[str] = Field(..., min_length=1) + + +class StoryItemMove(BaseModel): + """Request model for moving a story item (position and/or track).""" + start_time_ms: int = Field(..., ge=0) + track: int = 0 diff --git a/backend/stories.py b/backend/stories.py index 19b8bf86..407376d3 100644 --- a/backend/stories.py +++ b/backend/stories.py @@ -17,6 +17,7 @@ from .models import ( StoryItemDetail, StoryItemCreate, StoryItemBatchUpdate, + StoryItemMove, ) from .database import Story as DBStory, StoryItem as DBStoryItem, Generation as DBGeneration, VoiceProfile as DBVoiceProfile from .utils.audio import load_audio, save_audio @@ -127,6 +128,7 @@ async def get_story( story_id=item.story_id, generation_id=item.generation_id, start_time_ms=item.start_time_ms, + track=item.track, created_at=item.created_at, profile_id=generation.profile_id, profile_name=profile_name, @@ -249,6 +251,7 @@ async def add_item_to_story( story_id=existing.story_id, generation_id=existing.generation_id, start_time_ms=existing.start_time_ms, + track=existing.track, created_at=existing.created_at, profile_id=generation.profile_id, profile_name=profile.name if profile else "Unknown", @@ -288,12 +291,16 @@ async def add_item_to_story( # Add 200ms gap after the last item start_time_ms = max_end_time_ms + 200 + # Get track from data or default to 0 + track = data.track if data.track is not None else 0 + # Create item item = DBStoryItem( id=str(uuid.uuid4()), story_id=story_id, generation_id=data.generation_id, start_time_ms=start_time_ms, + track=track, created_at=datetime.utcnow(), ) @@ -313,6 +320,72 @@ async def add_item_to_story( story_id=item.story_id, generation_id=item.generation_id, start_time_ms=item.start_time_ms, + track=item.track, + created_at=item.created_at, + profile_id=generation.profile_id, + profile_name=profile.name if profile else "Unknown", + text=generation.text, + language=generation.language, + audio_path=generation.audio_path, + duration=generation.duration, + seed=generation.seed, + instruct=generation.instruct, + generation_created_at=generation.created_at, + ) + + +async def move_story_item( + story_id: str, + generation_id: str, + data: StoryItemMove, + db: Session, +) -> Optional[StoryItemDetail]: + """ + Move a story item (update position and/or track). + + Args: + story_id: Story ID + generation_id: Generation ID of the item to move + data: New position and track data + db: Database session + + Returns: + Updated item detail or None if not found + """ + # Get the item + item = db.query(DBStoryItem).filter_by( + story_id=story_id, + generation_id=generation_id + ).first() + if not item: + return None + + # Get the generation + generation = db.query(DBGeneration).filter_by(id=generation_id).first() + if not generation: + return None + + # Update position and track + item.start_time_ms = data.start_time_ms + item.track = data.track + + # Update story updated_at + story = db.query(DBStory).filter_by(id=story_id).first() + if story: + story.updated_at = datetime.utcnow() + + db.commit() + db.refresh(item) + + # Get profile name + profile = db.query(DBVoiceProfile).filter_by(id=generation.profile_id).first() + + return StoryItemDetail( + id=item.id, + story_id=item.story_id, + generation_id=item.generation_id, + start_time_ms=item.start_time_ms, + track=item.track, created_at=item.created_at, profile_id=generation.profile_id, profile_name=profile.name if profile else "Unknown", @@ -464,6 +537,7 @@ async def reorder_story_items( story_id=item.story_id, generation_id=item.generation_id, start_time_ms=item.start_time_ms, + track=item.track, created_at=item.created_at, profile_id=generation.profile_id, profile_name=profile_name, diff --git a/tauri/src-tauri/gen/Assets.car b/tauri/src-tauri/gen/Assets.car index cefe2fea..3e5a5613 100644 Binary files a/tauri/src-tauri/gen/Assets.car and b/tauri/src-tauri/gen/Assets.car differ