Enhance story item management with track editing functionality

- Introduced StoryTrackEditor component for managing story item positions and tracks.
- Updated StoriesTab to conditionally render the track editor based on selected story.
- Implemented moveStoryItem API endpoint to handle item repositioning and track changes.
- Enhanced story item data model to include track information.
- Improved audio playback management to support multiple tracks using Web Audio API.
- Added hooks for moving story items and managing playback timing.
This commit is contained in:
Jamie Pine
2026-01-28 19:35:53 -08:00
parent 3204e193fa
commit 1cf90c81dd
12 changed files with 891 additions and 170 deletions
+28 -11
View File
@@ -1,25 +1,42 @@
import { FloatingGenerateBox } from '@/components/Generation/FloatingGenerateBox';
import { StoryContent } from './StoryContent';
import { StoryList } from './StoryList';
import { StoryTrackEditor } from './StoryTrackEditor';
import { usePlayerStore } from '@/stores/playerStore';
import { useStoryStore } from '@/stores/storyStore';
import { useStory } from '@/lib/hooks/useStories';
export function StoriesTab() {
const audioUrl = usePlayerStore((state) => state.audioUrl);
const selectedStoryId = useStoryStore((state) => state.selectedStoryId);
const { data: story } = useStory(selectedStoryId);
const hasTrackEditor = selectedStoryId && story && story.items.length > 0;
return (
<div className="grid grid-cols-1 lg:grid-cols-2 gap-6 h-full min-h-0 overflow-hidden relative">
{/* Left Column - Story List */}
<div className="flex flex-col min-h-0 overflow-hidden">
<StoryList />
<div className="flex flex-col h-full min-h-0 overflow-hidden">
{/* Main content area */}
<div className="flex-1 min-h-0 grid grid-cols-1 lg:grid-cols-2 gap-6 overflow-hidden relative">
{/* Left Column - Story List */}
<div className="flex flex-col min-h-0 overflow-hidden">
<StoryList />
</div>
{/* Right Column - Story Content */}
<div className="flex flex-col min-h-0 overflow-hidden">
<StoryContent />
</div>
{/* Floating Generate Box */}
<FloatingGenerateBox isPlayerOpen={!!audioUrl || !!hasTrackEditor} showVoiceSelector />
</div>
{/* Right Column - Story Content */}
<div className="flex flex-col min-h-0 overflow-hidden">
<StoryContent />
</div>
{/* Floating Generate Box */}
<FloatingGenerateBox isPlayerOpen={!!audioUrl} showVoiceSelector />
{/* Track Editor - at bottom when a story with items is selected */}
{hasTrackEditor && (
<div className="shrink-0 mt-4 px-1">
<StoryTrackEditor storyId={story.id} items={story.items} />
</div>
)}
</div>
);
}
@@ -0,0 +1,388 @@
import { GripHorizontal, Minus, Plus } from 'lucide-react';
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { Button } from '@/components/ui/button';
import { useToast } from '@/components/ui/use-toast';
import { useMoveStoryItem } from '@/lib/hooks/useStories';
import { useStoryStore } from '@/stores/storyStore';
import type { StoryItemDetail } from '@/lib/api/types';
import { cn } from '@/lib/utils/cn';
interface StoryTrackEditorProps {
storyId: string;
items: StoryItemDetail[];
}
const TRACK_HEIGHT = 48;
const MIN_PIXELS_PER_SECOND = 10;
const MAX_PIXELS_PER_SECOND = 200;
const DEFAULT_PIXELS_PER_SECOND = 50;
const DEFAULT_TRACKS = [1, 0, -1]; // Default 3 tracks
const MIN_EDITOR_HEIGHT = 120;
const MAX_EDITOR_HEIGHT = 500;
const DEFAULT_EDITOR_HEIGHT = 200;
export function StoryTrackEditor({ storyId, items }: StoryTrackEditorProps) {
const [pixelsPerSecond, setPixelsPerSecond] = useState(DEFAULT_PIXELS_PER_SECOND);
const [draggingItem, setDraggingItem] = useState<string | null>(null);
const [dragOffset, setDragOffset] = useState({ x: 0, y: 0 });
const [dragPosition, setDragPosition] = useState({ x: 0, y: 0 });
const [editorHeight, setEditorHeight] = useState(DEFAULT_EDITOR_HEIGHT);
const [isResizing, setIsResizing] = useState(false);
const containerRef = useRef<HTMLDivElement>(null);
const tracksRef = useRef<HTMLDivElement>(null);
const resizeStartY = useRef(0);
const resizeStartHeight = useRef(0);
const moveItem = useMoveStoryItem();
const { toast } = useToast();
// Playback state
const currentTimeMs = useStoryStore((state) => state.currentTimeMs);
const playbackStoryId = useStoryStore((state) => state.playbackStoryId);
const seek = useStoryStore((state) => state.seek);
const isActiveStory = playbackStoryId === storyId;
// Calculate unique tracks from items, always showing at least 3 default tracks
const tracks = useMemo(() => {
const trackSet = new Set([...DEFAULT_TRACKS, ...items.map((item) => item.track)]);
return Array.from(trackSet).sort((a, b) => b - a); // Higher tracks on top
}, [items]);
// Calculate total duration
const totalDurationMs = useMemo(() => {
if (items.length === 0) return 10000; // Default 10 seconds
return Math.max(
...items.map((item) => item.start_time_ms + item.duration * 1000),
10000
);
}, [items]);
// Calculate timeline width
const timelineWidth = (totalDurationMs / 1000) * pixelsPerSecond + 200; // Extra padding
// Generate time markers
const timeMarkers = useMemo(() => {
const markers: number[] = [];
// Determine interval based on zoom level
let intervalMs = 5000; // 5 seconds
if (pixelsPerSecond > 100) intervalMs = 1000;
else if (pixelsPerSecond > 50) intervalMs = 2000;
else if (pixelsPerSecond < 20) intervalMs = 10000;
for (let ms = 0; ms <= totalDurationMs + intervalMs; ms += intervalMs) {
markers.push(ms);
}
return markers;
}, [totalDurationMs, pixelsPerSecond]);
const formatTime = (ms: number): string => {
const totalSeconds = Math.floor(ms / 1000);
const minutes = Math.floor(totalSeconds / 60);
const seconds = totalSeconds % 60;
return `${minutes}:${seconds.toString().padStart(2, '0')}`;
};
const msToPixels = useCallback(
(ms: number) => (ms / 1000) * pixelsPerSecond,
[pixelsPerSecond]
);
const pixelsToMs = useCallback(
(px: number) => (px / pixelsPerSecond) * 1000,
[pixelsPerSecond]
);
const handleZoomIn = () => {
setPixelsPerSecond((prev) => Math.min(prev * 1.5, MAX_PIXELS_PER_SECOND));
};
const handleZoomOut = () => {
setPixelsPerSecond((prev) => Math.max(prev / 1.5, MIN_PIXELS_PER_SECOND));
};
// Resize handlers
const handleResizeStart = useCallback((e: React.MouseEvent) => {
e.preventDefault();
setIsResizing(true);
resizeStartY.current = e.clientY;
resizeStartHeight.current = editorHeight;
}, [editorHeight]);
const handleResizeMove = useCallback((e: MouseEvent) => {
if (!isResizing) return;
const deltaY = resizeStartY.current - e.clientY;
const newHeight = Math.min(
MAX_EDITOR_HEIGHT,
Math.max(MIN_EDITOR_HEIGHT, resizeStartHeight.current + deltaY)
);
setEditorHeight(newHeight);
}, [isResizing]);
const handleResizeEnd = useCallback(() => {
setIsResizing(false);
}, []);
// Add global mouse listeners for resizing
useEffect(() => {
if (isResizing) {
window.addEventListener('mousemove', handleResizeMove);
window.addEventListener('mouseup', handleResizeEnd);
return () => {
window.removeEventListener('mousemove', handleResizeMove);
window.removeEventListener('mouseup', handleResizeEnd);
};
}
}, [isResizing, handleResizeMove, handleResizeEnd]);
const handleTimelineClick = (e: React.MouseEvent<HTMLDivElement>) => {
if (!tracksRef.current || draggingItem) return;
const rect = tracksRef.current.getBoundingClientRect();
const x = e.clientX - rect.left + tracksRef.current.scrollLeft;
const timeMs = Math.max(0, pixelsToMs(x));
seek(timeMs);
};
const handleDragStart = (
e: React.MouseEvent,
item: StoryItemDetail
) => {
e.stopPropagation();
if (!tracksRef.current) return;
const rect = e.currentTarget.getBoundingClientRect();
setDragOffset({
x: e.clientX - rect.left,
y: e.clientY - rect.top,
});
setDragPosition({
x: rect.left - tracksRef.current.getBoundingClientRect().left + tracksRef.current.scrollLeft,
y: rect.top - tracksRef.current.getBoundingClientRect().top,
});
setDraggingItem(item.generation_id);
};
const handleDragMove = useCallback(
(e: React.MouseEvent) => {
if (!draggingItem || !tracksRef.current) return;
const rect = tracksRef.current.getBoundingClientRect();
const x = e.clientX - rect.left + tracksRef.current.scrollLeft - dragOffset.x;
const y = e.clientY - rect.top - dragOffset.y;
setDragPosition({ x: Math.max(0, x), y });
},
[draggingItem, dragOffset]
);
const handleDragEnd = useCallback(() => {
if (!draggingItem || !tracksRef.current) {
setDraggingItem(null);
return;
}
const item = items.find((i) => i.generation_id === draggingItem);
if (!item) {
setDraggingItem(null);
return;
}
// Calculate new time from x position
const newTimeMs = Math.max(0, Math.round(pixelsToMs(dragPosition.x)));
// Calculate new track from y position
const trackIndex = Math.floor(dragPosition.y / TRACK_HEIGHT);
const clampedTrackIndex = Math.max(0, Math.min(trackIndex, tracks.length - 1));
const newTrack = tracks[clampedTrackIndex] ?? 0;
// Check if position changed
if (newTimeMs !== item.start_time_ms || newTrack !== item.track) {
moveItem.mutate(
{
storyId,
generationId: item.generation_id,
data: {
start_time_ms: newTimeMs,
track: newTrack,
},
},
{
onError: (error) => {
toast({
title: 'Failed to move item',
description: error.message,
variant: 'destructive',
});
},
}
);
}
setDraggingItem(null);
}, [draggingItem, dragPosition, items, tracks, pixelsToMs, storyId, moveItem, toast]);
// Get track index for rendering
const getTrackIndex = (trackNumber: number) => tracks.indexOf(trackNumber);
// Calculate clip position and dimensions
const getClipStyle = (item: StoryItemDetail) => {
const isDragging = draggingItem === item.generation_id;
const trackIndex = getTrackIndex(item.track);
const width = msToPixels(item.duration * 1000);
const left = isDragging ? dragPosition.x : msToPixels(item.start_time_ms);
const top = isDragging ? dragPosition.y : trackIndex * TRACK_HEIGHT;
return {
width: `${width}px`,
left: `${left}px`,
top: `${top}px`,
height: `${TRACK_HEIGHT - 4}px`,
};
};
// Playhead position
const playheadLeft = msToPixels(currentTimeMs);
// Calculate tracks area height
const tracksAreaHeight = tracks.length * TRACK_HEIGHT;
const timelineContainerHeight = editorHeight - 40; // Subtract toolbar height
if (items.length === 0) {
return (
<div className="h-[200px] border rounded-lg bg-card flex items-center justify-center text-muted-foreground">
<p className="text-sm">Add audio clips to see the track editor</p>
</div>
);
}
return (
<div className="border rounded-lg bg-card overflow-hidden relative" ref={containerRef}>
{/* Resize handle at top */}
<button
type="button"
className="absolute top-0 left-0 right-0 h-2 cursor-ns-resize flex items-center justify-center hover:bg-muted/50 transition-colors z-20 group"
onMouseDown={handleResizeStart}
aria-label="Resize track editor"
>
<GripHorizontal className="h-3 w-3 text-muted-foreground/50 group-hover:text-muted-foreground" />
</button>
{/* Toolbar */}
<div className="flex items-center gap-2 px-3 py-2 border-b bg-muted/30 mt-2">
<span className="text-xs text-muted-foreground">Zoom:</span>
<Button variant="ghost" size="icon" className="h-6 w-6" onClick={handleZoomOut}>
<Minus className="h-3 w-3" />
</Button>
<Button variant="ghost" size="icon" className="h-6 w-6" onClick={handleZoomIn}>
<Plus className="h-3 w-3" />
</Button>
<span className="text-xs text-muted-foreground ml-2">
{Math.round(pixelsPerSecond)}px/s
</span>
</div>
{/* Timeline container - drag handlers are intentional for drag-and-drop UX */}
{/* biome-ignore lint/a11y/noStaticElementInteractions: Container handles drag events for child clips */}
<div
ref={tracksRef}
className="overflow-auto relative"
style={{ height: `${timelineContainerHeight}px` }}
onMouseMove={draggingItem ? handleDragMove : undefined}
onMouseUp={draggingItem ? handleDragEnd : undefined}
onMouseLeave={draggingItem ? handleDragEnd : undefined}
>
{/* Time ruler */}
<div
className="h-6 border-b bg-muted/20 sticky top-0 z-10"
style={{ width: `${timelineWidth}px` }}
>
{timeMarkers.map((ms) => (
<div
key={ms}
className="absolute top-0 h-full flex flex-col justify-end"
style={{ left: `${msToPixels(ms)}px` }}
>
<div className="h-2 w-px bg-border" />
<span className="text-[10px] text-muted-foreground ml-1 select-none">
{formatTime(ms)}
</span>
</div>
))}
</div>
{/* Tracks area */}
<div
className="relative"
style={{ width: `${timelineWidth}px`, height: `${tracksAreaHeight}px` }}
>
{/* Track backgrounds */}
{tracks.map((trackNumber, index) => (
<div
key={trackNumber}
className={cn(
'absolute left-0 right-0 border-b',
index % 2 === 0 ? 'bg-background' : 'bg-muted/10'
)}
style={{
top: `${index * TRACK_HEIGHT}px`,
height: `${TRACK_HEIGHT}px`,
}}
>
<span className="absolute left-2 top-1/2 -translate-y-1/2 text-[10px] text-muted-foreground select-none">
Track {trackNumber}
</span>
</div>
))}
{/* Click area for seeking - z-index lower than clips */}
<button
type="button"
className="absolute inset-0 z-0 cursor-pointer"
onClick={handleTimelineClick}
aria-label="Seek timeline"
/>
{/* Audio clips */}
{items.map((item) => {
const isDragging = draggingItem === item.generation_id;
const style = getClipStyle(item);
return (
<button
type="button"
key={item.generation_id}
className={cn(
'absolute rounded cursor-move select-none overflow-hidden z-10',
'bg-accent/80 hover:bg-accent border border-accent-foreground/20',
'flex items-center px-2 text-left',
isDragging && 'opacity-80 shadow-lg z-20',
!isDragging && 'transition-all duration-100'
)}
style={style}
onMouseDown={(e) => handleDragStart(e, item)}
>
<div className="flex-1 min-w-0">
<p className="text-[10px] font-medium text-accent-foreground truncate">
{item.profile_name}
</p>
<p className="text-[9px] text-accent-foreground/70 truncate">
{item.text.substring(0, 30)}...
</p>
</div>
</button>
);
})}
{/* Playhead */}
{isActiveStory && (
<div
className="absolute top-0 bottom-0 w-px bg-primary z-30 pointer-events-none"
style={{ left: `${playheadLeft}px` }}
>
<div className="absolute -top-1 left-1/2 -translate-x-1/2 w-0 h-0 border-l-4 border-r-4 border-t-4 border-l-transparent border-r-transparent border-t-primary" />
</div>
)}
</div>
</div>
</div>
);
}
+8
View File
@@ -20,6 +20,7 @@ import type {
StoryItemDetail,
StoryItemBatchUpdate,
StoryItemReorder,
StoryItemMove,
} from './types';
class ApiClient {
@@ -425,6 +426,13 @@ class ApiClient {
});
}
async moveStoryItem(storyId: string, generationId: string, data: StoryItemMove): Promise<StoryItemDetail> {
return this.request<StoryItemDetail>(`/stories/${storyId}/items/${generationId}/move`, {
method: 'PUT',
body: JSON.stringify(data),
});
}
async exportStoryAudio(storyId: string): Promise<Blob> {
const url = `${this.getBaseUrl()}/stories/${storyId}/export-audio`;
const response = await fetch(url);
+7
View File
@@ -143,6 +143,7 @@ export interface StoryItemDetail {
story_id: string;
generation_id: string;
start_time_ms: number;
track: number;
created_at: string;
profile_id: string;
profile_name: string;
@@ -167,6 +168,7 @@ export interface StoryDetailResponse {
export interface StoryItemCreate {
generation_id: string;
start_time_ms?: number;
track?: number;
}
export interface StoryItemUpdateTime {
@@ -181,3 +183,8 @@ export interface StoryItemBatchUpdate {
export interface StoryItemReorder {
generation_ids: string[];
}
export interface StoryItemMove {
start_time_ms: number;
track: number;
}
+14 -1
View File
@@ -1,6 +1,6 @@
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
import { apiClient } from '@/lib/api/client';
import type { StoryCreate, StoryItemCreate, StoryItemBatchUpdate, StoryItemReorder } from '@/lib/api/types';
import type { StoryCreate, StoryItemCreate, StoryItemBatchUpdate, StoryItemReorder, StoryItemMove } from '@/lib/api/types';
import { isTauri } from '@/lib/tauri';
export function useStories() {
@@ -105,6 +105,19 @@ export function useReorderStoryItems() {
});
}
export function useMoveStoryItem() {
const queryClient = useQueryClient();
return useMutation({
mutationFn: ({ storyId, generationId, data }: { storyId: string; generationId: string; data: StoryItemMove }) =>
apiClient.moveStoryItem(storyId, generationId, data),
onSuccess: (_, variables) => {
queryClient.invalidateQueries({ queryKey: ['stories'] });
queryClient.invalidateQueries({ queryKey: ['stories', variables.storyId] });
},
});
}
export function useExportStoryAudio() {
return useMutation({
mutationFn: async ({ storyId, storyName }: { storyId: string; storyName: string }) => {
+316 -137
View File
@@ -1,77 +1,274 @@
import { useEffect, useRef, useCallback } from 'react';
import { useStoryStore } from '@/stores/storyStore';
import { apiClient } from '@/lib/api/client';
import type { StoryItemDetail } from '@/lib/api/types';
import { useStoryStore } from '@/stores/storyStore';
interface ActiveSource {
source: AudioBufferSourceNode;
generationId: string;
startTimeMs: number;
endTimeMs: number;
}
/**
* Hook for managing timecode-based story playback.
* Uses a single audio element for reliable playback.
* Hook for managing timecode-based story playback using Web Audio API.
* Supports multiple simultaneous audio sources for overlapping clips on different tracks.
* Uses AudioContext for sample-accurate timing synchronization.
*/
export function useStoryPlayback(_items: StoryItemDetail[] | undefined) {
export function useStoryPlayback(items: StoryItemDetail[] | undefined) {
const isPlaying = useStoryStore((state) => state.isPlaying);
const playbackItems = useStoryStore((state) => state.playbackItems);
const tick = useStoryStore((state) => state.tick);
const playbackStartContextTime = useStoryStore((state) => state.playbackStartContextTime);
const playbackStartStoryTime = useStoryStore((state) => state.playbackStartStoryTime);
const currentTimeMs = useStoryStore((state) => state.currentTimeMs);
const setPlaybackTiming = useStoryStore((state) => state.setPlaybackTiming);
// Single audio element for playback
const audioRef = useRef<HTMLAudioElement | null>(null);
const currentItemIdRef = useRef<string | null>(null);
// AudioContext instance (created once)
const audioContextRef = useRef<AudioContext | null>(null);
// Master gain for volume control
const masterGainRef = useRef<GainNode | null>(null);
// Preloaded AudioBuffers by generation_id
const audioBuffersRef = useRef<Map<string, AudioBuffer>>(new Map());
// Currently playing AudioBufferSourceNodes by generation_id
const activeSourcesRef = useRef<Map<string, ActiveSource>>(new Map());
// Animation frame for syncing visual playhead
const animationFrameRef = useRef<number | null>(null);
const lastTimeRef = useRef<number>(Date.now());
// Get or create audio element
const getAudio = useCallback(() => {
if (!audioRef.current) {
audioRef.current = new Audio();
audioRef.current.preload = 'auto';
// Get or create AudioContext and audio graph
const getAudioContext = useCallback(() => {
if (!audioContextRef.current) {
audioContextRef.current = new AudioContext();
console.log(
'[StoryPlayback] Created AudioContext, sample rate:',
audioContextRef.current.sampleRate,
);
// Create master gain node for volume control
// Set to 0.5 to prevent distortion from overlapping audio
masterGainRef.current = audioContextRef.current.createGain();
masterGainRef.current.gain.value = 0.05;
masterGainRef.current.connect(audioContextRef.current.destination);
}
return audioRef.current;
// Resume context if suspended (browser autoplay policy)
if (audioContextRef.current.state === 'suspended') {
audioContextRef.current.resume().catch(() => {
// Ignore resume errors
});
}
return audioContextRef.current;
}, []);
// Find the item that should be playing at a given time
const findActiveItem = useCallback((timeMs: number, items: StoryItemDetail[]): StoryItemDetail | null => {
// Stop a source
const stopSource = useCallback((generationId: string) => {
const activeSource = activeSourcesRef.current.get(generationId);
if (activeSource) {
try {
activeSource.source.stop();
} catch {
// Source may have already stopped
}
activeSourcesRef.current.delete(generationId);
}
}, []);
// Preload audio files as AudioBuffers
useEffect(() => {
if (!items || items.length === 0) {
// Clear preloaded buffers when no items
audioBuffersRef.current.clear();
return;
}
const currentIds = new Set(items.map((item) => item.generation_id));
const audioContext = getAudioContext();
// Remove buffers for items that no longer exist
for (const [id] of audioBuffersRef.current) {
if (!currentIds.has(id)) {
audioBuffersRef.current.delete(id);
}
}
// Preload audio for new items
const preloadPromises: Promise<void>[] = [];
for (const item of items) {
const itemStart = item.start_time_ms;
const itemEnd = item.start_time_ms + item.duration * 1000;
if (timeMs >= itemStart && timeMs < itemEnd) {
return item;
if (!audioBuffersRef.current.has(item.generation_id)) {
const audioUrl = apiClient.getAudioUrl(item.generation_id);
console.log('[StoryPlayback] Preloading audio buffer:', item.generation_id);
const preloadPromise = fetch(audioUrl)
.then((response) => response.arrayBuffer())
.then((arrayBuffer) => audioContext.decodeAudioData(arrayBuffer))
.then((audioBuffer) => {
audioBuffersRef.current.set(item.generation_id, audioBuffer);
console.log(
'[StoryPlayback] Preloaded buffer:',
item.generation_id,
'duration:',
audioBuffer.duration,
);
})
.catch((err) => {
console.error('[StoryPlayback] Failed to preload audio:', item.generation_id, err);
});
preloadPromises.push(preloadPromise);
}
}
return null;
}, []);
// Find the next item after a given time
const findNextItem = useCallback((timeMs: number, items: StoryItemDetail[]): StoryItemDetail | null => {
const sorted = [...items].sort((a, b) => a.start_time_ms - b.start_time_ms);
for (const item of sorted) {
if (item.start_time_ms > timeMs) {
return item;
}
}
return null;
}, []);
Promise.all(preloadPromises).then(() => {
console.log('[StoryPlayback] Preloaded', audioBuffersRef.current.size, 'audio buffers');
});
}, [items, getAudioContext]);
// Cleanup
// Cleanup AudioContext on unmount
useEffect(() => {
return () => {
if (audioRef.current) {
audioRef.current.pause();
audioRef.current.src = '';
// Stop all sources
for (const [generationId] of activeSourcesRef.current) {
stopSource(generationId);
}
activeSourcesRef.current.clear();
// Clean up audio graph
if (masterGainRef.current) {
masterGainRef.current.disconnect();
masterGainRef.current = null;
}
if (audioContextRef.current && audioContextRef.current.state !== 'closed') {
audioContextRef.current.close().catch(() => {
// Ignore errors when closing
});
audioContextRef.current = null;
}
if (animationFrameRef.current !== null) {
cancelAnimationFrame(animationFrameRef.current);
}
};
}, []);
}, [stopSource]);
// Main playback effect
// Find ALL items that should be playing at a given story time
const findActiveItems = useCallback(
(storyTimeMs: number, itemList: StoryItemDetail[]): StoryItemDetail[] => {
return itemList.filter((item) => {
const itemStart = item.start_time_ms;
const itemEnd = item.start_time_ms + item.duration * 1000;
return storyTimeMs >= itemStart && storyTimeMs < itemEnd;
});
},
[],
);
// Convert AudioContext time to story time (ms)
const contextTimeToStoryTime = useCallback(
(contextTime: number): number => {
if (playbackStartContextTime === null || playbackStartStoryTime === null) {
return 0;
}
const elapsedContextTime = contextTime - playbackStartContextTime;
return playbackStartStoryTime + elapsedContextTime * 1000;
},
[playbackStartContextTime, playbackStartStoryTime],
);
// Convert story time (ms) to AudioContext time
const storyTimeToContextTime = useCallback(
(storyTimeMs: number): number => {
if (playbackStartContextTime === null || playbackStartStoryTime === null) {
return 0;
}
const elapsedStoryTime = (storyTimeMs - playbackStartStoryTime) / 1000;
return playbackStartContextTime + elapsedStoryTime;
},
[playbackStartContextTime, playbackStartStoryTime],
);
// Stop all sources
const stopAllSources = useCallback(() => {
console.log('[StoryPlayback] Stopping all sources');
for (const [generationId] of activeSourcesRef.current) {
stopSource(generationId);
}
activeSourcesRef.current.clear();
}, [stopSource]);
// Schedule playback for all items that should be playing
const schedulePlayback = useCallback(
(storyTimeMs: number, itemList: StoryItemDetail[]) => {
const audioContext = getAudioContext();
const currentContextTime = audioContext.currentTime;
// Find all items that should be playing
const shouldBePlaying = findActiveItems(storyTimeMs, itemList);
const shouldBePlayingIds = new Set(shouldBePlaying.map((item) => item.generation_id));
// Stop sources that shouldn't be playing anymore
for (const [generationId] of activeSourcesRef.current) {
if (!shouldBePlayingIds.has(generationId)) {
stopSource(generationId);
}
}
// Schedule new sources for items that should be playing
for (const item of shouldBePlaying) {
if (!activeSourcesRef.current.has(item.generation_id)) {
const buffer = audioBuffersRef.current.get(item.generation_id);
if (!buffer) {
console.warn('[StoryPlayback] Buffer not loaded for:', item.generation_id);
continue;
}
// Calculate when this item should start in AudioContext time
const itemStartContextTime = storyTimeToContextTime(item.start_time_ms);
const itemEndStoryTime = item.start_time_ms + item.duration * 1000;
// Calculate offset into the buffer (if seeking mid-way)
const offsetIntoBuffer = Math.max(0, (storyTimeMs - item.start_time_ms) / 1000);
const duration = item.duration - offsetIntoBuffer;
// If the item should have already started, schedule it to start immediately
const startAtContextTime = Math.max(currentContextTime, itemStartContextTime);
console.log('[StoryPlayback] Scheduling source:', {
generationId: item.generation_id,
storyTimeMs,
itemStart: item.start_time_ms,
offsetIntoBuffer,
startAtContextTime,
duration,
});
const source = audioContext.createBufferSource();
source.buffer = buffer;
source.connect(masterGainRef.current || audioContext.destination);
const activeSource: ActiveSource = {
source,
generationId: item.generation_id,
startTimeMs: item.start_time_ms,
endTimeMs: itemEndStoryTime,
};
activeSourcesRef.current.set(item.generation_id, activeSource);
// Schedule playback
source.start(startAtContextTime, offsetIntoBuffer, duration);
// Clean up when source ends
source.onended = () => {
console.log('[StoryPlayback] Source ended:', item.generation_id);
activeSourcesRef.current.delete(item.generation_id);
};
}
}
},
[getAudioContext, findActiveItems, storyTimeToContextTime, stopSource],
);
// Sync visual playhead from AudioContext time
useEffect(() => {
const audio = getAudio();
if (!isPlaying || !playbackItems || playbackItems.length === 0) {
console.log('[StoryPlayback] Stopping playback');
audio.pause();
currentItemIdRef.current = null;
if (!isPlaying || playbackStartContextTime === null || playbackStartStoryTime === null) {
if (animationFrameRef.current !== null) {
cancelAnimationFrame(animationFrameRef.current);
animationFrameRef.current = null;
@@ -79,115 +276,97 @@ export function useStoryPlayback(_items: StoryItemDetail[] | undefined) {
return;
}
const items = playbackItems; // Capture for closure
console.log('[StoryPlayback] Starting playback');
const audioContext = getAudioContext();
const itemList = playbackItems || [];
const playItem = (item: StoryItemDetail, offsetMs: number = 0) => {
console.log('[StoryPlayback] Playing item:', item.generation_id, 'offset:', offsetMs);
currentItemIdRef.current = item.generation_id;
const audioUrl = apiClient.getAudioUrl(item.generation_id);
audio.src = audioUrl;
audio.onloadedmetadata = () => {
const offsetSeconds = Math.max(0, offsetMs / 1000);
audio.currentTime = offsetSeconds;
audio.play().catch(err => {
console.error('[StoryPlayback] Play failed:', err);
});
};
const syncPlayhead = () => {
if (!useStoryStore.getState().isPlaying) {
return;
}
audio.onerror = (e) => {
console.error('[StoryPlayback] Audio error:', e);
};
// When this audio ends, advance the clock and check for next item
audio.onended = () => {
console.log('[StoryPlayback] Audio ended');
const state = useStoryStore.getState();
if (!state.isPlaying || !state.playbackItems) return;
// Find what's next
const nextItem = findNextItem(state.currentTimeMs, state.playbackItems);
if (nextItem) {
// Jump to next item's start
useStoryStore.setState({ currentTimeMs: nextItem.start_time_ms });
playItem(nextItem, 0);
} else {
// No more items
console.log('[StoryPlayback] Story complete');
useStoryStore.getState().stop();
}
};
};
// Animation frame for updating the clock
const updateClock = () => {
if (!useStoryStore.getState().isPlaying) return;
const now = Date.now();
const deltaMs = now - lastTimeRef.current;
lastTimeRef.current = now;
// Update master clock
tick(deltaMs);
const currentTime = useStoryStore.getState().currentTimeMs;
const currentContextTime = audioContext.currentTime;
const currentStoryTime = contextTimeToStoryTime(currentContextTime);
const totalDuration = useStoryStore.getState().totalDurationMs;
// Check if we need to start playing a different item
const activeItem = findActiveItem(currentTime, items);
if (activeItem && currentItemIdRef.current !== activeItem.generation_id) {
// Need to switch to a different item
const offset = currentTime - activeItem.start_time_ms;
playItem(activeItem, offset);
} else if (!activeItem && currentItemIdRef.current) {
// We're in a gap between items, pause audio
audio.pause();
currentItemIdRef.current = null;
// Check if there's a next item to wait for
const nextItem = findNextItem(currentTime, items);
if (!nextItem && currentTime >= totalDuration) {
// Update store with current story time
useStoryStore.setState({ currentTimeMs: Math.min(currentStoryTime, totalDuration) });
// Schedule any items that should be playing
schedulePlayback(currentStoryTime, itemList);
// Check if we've reached the end
if (currentStoryTime >= totalDuration) {
// Check if all sources have ended
if (activeSourcesRef.current.size === 0) {
console.log('[StoryPlayback] Reached end');
useStoryStore.getState().stop();
return;
}
}
// Continue loop
animationFrameRef.current = requestAnimationFrame(updateClock);
// Continue sync loop
animationFrameRef.current = requestAnimationFrame(syncPlayhead);
};
// Start with the first item
const currentTime = useStoryStore.getState().currentTimeMs;
const activeItem = findActiveItem(currentTime, items);
if (activeItem) {
const offset = currentTime - activeItem.start_time_ms;
playItem(activeItem, offset);
} else {
// Maybe we're before all items start, find the first one
const firstItem = [...items].sort((a, b) => a.start_time_ms - b.start_time_ms)[0];
if (firstItem && currentTime < firstItem.start_time_ms) {
// Wait for the first item
console.log('[StoryPlayback] Waiting for first item at', firstItem.start_time_ms);
}
}
// Initial sync
const currentContextTime = audioContext.currentTime;
const currentStoryTime = contextTimeToStoryTime(currentContextTime);
schedulePlayback(currentStoryTime, itemList);
// Start clock
lastTimeRef.current = Date.now();
animationFrameRef.current = requestAnimationFrame(updateClock);
// Start sync loop
animationFrameRef.current = requestAnimationFrame(syncPlayhead);
return () => {
audio.onended = null;
audio.onloadedmetadata = null;
audio.onerror = null;
if (animationFrameRef.current !== null) {
cancelAnimationFrame(animationFrameRef.current);
animationFrameRef.current = null;
}
};
}, [isPlaying, playbackItems, getAudio, findActiveItem, findNextItem, tick]);
}, [
isPlaying,
playbackItems,
playbackStartContextTime,
playbackStartStoryTime,
getAudioContext,
contextTimeToStoryTime,
schedulePlayback,
]);
// Handle play/pause/seek changes - set timing anchors and schedule playback
useEffect(() => {
if (!isPlaying || !playbackItems || playbackItems.length === 0) {
console.log('[StoryPlayback] Stopping playback');
stopAllSources();
return;
}
const audioContext = getAudioContext();
const currentContextTime = audioContext.currentTime;
const currentStoryTime = currentTimeMs;
// If timing anchors are not set (or were reset by seek), set them now
if (playbackStartContextTime === null || playbackStartStoryTime === null) {
console.log('[StoryPlayback] Setting timing anchors:', {
contextTime: currentContextTime,
storyTime: currentStoryTime,
});
setPlaybackTiming(currentContextTime, currentStoryTime);
}
// Stop all existing sources
stopAllSources();
// Schedule playback from current position
schedulePlayback(currentStoryTime, playbackItems);
}, [
isPlaying,
playbackItems,
currentTimeMs,
playbackStartContextTime,
playbackStartStoryTime,
getAudioContext,
stopAllSources,
schedulePlayback,
setPlaybackTiming,
]);
}
+23 -21
View File
@@ -12,13 +12,16 @@ interface StoryPlaybackState {
totalDurationMs: number;
playbackStoryId: string | null;
playbackItems: StoryItemDetail[] | null;
// Web Audio API timing (null when not playing)
playbackStartContextTime: number | null; // AudioContext.currentTime when playback started
playbackStartStoryTime: number | null; // Story time (ms) when playback started
// Actions
play: (storyId: string, items: StoryItemDetail[]) => void;
pause: () => void;
stop: () => void;
seek: (timeMs: number) => void;
tick: (deltaMs: number) => void; // Called by animation frame
setPlaybackTiming: (contextTime: number, storyTime: number) => void; // Set timing anchors for Web Audio API
}
export const useStoryStore = create<StoryPlaybackState>((set, get) => ({
@@ -32,6 +35,8 @@ export const useStoryStore = create<StoryPlaybackState>((set, get) => ({
totalDurationMs: 0,
playbackStoryId: null,
playbackItems: null,
playbackStartContextTime: null,
playbackStartStoryTime: null,
// Actions
play: (storyId, items) => {
@@ -72,7 +77,10 @@ export const useStoryStore = create<StoryPlaybackState>((set, get) => ({
},
pause: () => {
set({ isPlaying: false });
set({
isPlaying: false,
// Keep timing anchors so we can resume from same position
});
},
stop: () => {
@@ -82,32 +90,26 @@ export const useStoryStore = create<StoryPlaybackState>((set, get) => ({
playbackStoryId: null,
playbackItems: null,
totalDurationMs: 0,
playbackStartContextTime: null,
playbackStartStoryTime: null,
});
},
seek: (timeMs) => {
const state = get();
const clampedTime = Math.max(0, Math.min(timeMs, state.totalDurationMs));
set({ currentTimeMs: clampedTime });
set({
currentTimeMs: clampedTime,
// Reset timing anchors - will be set by hook when playback resumes
playbackStartContextTime: null,
playbackStartStoryTime: null,
});
},
tick: (deltaMs) => {
const state = get();
if (!state.isPlaying || !state.playbackItems) {
return;
}
const newTime = state.currentTimeMs + deltaMs;
const clampedTime = Math.min(newTime, state.totalDurationMs);
// Auto-stop when reaching the end
if (clampedTime >= state.totalDurationMs) {
set({
currentTimeMs: state.totalDurationMs,
isPlaying: false,
});
} else {
set({ currentTimeMs: clampedTime });
}
setPlaybackTiming: (contextTime, storyTime) => {
set({
playbackStartContextTime: contextTime,
playbackStartStoryTime: storyTime,
});
},
}));
+11
View File
@@ -70,6 +70,7 @@ class StoryItem(Base):
story_id = Column(String, ForeignKey("stories.id"), nullable=False)
generation_id = Column(String, ForeignKey("generations.id"), nullable=False)
start_time_ms = Column(Integer, nullable=False, default=0) # Milliseconds from story start
track = Column(Integer, nullable=False, default=0) # Track number (0 = main track)
created_at = Column(DateTime, default=datetime.utcnow)
@@ -245,6 +246,16 @@ def _run_migrations(engine):
conn.commit()
print("Migrated story_items table to use start_time_ms (removed position column)")
# Migration: Add track column if it doesn't exist
# Re-check columns after potential position migration
columns = {col['name'] for col in inspector.get_columns('story_items')}
if 'track' not in columns:
print("Migrating story_items: adding track column")
with engine.connect() as conn:
conn.execute(text("ALTER TABLE story_items ADD COLUMN track INTEGER NOT NULL DEFAULT 0"))
conn.commit()
print("Added track column to story_items")
def get_db():
+14
View File
@@ -809,6 +809,20 @@ async def reorder_story_items(
return items
@app.put("/stories/{story_id}/items/{generation_id}/move", response_model=models.StoryItemDetail)
async def move_story_item(
story_id: str,
generation_id: str,
data: models.StoryItemMove,
db: Session = Depends(get_db),
):
"""Move a story item (update position and/or track)."""
item = await stories.move_story_item(story_id, generation_id, data, db)
if item is None:
raise HTTPException(status_code=404, detail="Story item not found")
return item
@app.get("/stories/{story_id}/export-audio")
async def export_story_audio(
story_id: str,
+8
View File
@@ -220,6 +220,7 @@ class StoryItemDetail(BaseModel):
story_id: str
generation_id: str
start_time_ms: int
track: int = 0
created_at: datetime
# Generation details
profile_id: str
@@ -253,6 +254,7 @@ class StoryItemCreate(BaseModel):
"""Request model for adding a generation to a story."""
generation_id: str
start_time_ms: Optional[int] = None # If not provided, will be calculated automatically
track: Optional[int] = 0 # Track number (0 = main track)
class StoryItemUpdateTime(BaseModel):
@@ -269,3 +271,9 @@ class StoryItemBatchUpdate(BaseModel):
class StoryItemReorder(BaseModel):
"""Request model for reordering story items."""
generation_ids: List[str] = Field(..., min_length=1)
class StoryItemMove(BaseModel):
"""Request model for moving a story item (position and/or track)."""
start_time_ms: int = Field(..., ge=0)
track: int = 0
+74
View File
@@ -17,6 +17,7 @@ from .models import (
StoryItemDetail,
StoryItemCreate,
StoryItemBatchUpdate,
StoryItemMove,
)
from .database import Story as DBStory, StoryItem as DBStoryItem, Generation as DBGeneration, VoiceProfile as DBVoiceProfile
from .utils.audio import load_audio, save_audio
@@ -127,6 +128,7 @@ async def get_story(
story_id=item.story_id,
generation_id=item.generation_id,
start_time_ms=item.start_time_ms,
track=item.track,
created_at=item.created_at,
profile_id=generation.profile_id,
profile_name=profile_name,
@@ -249,6 +251,7 @@ async def add_item_to_story(
story_id=existing.story_id,
generation_id=existing.generation_id,
start_time_ms=existing.start_time_ms,
track=existing.track,
created_at=existing.created_at,
profile_id=generation.profile_id,
profile_name=profile.name if profile else "Unknown",
@@ -288,12 +291,16 @@ async def add_item_to_story(
# Add 200ms gap after the last item
start_time_ms = max_end_time_ms + 200
# Get track from data or default to 0
track = data.track if data.track is not None else 0
# Create item
item = DBStoryItem(
id=str(uuid.uuid4()),
story_id=story_id,
generation_id=data.generation_id,
start_time_ms=start_time_ms,
track=track,
created_at=datetime.utcnow(),
)
@@ -313,6 +320,72 @@ async def add_item_to_story(
story_id=item.story_id,
generation_id=item.generation_id,
start_time_ms=item.start_time_ms,
track=item.track,
created_at=item.created_at,
profile_id=generation.profile_id,
profile_name=profile.name if profile else "Unknown",
text=generation.text,
language=generation.language,
audio_path=generation.audio_path,
duration=generation.duration,
seed=generation.seed,
instruct=generation.instruct,
generation_created_at=generation.created_at,
)
async def move_story_item(
story_id: str,
generation_id: str,
data: StoryItemMove,
db: Session,
) -> Optional[StoryItemDetail]:
"""
Move a story item (update position and/or track).
Args:
story_id: Story ID
generation_id: Generation ID of the item to move
data: New position and track data
db: Database session
Returns:
Updated item detail or None if not found
"""
# Get the item
item = db.query(DBStoryItem).filter_by(
story_id=story_id,
generation_id=generation_id
).first()
if not item:
return None
# Get the generation
generation = db.query(DBGeneration).filter_by(id=generation_id).first()
if not generation:
return None
# Update position and track
item.start_time_ms = data.start_time_ms
item.track = data.track
# Update story updated_at
story = db.query(DBStory).filter_by(id=story_id).first()
if story:
story.updated_at = datetime.utcnow()
db.commit()
db.refresh(item)
# Get profile name
profile = db.query(DBVoiceProfile).filter_by(id=generation.profile_id).first()
return StoryItemDetail(
id=item.id,
story_id=item.story_id,
generation_id=item.generation_id,
start_time_ms=item.start_time_ms,
track=item.track,
created_at=item.created_at,
profile_id=generation.profile_id,
profile_name=profile.name if profile else "Unknown",
@@ -464,6 +537,7 @@ async def reorder_story_items(
story_id=item.story_id,
generation_id=item.generation_id,
start_time_ms=item.start_time_ms,
track=item.track,
created_at=item.created_at,
profile_id=generation.profile_id,
profile_name=profile_name,
Binary file not shown.