mirror of
https://github.com/jamiepine/voicebox.git
synced 2026-09-18 14:20:42 -07:00
Enhance story item management with track editing functionality
- Introduced StoryTrackEditor component for managing story item positions and tracks. - Updated StoriesTab to conditionally render the track editor based on selected story. - Implemented moveStoryItem API endpoint to handle item repositioning and track changes. - Enhanced story item data model to include track information. - Improved audio playback management to support multiple tracks using Web Audio API. - Added hooks for moving story items and managing playback timing.
This commit is contained in:
@@ -1,25 +1,42 @@
|
||||
import { FloatingGenerateBox } from '@/components/Generation/FloatingGenerateBox';
|
||||
import { StoryContent } from './StoryContent';
|
||||
import { StoryList } from './StoryList';
|
||||
import { StoryTrackEditor } from './StoryTrackEditor';
|
||||
import { usePlayerStore } from '@/stores/playerStore';
|
||||
import { useStoryStore } from '@/stores/storyStore';
|
||||
import { useStory } from '@/lib/hooks/useStories';
|
||||
|
||||
export function StoriesTab() {
|
||||
const audioUrl = usePlayerStore((state) => state.audioUrl);
|
||||
const selectedStoryId = useStoryStore((state) => state.selectedStoryId);
|
||||
const { data: story } = useStory(selectedStoryId);
|
||||
|
||||
const hasTrackEditor = selectedStoryId && story && story.items.length > 0;
|
||||
|
||||
return (
|
||||
<div className="grid grid-cols-1 lg:grid-cols-2 gap-6 h-full min-h-0 overflow-hidden relative">
|
||||
{/* Left Column - Story List */}
|
||||
<div className="flex flex-col min-h-0 overflow-hidden">
|
||||
<StoryList />
|
||||
<div className="flex flex-col h-full min-h-0 overflow-hidden">
|
||||
{/* Main content area */}
|
||||
<div className="flex-1 min-h-0 grid grid-cols-1 lg:grid-cols-2 gap-6 overflow-hidden relative">
|
||||
{/* Left Column - Story List */}
|
||||
<div className="flex flex-col min-h-0 overflow-hidden">
|
||||
<StoryList />
|
||||
</div>
|
||||
|
||||
{/* Right Column - Story Content */}
|
||||
<div className="flex flex-col min-h-0 overflow-hidden">
|
||||
<StoryContent />
|
||||
</div>
|
||||
|
||||
{/* Floating Generate Box */}
|
||||
<FloatingGenerateBox isPlayerOpen={!!audioUrl || !!hasTrackEditor} showVoiceSelector />
|
||||
</div>
|
||||
|
||||
{/* Right Column - Story Content */}
|
||||
<div className="flex flex-col min-h-0 overflow-hidden">
|
||||
<StoryContent />
|
||||
</div>
|
||||
|
||||
{/* Floating Generate Box */}
|
||||
<FloatingGenerateBox isPlayerOpen={!!audioUrl} showVoiceSelector />
|
||||
{/* Track Editor - at bottom when a story with items is selected */}
|
||||
{hasTrackEditor && (
|
||||
<div className="shrink-0 mt-4 px-1">
|
||||
<StoryTrackEditor storyId={story.id} items={story.items} />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,388 @@
|
||||
import { GripHorizontal, Minus, Plus } from 'lucide-react';
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { useToast } from '@/components/ui/use-toast';
|
||||
import { useMoveStoryItem } from '@/lib/hooks/useStories';
|
||||
import { useStoryStore } from '@/stores/storyStore';
|
||||
import type { StoryItemDetail } from '@/lib/api/types';
|
||||
import { cn } from '@/lib/utils/cn';
|
||||
|
||||
interface StoryTrackEditorProps {
|
||||
storyId: string;
|
||||
items: StoryItemDetail[];
|
||||
}
|
||||
|
||||
const TRACK_HEIGHT = 48;
|
||||
const MIN_PIXELS_PER_SECOND = 10;
|
||||
const MAX_PIXELS_PER_SECOND = 200;
|
||||
const DEFAULT_PIXELS_PER_SECOND = 50;
|
||||
const DEFAULT_TRACKS = [1, 0, -1]; // Default 3 tracks
|
||||
const MIN_EDITOR_HEIGHT = 120;
|
||||
const MAX_EDITOR_HEIGHT = 500;
|
||||
const DEFAULT_EDITOR_HEIGHT = 200;
|
||||
|
||||
export function StoryTrackEditor({ storyId, items }: StoryTrackEditorProps) {
|
||||
const [pixelsPerSecond, setPixelsPerSecond] = useState(DEFAULT_PIXELS_PER_SECOND);
|
||||
const [draggingItem, setDraggingItem] = useState<string | null>(null);
|
||||
const [dragOffset, setDragOffset] = useState({ x: 0, y: 0 });
|
||||
const [dragPosition, setDragPosition] = useState({ x: 0, y: 0 });
|
||||
const [editorHeight, setEditorHeight] = useState(DEFAULT_EDITOR_HEIGHT);
|
||||
const [isResizing, setIsResizing] = useState(false);
|
||||
const containerRef = useRef<HTMLDivElement>(null);
|
||||
const tracksRef = useRef<HTMLDivElement>(null);
|
||||
const resizeStartY = useRef(0);
|
||||
const resizeStartHeight = useRef(0);
|
||||
const moveItem = useMoveStoryItem();
|
||||
const { toast } = useToast();
|
||||
|
||||
// Playback state
|
||||
const currentTimeMs = useStoryStore((state) => state.currentTimeMs);
|
||||
const playbackStoryId = useStoryStore((state) => state.playbackStoryId);
|
||||
const seek = useStoryStore((state) => state.seek);
|
||||
|
||||
const isActiveStory = playbackStoryId === storyId;
|
||||
|
||||
// Calculate unique tracks from items, always showing at least 3 default tracks
|
||||
const tracks = useMemo(() => {
|
||||
const trackSet = new Set([...DEFAULT_TRACKS, ...items.map((item) => item.track)]);
|
||||
return Array.from(trackSet).sort((a, b) => b - a); // Higher tracks on top
|
||||
}, [items]);
|
||||
|
||||
// Calculate total duration
|
||||
const totalDurationMs = useMemo(() => {
|
||||
if (items.length === 0) return 10000; // Default 10 seconds
|
||||
return Math.max(
|
||||
...items.map((item) => item.start_time_ms + item.duration * 1000),
|
||||
10000
|
||||
);
|
||||
}, [items]);
|
||||
|
||||
// Calculate timeline width
|
||||
const timelineWidth = (totalDurationMs / 1000) * pixelsPerSecond + 200; // Extra padding
|
||||
|
||||
// Generate time markers
|
||||
const timeMarkers = useMemo(() => {
|
||||
const markers: number[] = [];
|
||||
// Determine interval based on zoom level
|
||||
let intervalMs = 5000; // 5 seconds
|
||||
if (pixelsPerSecond > 100) intervalMs = 1000;
|
||||
else if (pixelsPerSecond > 50) intervalMs = 2000;
|
||||
else if (pixelsPerSecond < 20) intervalMs = 10000;
|
||||
|
||||
for (let ms = 0; ms <= totalDurationMs + intervalMs; ms += intervalMs) {
|
||||
markers.push(ms);
|
||||
}
|
||||
return markers;
|
||||
}, [totalDurationMs, pixelsPerSecond]);
|
||||
|
||||
const formatTime = (ms: number): string => {
|
||||
const totalSeconds = Math.floor(ms / 1000);
|
||||
const minutes = Math.floor(totalSeconds / 60);
|
||||
const seconds = totalSeconds % 60;
|
||||
return `${minutes}:${seconds.toString().padStart(2, '0')}`;
|
||||
};
|
||||
|
||||
const msToPixels = useCallback(
|
||||
(ms: number) => (ms / 1000) * pixelsPerSecond,
|
||||
[pixelsPerSecond]
|
||||
);
|
||||
|
||||
const pixelsToMs = useCallback(
|
||||
(px: number) => (px / pixelsPerSecond) * 1000,
|
||||
[pixelsPerSecond]
|
||||
);
|
||||
|
||||
const handleZoomIn = () => {
|
||||
setPixelsPerSecond((prev) => Math.min(prev * 1.5, MAX_PIXELS_PER_SECOND));
|
||||
};
|
||||
|
||||
const handleZoomOut = () => {
|
||||
setPixelsPerSecond((prev) => Math.max(prev / 1.5, MIN_PIXELS_PER_SECOND));
|
||||
};
|
||||
|
||||
// Resize handlers
|
||||
const handleResizeStart = useCallback((e: React.MouseEvent) => {
|
||||
e.preventDefault();
|
||||
setIsResizing(true);
|
||||
resizeStartY.current = e.clientY;
|
||||
resizeStartHeight.current = editorHeight;
|
||||
}, [editorHeight]);
|
||||
|
||||
const handleResizeMove = useCallback((e: MouseEvent) => {
|
||||
if (!isResizing) return;
|
||||
const deltaY = resizeStartY.current - e.clientY;
|
||||
const newHeight = Math.min(
|
||||
MAX_EDITOR_HEIGHT,
|
||||
Math.max(MIN_EDITOR_HEIGHT, resizeStartHeight.current + deltaY)
|
||||
);
|
||||
setEditorHeight(newHeight);
|
||||
}, [isResizing]);
|
||||
|
||||
const handleResizeEnd = useCallback(() => {
|
||||
setIsResizing(false);
|
||||
}, []);
|
||||
|
||||
// Add global mouse listeners for resizing
|
||||
useEffect(() => {
|
||||
if (isResizing) {
|
||||
window.addEventListener('mousemove', handleResizeMove);
|
||||
window.addEventListener('mouseup', handleResizeEnd);
|
||||
return () => {
|
||||
window.removeEventListener('mousemove', handleResizeMove);
|
||||
window.removeEventListener('mouseup', handleResizeEnd);
|
||||
};
|
||||
}
|
||||
}, [isResizing, handleResizeMove, handleResizeEnd]);
|
||||
|
||||
const handleTimelineClick = (e: React.MouseEvent<HTMLDivElement>) => {
|
||||
if (!tracksRef.current || draggingItem) return;
|
||||
const rect = tracksRef.current.getBoundingClientRect();
|
||||
const x = e.clientX - rect.left + tracksRef.current.scrollLeft;
|
||||
const timeMs = Math.max(0, pixelsToMs(x));
|
||||
seek(timeMs);
|
||||
};
|
||||
|
||||
const handleDragStart = (
|
||||
e: React.MouseEvent,
|
||||
item: StoryItemDetail
|
||||
) => {
|
||||
e.stopPropagation();
|
||||
if (!tracksRef.current) return;
|
||||
|
||||
const rect = e.currentTarget.getBoundingClientRect();
|
||||
setDragOffset({
|
||||
x: e.clientX - rect.left,
|
||||
y: e.clientY - rect.top,
|
||||
});
|
||||
setDragPosition({
|
||||
x: rect.left - tracksRef.current.getBoundingClientRect().left + tracksRef.current.scrollLeft,
|
||||
y: rect.top - tracksRef.current.getBoundingClientRect().top,
|
||||
});
|
||||
setDraggingItem(item.generation_id);
|
||||
};
|
||||
|
||||
const handleDragMove = useCallback(
|
||||
(e: React.MouseEvent) => {
|
||||
if (!draggingItem || !tracksRef.current) return;
|
||||
|
||||
const rect = tracksRef.current.getBoundingClientRect();
|
||||
const x = e.clientX - rect.left + tracksRef.current.scrollLeft - dragOffset.x;
|
||||
const y = e.clientY - rect.top - dragOffset.y;
|
||||
|
||||
setDragPosition({ x: Math.max(0, x), y });
|
||||
},
|
||||
[draggingItem, dragOffset]
|
||||
);
|
||||
|
||||
const handleDragEnd = useCallback(() => {
|
||||
if (!draggingItem || !tracksRef.current) {
|
||||
setDraggingItem(null);
|
||||
return;
|
||||
}
|
||||
|
||||
const item = items.find((i) => i.generation_id === draggingItem);
|
||||
if (!item) {
|
||||
setDraggingItem(null);
|
||||
return;
|
||||
}
|
||||
|
||||
// Calculate new time from x position
|
||||
const newTimeMs = Math.max(0, Math.round(pixelsToMs(dragPosition.x)));
|
||||
|
||||
// Calculate new track from y position
|
||||
const trackIndex = Math.floor(dragPosition.y / TRACK_HEIGHT);
|
||||
const clampedTrackIndex = Math.max(0, Math.min(trackIndex, tracks.length - 1));
|
||||
const newTrack = tracks[clampedTrackIndex] ?? 0;
|
||||
|
||||
// Check if position changed
|
||||
if (newTimeMs !== item.start_time_ms || newTrack !== item.track) {
|
||||
moveItem.mutate(
|
||||
{
|
||||
storyId,
|
||||
generationId: item.generation_id,
|
||||
data: {
|
||||
start_time_ms: newTimeMs,
|
||||
track: newTrack,
|
||||
},
|
||||
},
|
||||
{
|
||||
onError: (error) => {
|
||||
toast({
|
||||
title: 'Failed to move item',
|
||||
description: error.message,
|
||||
variant: 'destructive',
|
||||
});
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
setDraggingItem(null);
|
||||
}, [draggingItem, dragPosition, items, tracks, pixelsToMs, storyId, moveItem, toast]);
|
||||
|
||||
// Get track index for rendering
|
||||
const getTrackIndex = (trackNumber: number) => tracks.indexOf(trackNumber);
|
||||
|
||||
// Calculate clip position and dimensions
|
||||
const getClipStyle = (item: StoryItemDetail) => {
|
||||
const isDragging = draggingItem === item.generation_id;
|
||||
const trackIndex = getTrackIndex(item.track);
|
||||
const width = msToPixels(item.duration * 1000);
|
||||
const left = isDragging ? dragPosition.x : msToPixels(item.start_time_ms);
|
||||
const top = isDragging ? dragPosition.y : trackIndex * TRACK_HEIGHT;
|
||||
|
||||
return {
|
||||
width: `${width}px`,
|
||||
left: `${left}px`,
|
||||
top: `${top}px`,
|
||||
height: `${TRACK_HEIGHT - 4}px`,
|
||||
};
|
||||
};
|
||||
|
||||
// Playhead position
|
||||
const playheadLeft = msToPixels(currentTimeMs);
|
||||
|
||||
// Calculate tracks area height
|
||||
const tracksAreaHeight = tracks.length * TRACK_HEIGHT;
|
||||
const timelineContainerHeight = editorHeight - 40; // Subtract toolbar height
|
||||
|
||||
if (items.length === 0) {
|
||||
return (
|
||||
<div className="h-[200px] border rounded-lg bg-card flex items-center justify-center text-muted-foreground">
|
||||
<p className="text-sm">Add audio clips to see the track editor</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="border rounded-lg bg-card overflow-hidden relative" ref={containerRef}>
|
||||
{/* Resize handle at top */}
|
||||
<button
|
||||
type="button"
|
||||
className="absolute top-0 left-0 right-0 h-2 cursor-ns-resize flex items-center justify-center hover:bg-muted/50 transition-colors z-20 group"
|
||||
onMouseDown={handleResizeStart}
|
||||
aria-label="Resize track editor"
|
||||
>
|
||||
<GripHorizontal className="h-3 w-3 text-muted-foreground/50 group-hover:text-muted-foreground" />
|
||||
</button>
|
||||
|
||||
{/* Toolbar */}
|
||||
<div className="flex items-center gap-2 px-3 py-2 border-b bg-muted/30 mt-2">
|
||||
<span className="text-xs text-muted-foreground">Zoom:</span>
|
||||
<Button variant="ghost" size="icon" className="h-6 w-6" onClick={handleZoomOut}>
|
||||
<Minus className="h-3 w-3" />
|
||||
</Button>
|
||||
<Button variant="ghost" size="icon" className="h-6 w-6" onClick={handleZoomIn}>
|
||||
<Plus className="h-3 w-3" />
|
||||
</Button>
|
||||
<span className="text-xs text-muted-foreground ml-2">
|
||||
{Math.round(pixelsPerSecond)}px/s
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{/* Timeline container - drag handlers are intentional for drag-and-drop UX */}
|
||||
{/* biome-ignore lint/a11y/noStaticElementInteractions: Container handles drag events for child clips */}
|
||||
<div
|
||||
ref={tracksRef}
|
||||
className="overflow-auto relative"
|
||||
style={{ height: `${timelineContainerHeight}px` }}
|
||||
onMouseMove={draggingItem ? handleDragMove : undefined}
|
||||
onMouseUp={draggingItem ? handleDragEnd : undefined}
|
||||
onMouseLeave={draggingItem ? handleDragEnd : undefined}
|
||||
>
|
||||
{/* Time ruler */}
|
||||
<div
|
||||
className="h-6 border-b bg-muted/20 sticky top-0 z-10"
|
||||
style={{ width: `${timelineWidth}px` }}
|
||||
>
|
||||
{timeMarkers.map((ms) => (
|
||||
<div
|
||||
key={ms}
|
||||
className="absolute top-0 h-full flex flex-col justify-end"
|
||||
style={{ left: `${msToPixels(ms)}px` }}
|
||||
>
|
||||
<div className="h-2 w-px bg-border" />
|
||||
<span className="text-[10px] text-muted-foreground ml-1 select-none">
|
||||
{formatTime(ms)}
|
||||
</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{/* Tracks area */}
|
||||
<div
|
||||
className="relative"
|
||||
style={{ width: `${timelineWidth}px`, height: `${tracksAreaHeight}px` }}
|
||||
>
|
||||
{/* Track backgrounds */}
|
||||
{tracks.map((trackNumber, index) => (
|
||||
<div
|
||||
key={trackNumber}
|
||||
className={cn(
|
||||
'absolute left-0 right-0 border-b',
|
||||
index % 2 === 0 ? 'bg-background' : 'bg-muted/10'
|
||||
)}
|
||||
style={{
|
||||
top: `${index * TRACK_HEIGHT}px`,
|
||||
height: `${TRACK_HEIGHT}px`,
|
||||
}}
|
||||
>
|
||||
<span className="absolute left-2 top-1/2 -translate-y-1/2 text-[10px] text-muted-foreground select-none">
|
||||
Track {trackNumber}
|
||||
</span>
|
||||
</div>
|
||||
))}
|
||||
|
||||
{/* Click area for seeking - z-index lower than clips */}
|
||||
<button
|
||||
type="button"
|
||||
className="absolute inset-0 z-0 cursor-pointer"
|
||||
onClick={handleTimelineClick}
|
||||
aria-label="Seek timeline"
|
||||
/>
|
||||
|
||||
{/* Audio clips */}
|
||||
{items.map((item) => {
|
||||
const isDragging = draggingItem === item.generation_id;
|
||||
const style = getClipStyle(item);
|
||||
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
key={item.generation_id}
|
||||
className={cn(
|
||||
'absolute rounded cursor-move select-none overflow-hidden z-10',
|
||||
'bg-accent/80 hover:bg-accent border border-accent-foreground/20',
|
||||
'flex items-center px-2 text-left',
|
||||
isDragging && 'opacity-80 shadow-lg z-20',
|
||||
!isDragging && 'transition-all duration-100'
|
||||
)}
|
||||
style={style}
|
||||
onMouseDown={(e) => handleDragStart(e, item)}
|
||||
>
|
||||
<div className="flex-1 min-w-0">
|
||||
<p className="text-[10px] font-medium text-accent-foreground truncate">
|
||||
{item.profile_name}
|
||||
</p>
|
||||
<p className="text-[9px] text-accent-foreground/70 truncate">
|
||||
{item.text.substring(0, 30)}...
|
||||
</p>
|
||||
</div>
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
|
||||
{/* Playhead */}
|
||||
{isActiveStory && (
|
||||
<div
|
||||
className="absolute top-0 bottom-0 w-px bg-primary z-30 pointer-events-none"
|
||||
style={{ left: `${playheadLeft}px` }}
|
||||
>
|
||||
<div className="absolute -top-1 left-1/2 -translate-x-1/2 w-0 h-0 border-l-4 border-r-4 border-t-4 border-l-transparent border-r-transparent border-t-primary" />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -20,6 +20,7 @@ import type {
|
||||
StoryItemDetail,
|
||||
StoryItemBatchUpdate,
|
||||
StoryItemReorder,
|
||||
StoryItemMove,
|
||||
} from './types';
|
||||
|
||||
class ApiClient {
|
||||
@@ -425,6 +426,13 @@ class ApiClient {
|
||||
});
|
||||
}
|
||||
|
||||
async moveStoryItem(storyId: string, generationId: string, data: StoryItemMove): Promise<StoryItemDetail> {
|
||||
return this.request<StoryItemDetail>(`/stories/${storyId}/items/${generationId}/move`, {
|
||||
method: 'PUT',
|
||||
body: JSON.stringify(data),
|
||||
});
|
||||
}
|
||||
|
||||
async exportStoryAudio(storyId: string): Promise<Blob> {
|
||||
const url = `${this.getBaseUrl()}/stories/${storyId}/export-audio`;
|
||||
const response = await fetch(url);
|
||||
|
||||
@@ -143,6 +143,7 @@ export interface StoryItemDetail {
|
||||
story_id: string;
|
||||
generation_id: string;
|
||||
start_time_ms: number;
|
||||
track: number;
|
||||
created_at: string;
|
||||
profile_id: string;
|
||||
profile_name: string;
|
||||
@@ -167,6 +168,7 @@ export interface StoryDetailResponse {
|
||||
export interface StoryItemCreate {
|
||||
generation_id: string;
|
||||
start_time_ms?: number;
|
||||
track?: number;
|
||||
}
|
||||
|
||||
export interface StoryItemUpdateTime {
|
||||
@@ -181,3 +183,8 @@ export interface StoryItemBatchUpdate {
|
||||
export interface StoryItemReorder {
|
||||
generation_ids: string[];
|
||||
}
|
||||
|
||||
export interface StoryItemMove {
|
||||
start_time_ms: number;
|
||||
track: number;
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
|
||||
import { apiClient } from '@/lib/api/client';
|
||||
import type { StoryCreate, StoryItemCreate, StoryItemBatchUpdate, StoryItemReorder } from '@/lib/api/types';
|
||||
import type { StoryCreate, StoryItemCreate, StoryItemBatchUpdate, StoryItemReorder, StoryItemMove } from '@/lib/api/types';
|
||||
import { isTauri } from '@/lib/tauri';
|
||||
|
||||
export function useStories() {
|
||||
@@ -105,6 +105,19 @@ export function useReorderStoryItems() {
|
||||
});
|
||||
}
|
||||
|
||||
export function useMoveStoryItem() {
|
||||
const queryClient = useQueryClient();
|
||||
|
||||
return useMutation({
|
||||
mutationFn: ({ storyId, generationId, data }: { storyId: string; generationId: string; data: StoryItemMove }) =>
|
||||
apiClient.moveStoryItem(storyId, generationId, data),
|
||||
onSuccess: (_, variables) => {
|
||||
queryClient.invalidateQueries({ queryKey: ['stories'] });
|
||||
queryClient.invalidateQueries({ queryKey: ['stories', variables.storyId] });
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function useExportStoryAudio() {
|
||||
return useMutation({
|
||||
mutationFn: async ({ storyId, storyName }: { storyId: string; storyName: string }) => {
|
||||
|
||||
@@ -1,77 +1,274 @@
|
||||
import { useEffect, useRef, useCallback } from 'react';
|
||||
import { useStoryStore } from '@/stores/storyStore';
|
||||
import { apiClient } from '@/lib/api/client';
|
||||
import type { StoryItemDetail } from '@/lib/api/types';
|
||||
import { useStoryStore } from '@/stores/storyStore';
|
||||
|
||||
interface ActiveSource {
|
||||
source: AudioBufferSourceNode;
|
||||
generationId: string;
|
||||
startTimeMs: number;
|
||||
endTimeMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook for managing timecode-based story playback.
|
||||
* Uses a single audio element for reliable playback.
|
||||
* Hook for managing timecode-based story playback using Web Audio API.
|
||||
* Supports multiple simultaneous audio sources for overlapping clips on different tracks.
|
||||
* Uses AudioContext for sample-accurate timing synchronization.
|
||||
*/
|
||||
export function useStoryPlayback(_items: StoryItemDetail[] | undefined) {
|
||||
export function useStoryPlayback(items: StoryItemDetail[] | undefined) {
|
||||
const isPlaying = useStoryStore((state) => state.isPlaying);
|
||||
const playbackItems = useStoryStore((state) => state.playbackItems);
|
||||
const tick = useStoryStore((state) => state.tick);
|
||||
const playbackStartContextTime = useStoryStore((state) => state.playbackStartContextTime);
|
||||
const playbackStartStoryTime = useStoryStore((state) => state.playbackStartStoryTime);
|
||||
const currentTimeMs = useStoryStore((state) => state.currentTimeMs);
|
||||
const setPlaybackTiming = useStoryStore((state) => state.setPlaybackTiming);
|
||||
|
||||
// Single audio element for playback
|
||||
const audioRef = useRef<HTMLAudioElement | null>(null);
|
||||
const currentItemIdRef = useRef<string | null>(null);
|
||||
// AudioContext instance (created once)
|
||||
const audioContextRef = useRef<AudioContext | null>(null);
|
||||
// Master gain for volume control
|
||||
const masterGainRef = useRef<GainNode | null>(null);
|
||||
// Preloaded AudioBuffers by generation_id
|
||||
const audioBuffersRef = useRef<Map<string, AudioBuffer>>(new Map());
|
||||
// Currently playing AudioBufferSourceNodes by generation_id
|
||||
const activeSourcesRef = useRef<Map<string, ActiveSource>>(new Map());
|
||||
// Animation frame for syncing visual playhead
|
||||
const animationFrameRef = useRef<number | null>(null);
|
||||
const lastTimeRef = useRef<number>(Date.now());
|
||||
|
||||
// Get or create audio element
|
||||
const getAudio = useCallback(() => {
|
||||
if (!audioRef.current) {
|
||||
audioRef.current = new Audio();
|
||||
audioRef.current.preload = 'auto';
|
||||
// Get or create AudioContext and audio graph
|
||||
const getAudioContext = useCallback(() => {
|
||||
if (!audioContextRef.current) {
|
||||
audioContextRef.current = new AudioContext();
|
||||
console.log(
|
||||
'[StoryPlayback] Created AudioContext, sample rate:',
|
||||
audioContextRef.current.sampleRate,
|
||||
);
|
||||
|
||||
// Create master gain node for volume control
|
||||
// Set to 0.5 to prevent distortion from overlapping audio
|
||||
masterGainRef.current = audioContextRef.current.createGain();
|
||||
masterGainRef.current.gain.value = 0.05;
|
||||
masterGainRef.current.connect(audioContextRef.current.destination);
|
||||
}
|
||||
return audioRef.current;
|
||||
// Resume context if suspended (browser autoplay policy)
|
||||
if (audioContextRef.current.state === 'suspended') {
|
||||
audioContextRef.current.resume().catch(() => {
|
||||
// Ignore resume errors
|
||||
});
|
||||
}
|
||||
return audioContextRef.current;
|
||||
}, []);
|
||||
|
||||
// Find the item that should be playing at a given time
|
||||
const findActiveItem = useCallback((timeMs: number, items: StoryItemDetail[]): StoryItemDetail | null => {
|
||||
// Stop a source
|
||||
const stopSource = useCallback((generationId: string) => {
|
||||
const activeSource = activeSourcesRef.current.get(generationId);
|
||||
if (activeSource) {
|
||||
try {
|
||||
activeSource.source.stop();
|
||||
} catch {
|
||||
// Source may have already stopped
|
||||
}
|
||||
activeSourcesRef.current.delete(generationId);
|
||||
}
|
||||
}, []);
|
||||
|
||||
// Preload audio files as AudioBuffers
|
||||
useEffect(() => {
|
||||
if (!items || items.length === 0) {
|
||||
// Clear preloaded buffers when no items
|
||||
audioBuffersRef.current.clear();
|
||||
return;
|
||||
}
|
||||
|
||||
const currentIds = new Set(items.map((item) => item.generation_id));
|
||||
const audioContext = getAudioContext();
|
||||
|
||||
// Remove buffers for items that no longer exist
|
||||
for (const [id] of audioBuffersRef.current) {
|
||||
if (!currentIds.has(id)) {
|
||||
audioBuffersRef.current.delete(id);
|
||||
}
|
||||
}
|
||||
|
||||
// Preload audio for new items
|
||||
const preloadPromises: Promise<void>[] = [];
|
||||
for (const item of items) {
|
||||
const itemStart = item.start_time_ms;
|
||||
const itemEnd = item.start_time_ms + item.duration * 1000;
|
||||
if (timeMs >= itemStart && timeMs < itemEnd) {
|
||||
return item;
|
||||
if (!audioBuffersRef.current.has(item.generation_id)) {
|
||||
const audioUrl = apiClient.getAudioUrl(item.generation_id);
|
||||
console.log('[StoryPlayback] Preloading audio buffer:', item.generation_id);
|
||||
|
||||
const preloadPromise = fetch(audioUrl)
|
||||
.then((response) => response.arrayBuffer())
|
||||
.then((arrayBuffer) => audioContext.decodeAudioData(arrayBuffer))
|
||||
.then((audioBuffer) => {
|
||||
audioBuffersRef.current.set(item.generation_id, audioBuffer);
|
||||
console.log(
|
||||
'[StoryPlayback] Preloaded buffer:',
|
||||
item.generation_id,
|
||||
'duration:',
|
||||
audioBuffer.duration,
|
||||
);
|
||||
})
|
||||
.catch((err) => {
|
||||
console.error('[StoryPlayback] Failed to preload audio:', item.generation_id, err);
|
||||
});
|
||||
|
||||
preloadPromises.push(preloadPromise);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}, []);
|
||||
|
||||
// Find the next item after a given time
|
||||
const findNextItem = useCallback((timeMs: number, items: StoryItemDetail[]): StoryItemDetail | null => {
|
||||
const sorted = [...items].sort((a, b) => a.start_time_ms - b.start_time_ms);
|
||||
for (const item of sorted) {
|
||||
if (item.start_time_ms > timeMs) {
|
||||
return item;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}, []);
|
||||
Promise.all(preloadPromises).then(() => {
|
||||
console.log('[StoryPlayback] Preloaded', audioBuffersRef.current.size, 'audio buffers');
|
||||
});
|
||||
}, [items, getAudioContext]);
|
||||
|
||||
// Cleanup
|
||||
// Cleanup AudioContext on unmount
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (audioRef.current) {
|
||||
audioRef.current.pause();
|
||||
audioRef.current.src = '';
|
||||
// Stop all sources
|
||||
for (const [generationId] of activeSourcesRef.current) {
|
||||
stopSource(generationId);
|
||||
}
|
||||
activeSourcesRef.current.clear();
|
||||
|
||||
// Clean up audio graph
|
||||
if (masterGainRef.current) {
|
||||
masterGainRef.current.disconnect();
|
||||
masterGainRef.current = null;
|
||||
}
|
||||
if (audioContextRef.current && audioContextRef.current.state !== 'closed') {
|
||||
audioContextRef.current.close().catch(() => {
|
||||
// Ignore errors when closing
|
||||
});
|
||||
audioContextRef.current = null;
|
||||
}
|
||||
|
||||
if (animationFrameRef.current !== null) {
|
||||
cancelAnimationFrame(animationFrameRef.current);
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
}, [stopSource]);
|
||||
|
||||
// Main playback effect
|
||||
// Find ALL items that should be playing at a given story time
|
||||
const findActiveItems = useCallback(
|
||||
(storyTimeMs: number, itemList: StoryItemDetail[]): StoryItemDetail[] => {
|
||||
return itemList.filter((item) => {
|
||||
const itemStart = item.start_time_ms;
|
||||
const itemEnd = item.start_time_ms + item.duration * 1000;
|
||||
return storyTimeMs >= itemStart && storyTimeMs < itemEnd;
|
||||
});
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
// Convert AudioContext time to story time (ms)
|
||||
const contextTimeToStoryTime = useCallback(
|
||||
(contextTime: number): number => {
|
||||
if (playbackStartContextTime === null || playbackStartStoryTime === null) {
|
||||
return 0;
|
||||
}
|
||||
const elapsedContextTime = contextTime - playbackStartContextTime;
|
||||
return playbackStartStoryTime + elapsedContextTime * 1000;
|
||||
},
|
||||
[playbackStartContextTime, playbackStartStoryTime],
|
||||
);
|
||||
|
||||
// Convert story time (ms) to AudioContext time
|
||||
const storyTimeToContextTime = useCallback(
|
||||
(storyTimeMs: number): number => {
|
||||
if (playbackStartContextTime === null || playbackStartStoryTime === null) {
|
||||
return 0;
|
||||
}
|
||||
const elapsedStoryTime = (storyTimeMs - playbackStartStoryTime) / 1000;
|
||||
return playbackStartContextTime + elapsedStoryTime;
|
||||
},
|
||||
[playbackStartContextTime, playbackStartStoryTime],
|
||||
);
|
||||
|
||||
// Stop all sources
|
||||
const stopAllSources = useCallback(() => {
|
||||
console.log('[StoryPlayback] Stopping all sources');
|
||||
for (const [generationId] of activeSourcesRef.current) {
|
||||
stopSource(generationId);
|
||||
}
|
||||
activeSourcesRef.current.clear();
|
||||
}, [stopSource]);
|
||||
|
||||
// Schedule playback for all items that should be playing
|
||||
const schedulePlayback = useCallback(
|
||||
(storyTimeMs: number, itemList: StoryItemDetail[]) => {
|
||||
const audioContext = getAudioContext();
|
||||
const currentContextTime = audioContext.currentTime;
|
||||
|
||||
// Find all items that should be playing
|
||||
const shouldBePlaying = findActiveItems(storyTimeMs, itemList);
|
||||
const shouldBePlayingIds = new Set(shouldBePlaying.map((item) => item.generation_id));
|
||||
|
||||
// Stop sources that shouldn't be playing anymore
|
||||
for (const [generationId] of activeSourcesRef.current) {
|
||||
if (!shouldBePlayingIds.has(generationId)) {
|
||||
stopSource(generationId);
|
||||
}
|
||||
}
|
||||
|
||||
// Schedule new sources for items that should be playing
|
||||
for (const item of shouldBePlaying) {
|
||||
if (!activeSourcesRef.current.has(item.generation_id)) {
|
||||
const buffer = audioBuffersRef.current.get(item.generation_id);
|
||||
if (!buffer) {
|
||||
console.warn('[StoryPlayback] Buffer not loaded for:', item.generation_id);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Calculate when this item should start in AudioContext time
|
||||
const itemStartContextTime = storyTimeToContextTime(item.start_time_ms);
|
||||
const itemEndStoryTime = item.start_time_ms + item.duration * 1000;
|
||||
|
||||
// Calculate offset into the buffer (if seeking mid-way)
|
||||
const offsetIntoBuffer = Math.max(0, (storyTimeMs - item.start_time_ms) / 1000);
|
||||
const duration = item.duration - offsetIntoBuffer;
|
||||
|
||||
// If the item should have already started, schedule it to start immediately
|
||||
const startAtContextTime = Math.max(currentContextTime, itemStartContextTime);
|
||||
|
||||
console.log('[StoryPlayback] Scheduling source:', {
|
||||
generationId: item.generation_id,
|
||||
storyTimeMs,
|
||||
itemStart: item.start_time_ms,
|
||||
offsetIntoBuffer,
|
||||
startAtContextTime,
|
||||
duration,
|
||||
});
|
||||
|
||||
const source = audioContext.createBufferSource();
|
||||
source.buffer = buffer;
|
||||
source.connect(masterGainRef.current || audioContext.destination);
|
||||
|
||||
const activeSource: ActiveSource = {
|
||||
source,
|
||||
generationId: item.generation_id,
|
||||
startTimeMs: item.start_time_ms,
|
||||
endTimeMs: itemEndStoryTime,
|
||||
};
|
||||
|
||||
activeSourcesRef.current.set(item.generation_id, activeSource);
|
||||
|
||||
// Schedule playback
|
||||
source.start(startAtContextTime, offsetIntoBuffer, duration);
|
||||
|
||||
// Clean up when source ends
|
||||
source.onended = () => {
|
||||
console.log('[StoryPlayback] Source ended:', item.generation_id);
|
||||
activeSourcesRef.current.delete(item.generation_id);
|
||||
};
|
||||
}
|
||||
}
|
||||
},
|
||||
[getAudioContext, findActiveItems, storyTimeToContextTime, stopSource],
|
||||
);
|
||||
|
||||
// Sync visual playhead from AudioContext time
|
||||
useEffect(() => {
|
||||
const audio = getAudio();
|
||||
|
||||
if (!isPlaying || !playbackItems || playbackItems.length === 0) {
|
||||
console.log('[StoryPlayback] Stopping playback');
|
||||
audio.pause();
|
||||
currentItemIdRef.current = null;
|
||||
|
||||
if (!isPlaying || playbackStartContextTime === null || playbackStartStoryTime === null) {
|
||||
if (animationFrameRef.current !== null) {
|
||||
cancelAnimationFrame(animationFrameRef.current);
|
||||
animationFrameRef.current = null;
|
||||
@@ -79,115 +276,97 @@ export function useStoryPlayback(_items: StoryItemDetail[] | undefined) {
|
||||
return;
|
||||
}
|
||||
|
||||
const items = playbackItems; // Capture for closure
|
||||
console.log('[StoryPlayback] Starting playback');
|
||||
const audioContext = getAudioContext();
|
||||
const itemList = playbackItems || [];
|
||||
|
||||
const playItem = (item: StoryItemDetail, offsetMs: number = 0) => {
|
||||
console.log('[StoryPlayback] Playing item:', item.generation_id, 'offset:', offsetMs);
|
||||
currentItemIdRef.current = item.generation_id;
|
||||
|
||||
const audioUrl = apiClient.getAudioUrl(item.generation_id);
|
||||
audio.src = audioUrl;
|
||||
|
||||
audio.onloadedmetadata = () => {
|
||||
const offsetSeconds = Math.max(0, offsetMs / 1000);
|
||||
audio.currentTime = offsetSeconds;
|
||||
audio.play().catch(err => {
|
||||
console.error('[StoryPlayback] Play failed:', err);
|
||||
});
|
||||
};
|
||||
const syncPlayhead = () => {
|
||||
if (!useStoryStore.getState().isPlaying) {
|
||||
return;
|
||||
}
|
||||
|
||||
audio.onerror = (e) => {
|
||||
console.error('[StoryPlayback] Audio error:', e);
|
||||
};
|
||||
|
||||
// When this audio ends, advance the clock and check for next item
|
||||
audio.onended = () => {
|
||||
console.log('[StoryPlayback] Audio ended');
|
||||
const state = useStoryStore.getState();
|
||||
if (!state.isPlaying || !state.playbackItems) return;
|
||||
|
||||
// Find what's next
|
||||
const nextItem = findNextItem(state.currentTimeMs, state.playbackItems);
|
||||
if (nextItem) {
|
||||
// Jump to next item's start
|
||||
useStoryStore.setState({ currentTimeMs: nextItem.start_time_ms });
|
||||
playItem(nextItem, 0);
|
||||
} else {
|
||||
// No more items
|
||||
console.log('[StoryPlayback] Story complete');
|
||||
useStoryStore.getState().stop();
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
// Animation frame for updating the clock
|
||||
const updateClock = () => {
|
||||
if (!useStoryStore.getState().isPlaying) return;
|
||||
|
||||
const now = Date.now();
|
||||
const deltaMs = now - lastTimeRef.current;
|
||||
lastTimeRef.current = now;
|
||||
|
||||
// Update master clock
|
||||
tick(deltaMs);
|
||||
|
||||
const currentTime = useStoryStore.getState().currentTimeMs;
|
||||
const currentContextTime = audioContext.currentTime;
|
||||
const currentStoryTime = contextTimeToStoryTime(currentContextTime);
|
||||
const totalDuration = useStoryStore.getState().totalDurationMs;
|
||||
|
||||
// Check if we need to start playing a different item
|
||||
const activeItem = findActiveItem(currentTime, items);
|
||||
|
||||
if (activeItem && currentItemIdRef.current !== activeItem.generation_id) {
|
||||
// Need to switch to a different item
|
||||
const offset = currentTime - activeItem.start_time_ms;
|
||||
playItem(activeItem, offset);
|
||||
} else if (!activeItem && currentItemIdRef.current) {
|
||||
// We're in a gap between items, pause audio
|
||||
audio.pause();
|
||||
currentItemIdRef.current = null;
|
||||
|
||||
// Check if there's a next item to wait for
|
||||
const nextItem = findNextItem(currentTime, items);
|
||||
if (!nextItem && currentTime >= totalDuration) {
|
||||
// Update store with current story time
|
||||
useStoryStore.setState({ currentTimeMs: Math.min(currentStoryTime, totalDuration) });
|
||||
|
||||
// Schedule any items that should be playing
|
||||
schedulePlayback(currentStoryTime, itemList);
|
||||
|
||||
// Check if we've reached the end
|
||||
if (currentStoryTime >= totalDuration) {
|
||||
// Check if all sources have ended
|
||||
if (activeSourcesRef.current.size === 0) {
|
||||
console.log('[StoryPlayback] Reached end');
|
||||
useStoryStore.getState().stop();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Continue loop
|
||||
animationFrameRef.current = requestAnimationFrame(updateClock);
|
||||
// Continue sync loop
|
||||
animationFrameRef.current = requestAnimationFrame(syncPlayhead);
|
||||
};
|
||||
|
||||
// Start with the first item
|
||||
const currentTime = useStoryStore.getState().currentTimeMs;
|
||||
const activeItem = findActiveItem(currentTime, items);
|
||||
|
||||
if (activeItem) {
|
||||
const offset = currentTime - activeItem.start_time_ms;
|
||||
playItem(activeItem, offset);
|
||||
} else {
|
||||
// Maybe we're before all items start, find the first one
|
||||
const firstItem = [...items].sort((a, b) => a.start_time_ms - b.start_time_ms)[0];
|
||||
if (firstItem && currentTime < firstItem.start_time_ms) {
|
||||
// Wait for the first item
|
||||
console.log('[StoryPlayback] Waiting for first item at', firstItem.start_time_ms);
|
||||
}
|
||||
}
|
||||
// Initial sync
|
||||
const currentContextTime = audioContext.currentTime;
|
||||
const currentStoryTime = contextTimeToStoryTime(currentContextTime);
|
||||
schedulePlayback(currentStoryTime, itemList);
|
||||
|
||||
// Start clock
|
||||
lastTimeRef.current = Date.now();
|
||||
animationFrameRef.current = requestAnimationFrame(updateClock);
|
||||
// Start sync loop
|
||||
animationFrameRef.current = requestAnimationFrame(syncPlayhead);
|
||||
|
||||
return () => {
|
||||
audio.onended = null;
|
||||
audio.onloadedmetadata = null;
|
||||
audio.onerror = null;
|
||||
if (animationFrameRef.current !== null) {
|
||||
cancelAnimationFrame(animationFrameRef.current);
|
||||
animationFrameRef.current = null;
|
||||
}
|
||||
};
|
||||
}, [isPlaying, playbackItems, getAudio, findActiveItem, findNextItem, tick]);
|
||||
}, [
|
||||
isPlaying,
|
||||
playbackItems,
|
||||
playbackStartContextTime,
|
||||
playbackStartStoryTime,
|
||||
getAudioContext,
|
||||
contextTimeToStoryTime,
|
||||
schedulePlayback,
|
||||
]);
|
||||
|
||||
// Handle play/pause/seek changes - set timing anchors and schedule playback
|
||||
useEffect(() => {
|
||||
if (!isPlaying || !playbackItems || playbackItems.length === 0) {
|
||||
console.log('[StoryPlayback] Stopping playback');
|
||||
stopAllSources();
|
||||
return;
|
||||
}
|
||||
|
||||
const audioContext = getAudioContext();
|
||||
const currentContextTime = audioContext.currentTime;
|
||||
const currentStoryTime = currentTimeMs;
|
||||
|
||||
// If timing anchors are not set (or were reset by seek), set them now
|
||||
if (playbackStartContextTime === null || playbackStartStoryTime === null) {
|
||||
console.log('[StoryPlayback] Setting timing anchors:', {
|
||||
contextTime: currentContextTime,
|
||||
storyTime: currentStoryTime,
|
||||
});
|
||||
setPlaybackTiming(currentContextTime, currentStoryTime);
|
||||
}
|
||||
|
||||
// Stop all existing sources
|
||||
stopAllSources();
|
||||
|
||||
// Schedule playback from current position
|
||||
schedulePlayback(currentStoryTime, playbackItems);
|
||||
}, [
|
||||
isPlaying,
|
||||
playbackItems,
|
||||
currentTimeMs,
|
||||
playbackStartContextTime,
|
||||
playbackStartStoryTime,
|
||||
getAudioContext,
|
||||
stopAllSources,
|
||||
schedulePlayback,
|
||||
setPlaybackTiming,
|
||||
]);
|
||||
}
|
||||
|
||||
@@ -12,13 +12,16 @@ interface StoryPlaybackState {
|
||||
totalDurationMs: number;
|
||||
playbackStoryId: string | null;
|
||||
playbackItems: StoryItemDetail[] | null;
|
||||
// Web Audio API timing (null when not playing)
|
||||
playbackStartContextTime: number | null; // AudioContext.currentTime when playback started
|
||||
playbackStartStoryTime: number | null; // Story time (ms) when playback started
|
||||
|
||||
// Actions
|
||||
play: (storyId: string, items: StoryItemDetail[]) => void;
|
||||
pause: () => void;
|
||||
stop: () => void;
|
||||
seek: (timeMs: number) => void;
|
||||
tick: (deltaMs: number) => void; // Called by animation frame
|
||||
setPlaybackTiming: (contextTime: number, storyTime: number) => void; // Set timing anchors for Web Audio API
|
||||
}
|
||||
|
||||
export const useStoryStore = create<StoryPlaybackState>((set, get) => ({
|
||||
@@ -32,6 +35,8 @@ export const useStoryStore = create<StoryPlaybackState>((set, get) => ({
|
||||
totalDurationMs: 0,
|
||||
playbackStoryId: null,
|
||||
playbackItems: null,
|
||||
playbackStartContextTime: null,
|
||||
playbackStartStoryTime: null,
|
||||
|
||||
// Actions
|
||||
play: (storyId, items) => {
|
||||
@@ -72,7 +77,10 @@ export const useStoryStore = create<StoryPlaybackState>((set, get) => ({
|
||||
},
|
||||
|
||||
pause: () => {
|
||||
set({ isPlaying: false });
|
||||
set({
|
||||
isPlaying: false,
|
||||
// Keep timing anchors so we can resume from same position
|
||||
});
|
||||
},
|
||||
|
||||
stop: () => {
|
||||
@@ -82,32 +90,26 @@ export const useStoryStore = create<StoryPlaybackState>((set, get) => ({
|
||||
playbackStoryId: null,
|
||||
playbackItems: null,
|
||||
totalDurationMs: 0,
|
||||
playbackStartContextTime: null,
|
||||
playbackStartStoryTime: null,
|
||||
});
|
||||
},
|
||||
|
||||
seek: (timeMs) => {
|
||||
const state = get();
|
||||
const clampedTime = Math.max(0, Math.min(timeMs, state.totalDurationMs));
|
||||
set({ currentTimeMs: clampedTime });
|
||||
set({
|
||||
currentTimeMs: clampedTime,
|
||||
// Reset timing anchors - will be set by hook when playback resumes
|
||||
playbackStartContextTime: null,
|
||||
playbackStartStoryTime: null,
|
||||
});
|
||||
},
|
||||
|
||||
tick: (deltaMs) => {
|
||||
const state = get();
|
||||
if (!state.isPlaying || !state.playbackItems) {
|
||||
return;
|
||||
}
|
||||
|
||||
const newTime = state.currentTimeMs + deltaMs;
|
||||
const clampedTime = Math.min(newTime, state.totalDurationMs);
|
||||
|
||||
// Auto-stop when reaching the end
|
||||
if (clampedTime >= state.totalDurationMs) {
|
||||
set({
|
||||
currentTimeMs: state.totalDurationMs,
|
||||
isPlaying: false,
|
||||
});
|
||||
} else {
|
||||
set({ currentTimeMs: clampedTime });
|
||||
}
|
||||
setPlaybackTiming: (contextTime, storyTime) => {
|
||||
set({
|
||||
playbackStartContextTime: contextTime,
|
||||
playbackStartStoryTime: storyTime,
|
||||
});
|
||||
},
|
||||
}));
|
||||
|
||||
@@ -70,6 +70,7 @@ class StoryItem(Base):
|
||||
story_id = Column(String, ForeignKey("stories.id"), nullable=False)
|
||||
generation_id = Column(String, ForeignKey("generations.id"), nullable=False)
|
||||
start_time_ms = Column(Integer, nullable=False, default=0) # Milliseconds from story start
|
||||
track = Column(Integer, nullable=False, default=0) # Track number (0 = main track)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
|
||||
@@ -245,6 +246,16 @@ def _run_migrations(engine):
|
||||
|
||||
conn.commit()
|
||||
print("Migrated story_items table to use start_time_ms (removed position column)")
|
||||
|
||||
# Migration: Add track column if it doesn't exist
|
||||
# Re-check columns after potential position migration
|
||||
columns = {col['name'] for col in inspector.get_columns('story_items')}
|
||||
if 'track' not in columns:
|
||||
print("Migrating story_items: adding track column")
|
||||
with engine.connect() as conn:
|
||||
conn.execute(text("ALTER TABLE story_items ADD COLUMN track INTEGER NOT NULL DEFAULT 0"))
|
||||
conn.commit()
|
||||
print("Added track column to story_items")
|
||||
|
||||
|
||||
def get_db():
|
||||
|
||||
@@ -809,6 +809,20 @@ async def reorder_story_items(
|
||||
return items
|
||||
|
||||
|
||||
@app.put("/stories/{story_id}/items/{generation_id}/move", response_model=models.StoryItemDetail)
|
||||
async def move_story_item(
|
||||
story_id: str,
|
||||
generation_id: str,
|
||||
data: models.StoryItemMove,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Move a story item (update position and/or track)."""
|
||||
item = await stories.move_story_item(story_id, generation_id, data, db)
|
||||
if item is None:
|
||||
raise HTTPException(status_code=404, detail="Story item not found")
|
||||
return item
|
||||
|
||||
|
||||
@app.get("/stories/{story_id}/export-audio")
|
||||
async def export_story_audio(
|
||||
story_id: str,
|
||||
|
||||
@@ -220,6 +220,7 @@ class StoryItemDetail(BaseModel):
|
||||
story_id: str
|
||||
generation_id: str
|
||||
start_time_ms: int
|
||||
track: int = 0
|
||||
created_at: datetime
|
||||
# Generation details
|
||||
profile_id: str
|
||||
@@ -253,6 +254,7 @@ class StoryItemCreate(BaseModel):
|
||||
"""Request model for adding a generation to a story."""
|
||||
generation_id: str
|
||||
start_time_ms: Optional[int] = None # If not provided, will be calculated automatically
|
||||
track: Optional[int] = 0 # Track number (0 = main track)
|
||||
|
||||
|
||||
class StoryItemUpdateTime(BaseModel):
|
||||
@@ -269,3 +271,9 @@ class StoryItemBatchUpdate(BaseModel):
|
||||
class StoryItemReorder(BaseModel):
|
||||
"""Request model for reordering story items."""
|
||||
generation_ids: List[str] = Field(..., min_length=1)
|
||||
|
||||
|
||||
class StoryItemMove(BaseModel):
|
||||
"""Request model for moving a story item (position and/or track)."""
|
||||
start_time_ms: int = Field(..., ge=0)
|
||||
track: int = 0
|
||||
|
||||
@@ -17,6 +17,7 @@ from .models import (
|
||||
StoryItemDetail,
|
||||
StoryItemCreate,
|
||||
StoryItemBatchUpdate,
|
||||
StoryItemMove,
|
||||
)
|
||||
from .database import Story as DBStory, StoryItem as DBStoryItem, Generation as DBGeneration, VoiceProfile as DBVoiceProfile
|
||||
from .utils.audio import load_audio, save_audio
|
||||
@@ -127,6 +128,7 @@ async def get_story(
|
||||
story_id=item.story_id,
|
||||
generation_id=item.generation_id,
|
||||
start_time_ms=item.start_time_ms,
|
||||
track=item.track,
|
||||
created_at=item.created_at,
|
||||
profile_id=generation.profile_id,
|
||||
profile_name=profile_name,
|
||||
@@ -249,6 +251,7 @@ async def add_item_to_story(
|
||||
story_id=existing.story_id,
|
||||
generation_id=existing.generation_id,
|
||||
start_time_ms=existing.start_time_ms,
|
||||
track=existing.track,
|
||||
created_at=existing.created_at,
|
||||
profile_id=generation.profile_id,
|
||||
profile_name=profile.name if profile else "Unknown",
|
||||
@@ -288,12 +291,16 @@ async def add_item_to_story(
|
||||
# Add 200ms gap after the last item
|
||||
start_time_ms = max_end_time_ms + 200
|
||||
|
||||
# Get track from data or default to 0
|
||||
track = data.track if data.track is not None else 0
|
||||
|
||||
# Create item
|
||||
item = DBStoryItem(
|
||||
id=str(uuid.uuid4()),
|
||||
story_id=story_id,
|
||||
generation_id=data.generation_id,
|
||||
start_time_ms=start_time_ms,
|
||||
track=track,
|
||||
created_at=datetime.utcnow(),
|
||||
)
|
||||
|
||||
@@ -313,6 +320,72 @@ async def add_item_to_story(
|
||||
story_id=item.story_id,
|
||||
generation_id=item.generation_id,
|
||||
start_time_ms=item.start_time_ms,
|
||||
track=item.track,
|
||||
created_at=item.created_at,
|
||||
profile_id=generation.profile_id,
|
||||
profile_name=profile.name if profile else "Unknown",
|
||||
text=generation.text,
|
||||
language=generation.language,
|
||||
audio_path=generation.audio_path,
|
||||
duration=generation.duration,
|
||||
seed=generation.seed,
|
||||
instruct=generation.instruct,
|
||||
generation_created_at=generation.created_at,
|
||||
)
|
||||
|
||||
|
||||
async def move_story_item(
|
||||
story_id: str,
|
||||
generation_id: str,
|
||||
data: StoryItemMove,
|
||||
db: Session,
|
||||
) -> Optional[StoryItemDetail]:
|
||||
"""
|
||||
Move a story item (update position and/or track).
|
||||
|
||||
Args:
|
||||
story_id: Story ID
|
||||
generation_id: Generation ID of the item to move
|
||||
data: New position and track data
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
Updated item detail or None if not found
|
||||
"""
|
||||
# Get the item
|
||||
item = db.query(DBStoryItem).filter_by(
|
||||
story_id=story_id,
|
||||
generation_id=generation_id
|
||||
).first()
|
||||
if not item:
|
||||
return None
|
||||
|
||||
# Get the generation
|
||||
generation = db.query(DBGeneration).filter_by(id=generation_id).first()
|
||||
if not generation:
|
||||
return None
|
||||
|
||||
# Update position and track
|
||||
item.start_time_ms = data.start_time_ms
|
||||
item.track = data.track
|
||||
|
||||
# Update story updated_at
|
||||
story = db.query(DBStory).filter_by(id=story_id).first()
|
||||
if story:
|
||||
story.updated_at = datetime.utcnow()
|
||||
|
||||
db.commit()
|
||||
db.refresh(item)
|
||||
|
||||
# Get profile name
|
||||
profile = db.query(DBVoiceProfile).filter_by(id=generation.profile_id).first()
|
||||
|
||||
return StoryItemDetail(
|
||||
id=item.id,
|
||||
story_id=item.story_id,
|
||||
generation_id=item.generation_id,
|
||||
start_time_ms=item.start_time_ms,
|
||||
track=item.track,
|
||||
created_at=item.created_at,
|
||||
profile_id=generation.profile_id,
|
||||
profile_name=profile.name if profile else "Unknown",
|
||||
@@ -464,6 +537,7 @@ async def reorder_story_items(
|
||||
story_id=item.story_id,
|
||||
generation_id=item.generation_id,
|
||||
start_time_ms=item.start_time_ms,
|
||||
track=item.track,
|
||||
created_at=item.created_at,
|
||||
profile_id=generation.profile_id,
|
||||
profile_name=profile_name,
|
||||
|
||||
Binary file not shown.
Reference in New Issue
Block a user