diff --git a/public/channelSync.js b/public/channelSync.js index c8b0b76..3d9db5b 100644 --- a/public/channelSync.js +++ b/public/channelSync.js @@ -280,6 +280,13 @@ M.showToast(data.message, data.toastType || "info"); return; } + // Handle fetch progress from ytdlp + if (data.type && data.type.startsWith("fetch_")) { + if (M.handleFetchProgress) { + M.handleFetchProgress(data); + } + return; + } // Normal channel state update M.handleUpdate(data); }; diff --git a/public/init.js b/public/init.js index 4631d8d..9e2c83a 100644 --- a/public/init.js +++ b/public/init.js @@ -59,8 +59,18 @@ initPanelTabs(); }); + // Update UI based on server status + function updateFeatureVisibility() { + const fetchBtn = M.$("#btn-fetch-url"); + if (fetchBtn) { + const ytdlpEnabled = M.serverStatus?.ytdlp?.enabled && M.serverStatus?.ytdlp?.available; + fetchBtn.style.display = ytdlpEnabled ? "" : "none"; + } + } + // Initialize the application Promise.all([initStorage(), loadServerStatus()]).then(async () => { + updateFeatureVisibility(); await M.loadLibrary(); await M.loadCurrentUser(); if (M.currentUser) { diff --git a/public/upload.js b/public/upload.js index 7dd5765..7e3c33d 100644 --- a/public/upload.js +++ b/public/upload.js @@ -89,9 +89,7 @@ return; } closeFetchDialog(); - - // Create a task for this fetch - const task = createTask(`Fetching: ${url.substring(0, 50)}...`); + M.showToast("Checking URL..."); try { const res = await fetch("/api/fetch", { @@ -102,15 +100,39 @@ if (res.ok) { const data = await res.json(); - task.setComplete(); - M.showToast(data.message || "Fetch started"); + + if (data.type === "playlist") { + // Ask user to confirm playlist download + const confirmed = confirm(`Download playlist "${data.title}" with ${data.count} items?\n\nItems will be downloaded slowly (one every ~3 minutes) to avoid overloading the server.`); + + if (confirmed) { + // Confirm playlist download + const confirmRes = await fetch("/api/fetch/confirm", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ items: data.items }) + }); + + if (confirmRes.ok) { + const confirmData = await confirmRes.json(); + M.showToast(confirmData.message); + // Tasks will be created by WebSocket progress messages + } else { + const err = await confirmRes.json().catch(() => ({})); + M.showToast(err.error || "Failed to queue playlist", "error"); + } + } + } else if (data.type === "single") { + M.showToast(`Queued: ${data.title}`); + // Task will be created by WebSocket progress messages + } else { + M.showToast(data.message || "Fetch started"); + } } else { const err = await res.json().catch(() => ({})); - task.setError(err.error || "Failed"); M.showToast(err.error || "Fetch failed", "error"); } } catch (e) { - task.setError("Error"); M.showToast("Fetch failed", "error"); } }; @@ -175,12 +197,36 @@ }; // Task management + const fetchTasks = new Map(); // Map + function updateTasksEmpty() { const hasTasks = tasksList.children.length > 0; tasksEmpty.classList.toggle("hidden", hasTasks); } - function createTask(filename) { + // Handle WebSocket fetch progress messages + M.handleFetchProgress = function(data) { + let task = fetchTasks.get(data.id); + + // Create task if we don't have one for this id + if (!task && data.status !== "complete" && data.status !== "error") { + task = createTask(data.title || "Downloading...", data.id); + } + + if (!task) return; + + if (data.status === "downloading" || data.status === "queued") { + task.setProgress(data.progress || 0); + } else if (data.status === "complete") { + task.setComplete(); + fetchTasks.delete(data.id); + } else if (data.status === "error") { + task.setError(data.error || "Failed"); + fetchTasks.delete(data.id); + } + }; + + function createTask(filename, fetchId) { const task = document.createElement("div"); task.className = "task-item"; task.innerHTML = ` @@ -197,7 +243,7 @@ const tasksTab = document.querySelector('.panel-tab[data-tab="tasks"]'); if (tasksTab) tasksTab.click(); - return { + const taskHandle = { setProgress(percent) { task.querySelector(".task-progress").textContent = `${Math.round(percent)}%`; task.querySelector(".task-bar").style.width = `${percent}%`; @@ -222,6 +268,13 @@ }, 5000); } }; + + // Store fetch tasks for WebSocket updates + if (fetchId) { + fetchTasks.set(fetchId, taskHandle); + } + + return taskHandle; } function uploadFile(file) { diff --git a/server.ts b/server.ts index cb48da6..0c24ce1 100644 --- a/server.ts +++ b/server.ts @@ -33,16 +33,37 @@ import { getClientInfo, } from "./auth"; import { Library } from "./library"; -import { createFetchRequest, getUserRequests, startDownload } from "./ytdlp"; +import { + initYtdlp, + getStatus as getYtdlpStatus, + isAvailable as isYtdlpAvailable, + checkUrl, + addToFastQueue, + addToSlowQueue, + getUserQueues, + setProgressCallback, + type QueueItem +} from "./ytdlp"; // Load config +interface YtdlpConfig { + enabled: boolean; + command: string; + ffmpegCommand: string; + updateCommand: string | null; + fastQueueConcurrent: number; + slowQueueInterval: number; + allowPlaylists: boolean; + autoUpdate: boolean; + updateCheckInterval: number; +} + interface Config { port: number; musicDir: string; allowGuests: boolean; - defaultPermissions: { - channel?: string; - }; + defaultPermissions: string[]; + ytdlp?: YtdlpConfig; } const CONFIG_PATH = join(import.meta.dir, "config.json"); @@ -51,7 +72,18 @@ const DEFAULT_CONFIG: Config = { port: 3001, musicDir: "./music", allowGuests: true, - defaultPermissions: ["listen", "control"] + defaultPermissions: ["listen", "control"], + ytdlp: { + enabled: false, + command: "yt-dlp", + ffmpegCommand: "ffmpeg", + updateCommand: "yt-dlp -U", + fastQueueConcurrent: 2, + slowQueueInterval: 180, + allowPlaylists: true, + autoUpdate: true, + updateCheckInterval: 86400 + } }; // Create default config if missing @@ -70,6 +102,18 @@ const PUBLIC_DIR = join(import.meta.dir, "public"); console.log(`Config loaded: port=${config.port}, musicDir=${MUSIC_DIR}, allowGuests=${config.allowGuests}`); +// Initialize yt-dlp if configured +const ytdlpConfig = config.ytdlp || DEFAULT_CONFIG.ytdlp!; +const ytdlpStatus = await initYtdlp({ + enabled: ytdlpConfig.enabled, + command: ytdlpConfig.command, + ffmpegCommand: ytdlpConfig.ffmpegCommand, + musicDir: MUSIC_DIR, + fastQueueConcurrent: ytdlpConfig.fastQueueConcurrent, + slowQueueInterval: ytdlpConfig.slowQueueInterval, + allowPlaylists: ytdlpConfig.allowPlaylists +}); + // Initialize library const library = new Library(MUSIC_DIR); @@ -245,6 +289,30 @@ function broadcastToAll(message: object) { console.log(`[Broadcast] Sent to ${clientCount} clients`); } +// Send message to specific user's connections +function sendToUser(userId: number, message: object) { + const connections = userConnections.get(userId); + if (connections) { + const data = JSON.stringify(message); + for (const ws of connections) { + ws.send(data); + } + } +} + +// Set up ytdlp progress callback +setProgressCallback((item) => { + sendToUser(item.userId, { + type: `fetch_${item.status === "downloading" ? "progress" : item.status}`, + id: item.id, + title: item.title, + status: item.status, + progress: item.progress, + queueType: item.queueType, + error: item.error + }); +}); + // Broadcast channel list to all clients function broadcastChannelList() { const list = [...channels.values()].map(c => c.getListInfo()); @@ -375,6 +443,7 @@ serve({ allowSignups: true, channelCount: channels.size, defaultPermissions: config.defaultPermissions, + ytdlp: getYtdlpStatus() }); } @@ -582,38 +651,96 @@ serve({ return Response.json({ error: "Guests cannot fetch from URLs" }, { status: 403 }); } + // Check if feature is enabled + if (!ytdlpConfig.enabled) { + return Response.json({ error: "Feature disabled" }, { status: 403 }); + } + if (!isYtdlpAvailable()) { + return Response.json({ error: "yt-dlp not available" }, { status: 503 }); + } + try { const { url } = await req.json(); if (!url || typeof url !== "string") { return Response.json({ error: "URL is required" }, { status: 400 }); } - // Create fetch request - const request = createFetchRequest(url, user.id); - console.log(`[Fetch] ${user.username} requested: ${url} (id=${request.id})`); + console.log(`[Fetch] ${user.username} checking URL: ${url}`); - // Start download in background (don't await) - startDownload(request, config.musicDir); + // Check URL to detect playlist vs single video + const info = await checkUrl(url); - return Response.json({ - message: "Fetch started", - requestId: request.id + if (info.type === "playlist") { + if (!ytdlpConfig.allowPlaylists) { + return Response.json({ error: "Playlist downloads are disabled" }, { status: 403 }); + } + // Return playlist info for confirmation + return Response.json(info, { headers }); + } else { + // Single video - add to fast queue immediately + const item = addToFastQueue(info.url, info.title, user.id); + console.log(`[Fetch] ${user.username} queued: ${info.title} (id=${item.id})`); + return Response.json({ + type: "single", + id: item.id, + title: item.title, + queueType: "fast" + }, { headers }); + } + } catch (e: any) { + console.error("[Fetch] Error:", e); + return Response.json({ error: e.message || "Invalid request" }, { status: 400 }); + } + } + + // API: confirm playlist download + if (path === "/api/fetch/confirm" && req.method === "POST") { + const { user, headers } = getOrCreateUser(req, server); + if (!user) { + return Response.json({ error: "Authentication required" }, { status: 401 }); + } + if (user.is_guest) { + return Response.json({ error: "Guests cannot fetch from URLs" }, { status: 403 }); + } + if (!ytdlpConfig.enabled || !isYtdlpAvailable()) { + return Response.json({ error: "Feature not available" }, { status: 503 }); + } + + try { + const { items } = await req.json(); + if (!Array.isArray(items) || items.length === 0) { + return Response.json({ error: "Items required" }, { status: 400 }); + } + + const queueItems = addToSlowQueue(items, user.id); + const estimatedMinutes = Math.ceil(queueItems.length * ytdlpConfig.slowQueueInterval / 60); + const hours = Math.floor(estimatedMinutes / 60); + const mins = estimatedMinutes % 60; + const estimatedTime = hours > 0 ? `${hours}h ${mins}m` : `${mins}m`; + + console.log(`[Fetch] ${user.username} confirmed playlist: ${queueItems.length} items`); + + return Response.json({ + message: `Added ${queueItems.length} items to queue`, + queueType: "slow", + estimatedTime, + items: queueItems.map(i => ({ id: i.id, title: i.title })) }, { headers }); } catch (e) { - console.error("[Fetch] Error:", e); + console.error("[Fetch] Confirm error:", e); return Response.json({ error: "Invalid request" }, { status: 400 }); } } - // API: get fetch requests for current user + // API: get fetch queue status for current user if (path === "/api/fetch" && req.method === "GET") { const { user, headers } = getOrCreateUser(req, server); if (!user) { return Response.json({ error: "Authentication required" }, { status: 401 }); } - const requests = getUserRequests(user.id); - return Response.json({ requests }, { headers }); + const queues = getUserQueues(user.id); + return Response.json(queues, { headers }); } // Auth: signup diff --git a/ytdlp.ts b/ytdlp.ts index 3d16e38..ec52c36 100644 --- a/ytdlp.ts +++ b/ytdlp.ts @@ -1,79 +1,416 @@ // MusicRoom - yt-dlp integration module // Handles fetching audio from URLs via yt-dlp -export interface FetchRequest { +import { spawn } from "child_process"; +import { join } from "path"; + +export interface QueueItem { id: string; url: string; - status: "pending" | "downloading" | "processing" | "complete" | "error"; + title: string; + userId: number; + status: "queued" | "downloading" | "complete" | "error"; progress: number; + queueType: "fast" | "slow"; error?: string; filename?: string; createdAt: number; completedAt?: number; - userId: number; } -// In-memory store of active fetch requests -const fetchRequests = new Map(); +export interface YtdlpStatus { + available: boolean; + enabled: boolean; + version: string | null; + ffmpeg: boolean; +} -// Generate unique request ID +export interface PlaylistInfo { + type: "playlist"; + title: string; + count: number; + items: { id: string; url: string; title: string }[]; + requiresConfirmation: true; +} + +export interface SingleVideoInfo { + type: "single"; + id: string; + title: string; + url: string; +} + +type ProgressCallback = (item: QueueItem) => void; + +// Configuration +let ytdlpCommand = "yt-dlp"; +let ffmpegCommand = "ffmpeg"; +let musicDir = "./music"; +let fastQueueConcurrent = 2; +let slowQueueInterval = 180; +let allowPlaylists = true; + +// Status +let ytdlpAvailable = false; +let ytdlpVersion: string | null = null; +let ffmpegAvailable = false; +let featureEnabled = false; + +// Queues +const fastQueue: QueueItem[] = []; +const slowQueue: QueueItem[] = []; +let activeDownloads = 0; +let slowQueueTimer: ReturnType | null = null; +let lastSlowDownload = 0; + +// Callbacks +let onProgress: ProgressCallback | null = null; + +// Generate unique ID function generateId(): string { return Math.random().toString(36).substring(2, 10); } -// Get all requests for a user -export function getUserRequests(userId: number): FetchRequest[] { - return [...fetchRequests.values()].filter(r => r.userId === userId); +// Initialize ytdlp module +export async function initYtdlp(config: { + enabled: boolean; + command: string; + ffmpegCommand: string; + musicDir: string; + fastQueueConcurrent: number; + slowQueueInterval: number; + allowPlaylists: boolean; +}): Promise { + featureEnabled = config.enabled; + ytdlpCommand = config.command; + ffmpegCommand = config.ffmpegCommand; + musicDir = config.musicDir; + fastQueueConcurrent = config.fastQueueConcurrent; + slowQueueInterval = config.slowQueueInterval; + allowPlaylists = config.allowPlaylists; + + if (!featureEnabled) { + console.log("[ytdlp] Feature disabled in config"); + return { available: false, enabled: false, version: null, ffmpeg: false }; + } + + // Check yt-dlp availability + try { + ytdlpVersion = await runCommand(ytdlpCommand, ["--version"]); + ytdlpAvailable = true; + console.log(`[ytdlp] Found yt-dlp version: ${ytdlpVersion.trim()}`); + } catch (e) { + console.error(`[ytdlp] yt-dlp not found (command: ${ytdlpCommand})`); + ytdlpAvailable = false; + featureEnabled = false; + } + + // Check ffmpeg availability + try { + await runCommand(ffmpegCommand, ["-version"]); + ffmpegAvailable = true; + console.log("[ytdlp] ffmpeg available"); + } catch (e) { + console.warn("[ytdlp] ffmpeg not found - audio extraction may fail"); + ffmpegAvailable = false; + } + + // Start slow queue processor + if (featureEnabled) { + startSlowQueueProcessor(); + } + + return getStatus(); } -// Get request by ID -export function getRequest(id: string): FetchRequest | undefined { - return fetchRequests.get(id); +// Run a command and return stdout +function runCommand(cmd: string, args: string[]): Promise { + const fullCmd = `${cmd} ${args.join(" ")}`; + console.log(`[ytdlp] Running: ${fullCmd}`); + return new Promise((resolve, reject) => { + const proc = spawn(cmd, args, { shell: true }); + let stdout = ""; + let stderr = ""; + proc.stdout.on("data", (data) => { stdout += data; }); + proc.stderr.on("data", (data) => { stderr += data; }); + proc.on("close", (code) => { + console.log(`[ytdlp] Command exited with code ${code}`); + if (code === 0) resolve(stdout); + else reject(new Error(stderr || `Exit code ${code}`)); + }); + proc.on("error", reject); + }); } -// Create a new fetch request -export function createFetchRequest(url: string, userId: number): FetchRequest { - const request: FetchRequest = { +// Get current status +export function getStatus(): YtdlpStatus { + return { + available: ytdlpAvailable, + enabled: featureEnabled, + version: ytdlpVersion, + ffmpeg: ffmpegAvailable + }; +} + +// Check if feature is enabled and available +export function isAvailable(): boolean { + return featureEnabled && ytdlpAvailable; +} + +// Set progress callback +export function setProgressCallback(callback: ProgressCallback): void { + onProgress = callback; +} + +// Get all queue items +export function getQueues(): { fastQueue: QueueItem[]; slowQueue: QueueItem[]; slowQueueNextIn: number } { + const now = Date.now(); + const nextIn = Math.max(0, Math.floor((lastSlowDownload + slowQueueInterval * 1000 - now) / 1000)); + return { + fastQueue: [...fastQueue], + slowQueue: [...slowQueue], + slowQueueNextIn: nextIn + }; +} + +// Get queue items for a specific user +export function getUserQueues(userId: number): { fastQueue: QueueItem[]; slowQueue: QueueItem[]; slowQueueNextIn: number } { + const queues = getQueues(); + return { + fastQueue: queues.fastQueue.filter(i => i.userId === userId), + slowQueue: queues.slowQueue.filter(i => i.userId === userId), + slowQueueNextIn: queues.slowQueueNextIn + }; +} + +// Check URL and detect if it's a playlist +export async function checkUrl(url: string): Promise { + const args = ["--flat-playlist", "--dump-json", "--no-warnings", url]; + const output = await runCommand(ytdlpCommand, args); + + // Parse JSON lines + const lines = output.trim().split("\n").filter(l => l); + + if (lines.length === 0) { + throw new Error("No video found"); + } + + if (lines.length === 1) { + const data = JSON.parse(lines[0]); + if (data._type === "playlist") { + // It's a playlist with entries + const items = (data.entries || []).map((e: any) => ({ + id: generateId(), + url: e.url || e.webpage_url || `https://youtube.com/watch?v=${e.id}`, + title: e.title || "Unknown" + })); + return { + type: "playlist", + title: data.title || "Playlist", + count: items.length, + items, + requiresConfirmation: true + }; + } else { + // Single video + return { + type: "single", + id: generateId(), + title: data.title || "Unknown", + url + }; + } + } else { + // Multiple JSON lines = playlist + const items = lines.map(line => { + const data = JSON.parse(line); + return { + id: generateId(), + url: data.url || data.webpage_url || url, + title: data.title || "Unknown" + }; + }); + return { + type: "playlist", + title: "Playlist", + count: items.length, + items, + requiresConfirmation: true + }; + } +} + +// Add single video to fast queue +export function addToFastQueue(url: string, title: string, userId: number): QueueItem { + const item: QueueItem = { id: generateId(), url, - status: "pending", - progress: 0, - createdAt: Date.now(), + title, userId, + status: "queued", + progress: 0, + queueType: "fast", + createdAt: Date.now() }; - fetchRequests.set(request.id, request); - return request; + fastQueue.push(item); + processNextFast(); + return item; } -// Update request status -export function updateRequest(id: string, updates: Partial): void { - const request = fetchRequests.get(id); - if (request) { - Object.assign(request, updates); +// Add items to slow queue (for playlists) +export function addToSlowQueue(items: { url: string; title: string }[], userId: number): QueueItem[] { + const queueItems: QueueItem[] = items.map(item => ({ + id: generateId(), + url: item.url, + title: item.title, + userId, + status: "queued" as const, + progress: 0, + queueType: "slow" as const, + createdAt: Date.now() + })); + slowQueue.push(...queueItems); + return queueItems; +} + +// Process next item in fast queue +function processNextFast(): void { + if (activeDownloads >= fastQueueConcurrent) return; + + const item = fastQueue.find(i => i.status === "queued"); + if (!item) return; + + activeDownloads++; + downloadItem(item).finally(() => { + activeDownloads--; + processNextFast(); + }); +} + +// Start slow queue processor +function startSlowQueueProcessor(): void { + if (slowQueueTimer) return; + + const processNext = () => { + const item = slowQueue.find(i => i.status === "queued"); + if (item) { + lastSlowDownload = Date.now(); + downloadItem(item).finally(() => { + slowQueueTimer = setTimeout(processNext, slowQueueInterval * 1000); + }); + } else { + slowQueueTimer = setTimeout(processNext, 5000); // Check again in 5s + } + }; + + // Start immediately if there are items + const hasQueued = slowQueue.some(i => i.status === "queued"); + if (hasQueued) { + processNext(); + } else { + slowQueueTimer = setTimeout(processNext, 5000); } } -// Remove completed/failed requests older than given age (ms) -export function cleanupOldRequests(maxAge: number = 3600000): void { +// Download a single item +async function downloadItem(item: QueueItem): Promise { + item.status = "downloading"; + item.progress = 0; + notifyProgress(item); + + console.log(`[ytdlp] Starting download: ${item.title} (${item.url})`); + + try { + const outputTemplate = join(musicDir, "%(title)s.%(ext)s"); + const args = [ + "-x", + "--audio-format", "mp3", + "-o", outputTemplate, + "--progress", + "--newline", + "--no-warnings", + item.url + ]; + + const fullCmd = `${ytdlpCommand} ${args.join(" ")}`; + console.log(`[ytdlp] Running: ${fullCmd}`); + + await new Promise((resolve, reject) => { + const proc = spawn(ytdlpCommand, args, { shell: true }); + + proc.stdout.on("data", (data) => { + const line = data.toString(); + console.log(`[ytdlp] ${line.trim()}`); + // Parse progress from yt-dlp output + const match = line.match(/(\d+\.?\d*)%/); + if (match) { + item.progress = parseFloat(match[1]); + notifyProgress(item); + } + }); + + proc.stderr.on("data", (data) => { + console.error(`[ytdlp] stderr: ${data}`); + }); + + proc.on("close", (code) => { + console.log(`[ytdlp] Download finished with code ${code}`); + if (code === 0) resolve(); + else reject(new Error(`yt-dlp exited with code ${code}`)); + }); + + proc.on("error", reject); + }); + + console.log(`[ytdlp] Complete: ${item.title}`); + item.status = "complete"; + item.progress = 100; + item.completedAt = Date.now(); + notifyProgress(item); + + // Remove from queue after delay + setTimeout(() => removeFromQueue(item), 5000); + + } catch (e: any) { + item.status = "error"; + item.error = e.message || "Download failed"; + notifyProgress(item); + + // Remove from queue after delay + setTimeout(() => removeFromQueue(item), 10000); + } +} + +// Remove item from queue +function removeFromQueue(item: QueueItem): void { + if (item.queueType === "fast") { + const idx = fastQueue.findIndex(i => i.id === item.id); + if (idx !== -1) fastQueue.splice(idx, 1); + } else { + const idx = slowQueue.findIndex(i => i.id === item.id); + if (idx !== -1) slowQueue.splice(idx, 1); + } +} + +// Notify progress callback +function notifyProgress(item: QueueItem): void { + if (onProgress) { + onProgress(item); + } +} + +// Cleanup old completed/failed items +export function cleanupOldItems(maxAge: number = 3600000): void { const now = Date.now(); - for (const [id, request] of fetchRequests) { - if (request.status === "complete" || request.status === "error") { - if (now - request.createdAt > maxAge) { - fetchRequests.delete(id); + const cleanup = (queue: QueueItem[]) => { + for (let i = queue.length - 1; i >= 0; i--) { + const item = queue[i]; + if ((item.status === "complete" || item.status === "error") && + now - item.createdAt > maxAge) { + queue.splice(i, 1); } } - } -} - -// TODO: Implement actual yt-dlp download -// This will: -// 1. Spawn yt-dlp process with URL -// 2. Parse progress output -// 3. Update request status -// 4. Move completed file to music directory -// 5. Trigger library rescan -export async function startDownload(request: FetchRequest, musicDir: string): Promise { - // Stub - to be implemented - console.log(`[ytdlp] Would download: ${request.url} to ${musicDir}`); - updateRequest(request.id, { status: "error", error: "Not implemented yet" }); + }; + cleanup(fastQueue); + cleanup(slowQueue); }