ytdlp upgrade

This commit is contained in:
peterino2 2026-02-04 02:02:25 -08:00
parent cc9324fb65
commit 5a318e7d8c
5 changed files with 605 additions and 71 deletions

View File

@ -280,6 +280,13 @@
M.showToast(data.message, data.toastType || "info");
return;
}
// Handle fetch progress from ytdlp
if (data.type && data.type.startsWith("fetch_")) {
if (M.handleFetchProgress) {
M.handleFetchProgress(data);
}
return;
}
// Normal channel state update
M.handleUpdate(data);
};

View File

@ -59,8 +59,18 @@
initPanelTabs();
});
// Update UI based on server status
function updateFeatureVisibility() {
const fetchBtn = M.$("#btn-fetch-url");
if (fetchBtn) {
const ytdlpEnabled = M.serverStatus?.ytdlp?.enabled && M.serverStatus?.ytdlp?.available;
fetchBtn.style.display = ytdlpEnabled ? "" : "none";
}
}
// Initialize the application
Promise.all([initStorage(), loadServerStatus()]).then(async () => {
updateFeatureVisibility();
await M.loadLibrary();
await M.loadCurrentUser();
if (M.currentUser) {

View File

@ -89,9 +89,7 @@
return;
}
closeFetchDialog();
// Create a task for this fetch
const task = createTask(`Fetching: ${url.substring(0, 50)}...`);
M.showToast("Checking URL...");
try {
const res = await fetch("/api/fetch", {
@ -102,15 +100,39 @@
if (res.ok) {
const data = await res.json();
task.setComplete();
M.showToast(data.message || "Fetch started");
if (data.type === "playlist") {
// Ask user to confirm playlist download
const confirmed = confirm(`Download playlist "${data.title}" with ${data.count} items?\n\nItems will be downloaded slowly (one every ~3 minutes) to avoid overloading the server.`);
if (confirmed) {
// Confirm playlist download
const confirmRes = await fetch("/api/fetch/confirm", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ items: data.items })
});
if (confirmRes.ok) {
const confirmData = await confirmRes.json();
M.showToast(confirmData.message);
// Tasks will be created by WebSocket progress messages
} else {
const err = await confirmRes.json().catch(() => ({}));
M.showToast(err.error || "Failed to queue playlist", "error");
}
}
} else if (data.type === "single") {
M.showToast(`Queued: ${data.title}`);
// Task will be created by WebSocket progress messages
} else {
M.showToast(data.message || "Fetch started");
}
} else {
const err = await res.json().catch(() => ({}));
task.setError(err.error || "Failed");
M.showToast(err.error || "Fetch failed", "error");
}
} catch (e) {
task.setError("Error");
M.showToast("Fetch failed", "error");
}
};
@ -175,12 +197,36 @@
};
// Task management
const fetchTasks = new Map(); // Map<id, taskHandle>
function updateTasksEmpty() {
const hasTasks = tasksList.children.length > 0;
tasksEmpty.classList.toggle("hidden", hasTasks);
}
function createTask(filename) {
// Handle WebSocket fetch progress messages
M.handleFetchProgress = function(data) {
let task = fetchTasks.get(data.id);
// Create task if we don't have one for this id
if (!task && data.status !== "complete" && data.status !== "error") {
task = createTask(data.title || "Downloading...", data.id);
}
if (!task) return;
if (data.status === "downloading" || data.status === "queued") {
task.setProgress(data.progress || 0);
} else if (data.status === "complete") {
task.setComplete();
fetchTasks.delete(data.id);
} else if (data.status === "error") {
task.setError(data.error || "Failed");
fetchTasks.delete(data.id);
}
};
function createTask(filename, fetchId) {
const task = document.createElement("div");
task.className = "task-item";
task.innerHTML = `
@ -197,7 +243,7 @@
const tasksTab = document.querySelector('.panel-tab[data-tab="tasks"]');
if (tasksTab) tasksTab.click();
return {
const taskHandle = {
setProgress(percent) {
task.querySelector(".task-progress").textContent = `${Math.round(percent)}%`;
task.querySelector(".task-bar").style.width = `${percent}%`;
@ -222,6 +268,13 @@
}, 5000);
}
};
// Store fetch tasks for WebSocket updates
if (fetchId) {
fetchTasks.set(fetchId, taskHandle);
}
return taskHandle;
}
function uploadFile(file) {

159
server.ts
View File

@ -33,16 +33,37 @@ import {
getClientInfo,
} from "./auth";
import { Library } from "./library";
import { createFetchRequest, getUserRequests, startDownload } from "./ytdlp";
import {
initYtdlp,
getStatus as getYtdlpStatus,
isAvailable as isYtdlpAvailable,
checkUrl,
addToFastQueue,
addToSlowQueue,
getUserQueues,
setProgressCallback,
type QueueItem
} from "./ytdlp";
// Load config
interface YtdlpConfig {
enabled: boolean;
command: string;
ffmpegCommand: string;
updateCommand: string | null;
fastQueueConcurrent: number;
slowQueueInterval: number;
allowPlaylists: boolean;
autoUpdate: boolean;
updateCheckInterval: number;
}
interface Config {
port: number;
musicDir: string;
allowGuests: boolean;
defaultPermissions: {
channel?: string;
};
defaultPermissions: string[];
ytdlp?: YtdlpConfig;
}
const CONFIG_PATH = join(import.meta.dir, "config.json");
@ -51,7 +72,18 @@ const DEFAULT_CONFIG: Config = {
port: 3001,
musicDir: "./music",
allowGuests: true,
defaultPermissions: ["listen", "control"]
defaultPermissions: ["listen", "control"],
ytdlp: {
enabled: false,
command: "yt-dlp",
ffmpegCommand: "ffmpeg",
updateCommand: "yt-dlp -U",
fastQueueConcurrent: 2,
slowQueueInterval: 180,
allowPlaylists: true,
autoUpdate: true,
updateCheckInterval: 86400
}
};
// Create default config if missing
@ -70,6 +102,18 @@ const PUBLIC_DIR = join(import.meta.dir, "public");
console.log(`Config loaded: port=${config.port}, musicDir=${MUSIC_DIR}, allowGuests=${config.allowGuests}`);
// Initialize yt-dlp if configured
const ytdlpConfig = config.ytdlp || DEFAULT_CONFIG.ytdlp!;
const ytdlpStatus = await initYtdlp({
enabled: ytdlpConfig.enabled,
command: ytdlpConfig.command,
ffmpegCommand: ytdlpConfig.ffmpegCommand,
musicDir: MUSIC_DIR,
fastQueueConcurrent: ytdlpConfig.fastQueueConcurrent,
slowQueueInterval: ytdlpConfig.slowQueueInterval,
allowPlaylists: ytdlpConfig.allowPlaylists
});
// Initialize library
const library = new Library(MUSIC_DIR);
@ -245,6 +289,30 @@ function broadcastToAll(message: object) {
console.log(`[Broadcast] Sent to ${clientCount} clients`);
}
// Send message to specific user's connections
function sendToUser(userId: number, message: object) {
const connections = userConnections.get(userId);
if (connections) {
const data = JSON.stringify(message);
for (const ws of connections) {
ws.send(data);
}
}
}
// Set up ytdlp progress callback
setProgressCallback((item) => {
sendToUser(item.userId, {
type: `fetch_${item.status === "downloading" ? "progress" : item.status}`,
id: item.id,
title: item.title,
status: item.status,
progress: item.progress,
queueType: item.queueType,
error: item.error
});
});
// Broadcast channel list to all clients
function broadcastChannelList() {
const list = [...channels.values()].map(c => c.getListInfo());
@ -375,6 +443,7 @@ serve({
allowSignups: true,
channelCount: channels.size,
defaultPermissions: config.defaultPermissions,
ytdlp: getYtdlpStatus()
});
}
@ -582,38 +651,96 @@ serve({
return Response.json({ error: "Guests cannot fetch from URLs" }, { status: 403 });
}
// Check if feature is enabled
if (!ytdlpConfig.enabled) {
return Response.json({ error: "Feature disabled" }, { status: 403 });
}
if (!isYtdlpAvailable()) {
return Response.json({ error: "yt-dlp not available" }, { status: 503 });
}
try {
const { url } = await req.json();
if (!url || typeof url !== "string") {
return Response.json({ error: "URL is required" }, { status: 400 });
}
// Create fetch request
const request = createFetchRequest(url, user.id);
console.log(`[Fetch] ${user.username} requested: ${url} (id=${request.id})`);
console.log(`[Fetch] ${user.username} checking URL: ${url}`);
// Start download in background (don't await)
startDownload(request, config.musicDir);
// Check URL to detect playlist vs single video
const info = await checkUrl(url);
if (info.type === "playlist") {
if (!ytdlpConfig.allowPlaylists) {
return Response.json({ error: "Playlist downloads are disabled" }, { status: 403 });
}
// Return playlist info for confirmation
return Response.json(info, { headers });
} else {
// Single video - add to fast queue immediately
const item = addToFastQueue(info.url, info.title, user.id);
console.log(`[Fetch] ${user.username} queued: ${info.title} (id=${item.id})`);
return Response.json({
type: "single",
id: item.id,
title: item.title,
queueType: "fast"
}, { headers });
}
} catch (e: any) {
console.error("[Fetch] Error:", e);
return Response.json({ error: e.message || "Invalid request" }, { status: 400 });
}
}
// API: confirm playlist download
if (path === "/api/fetch/confirm" && req.method === "POST") {
const { user, headers } = getOrCreateUser(req, server);
if (!user) {
return Response.json({ error: "Authentication required" }, { status: 401 });
}
if (user.is_guest) {
return Response.json({ error: "Guests cannot fetch from URLs" }, { status: 403 });
}
if (!ytdlpConfig.enabled || !isYtdlpAvailable()) {
return Response.json({ error: "Feature not available" }, { status: 503 });
}
try {
const { items } = await req.json();
if (!Array.isArray(items) || items.length === 0) {
return Response.json({ error: "Items required" }, { status: 400 });
}
const queueItems = addToSlowQueue(items, user.id);
const estimatedMinutes = Math.ceil(queueItems.length * ytdlpConfig.slowQueueInterval / 60);
const hours = Math.floor(estimatedMinutes / 60);
const mins = estimatedMinutes % 60;
const estimatedTime = hours > 0 ? `${hours}h ${mins}m` : `${mins}m`;
console.log(`[Fetch] ${user.username} confirmed playlist: ${queueItems.length} items`);
return Response.json({
message: "Fetch started",
requestId: request.id
message: `Added ${queueItems.length} items to queue`,
queueType: "slow",
estimatedTime,
items: queueItems.map(i => ({ id: i.id, title: i.title }))
}, { headers });
} catch (e) {
console.error("[Fetch] Error:", e);
console.error("[Fetch] Confirm error:", e);
return Response.json({ error: "Invalid request" }, { status: 400 });
}
}
// API: get fetch requests for current user
// API: get fetch queue status for current user
if (path === "/api/fetch" && req.method === "GET") {
const { user, headers } = getOrCreateUser(req, server);
if (!user) {
return Response.json({ error: "Authentication required" }, { status: 401 });
}
const requests = getUserRequests(user.id);
return Response.json({ requests }, { headers });
const queues = getUserQueues(user.id);
return Response.json(queues, { headers });
}
// Auth: signup

427
ytdlp.ts
View File

@ -1,79 +1,416 @@
// MusicRoom - yt-dlp integration module
// Handles fetching audio from URLs via yt-dlp
export interface FetchRequest {
import { spawn } from "child_process";
import { join } from "path";
export interface QueueItem {
id: string;
url: string;
status: "pending" | "downloading" | "processing" | "complete" | "error";
title: string;
userId: number;
status: "queued" | "downloading" | "complete" | "error";
progress: number;
queueType: "fast" | "slow";
error?: string;
filename?: string;
createdAt: number;
completedAt?: number;
userId: number;
}
// In-memory store of active fetch requests
const fetchRequests = new Map<string, FetchRequest>();
export interface YtdlpStatus {
available: boolean;
enabled: boolean;
version: string | null;
ffmpeg: boolean;
}
// Generate unique request ID
export interface PlaylistInfo {
type: "playlist";
title: string;
count: number;
items: { id: string; url: string; title: string }[];
requiresConfirmation: true;
}
export interface SingleVideoInfo {
type: "single";
id: string;
title: string;
url: string;
}
type ProgressCallback = (item: QueueItem) => void;
// Configuration
let ytdlpCommand = "yt-dlp";
let ffmpegCommand = "ffmpeg";
let musicDir = "./music";
let fastQueueConcurrent = 2;
let slowQueueInterval = 180;
let allowPlaylists = true;
// Status
let ytdlpAvailable = false;
let ytdlpVersion: string | null = null;
let ffmpegAvailable = false;
let featureEnabled = false;
// Queues
const fastQueue: QueueItem[] = [];
const slowQueue: QueueItem[] = [];
let activeDownloads = 0;
let slowQueueTimer: ReturnType<typeof setTimeout> | null = null;
let lastSlowDownload = 0;
// Callbacks
let onProgress: ProgressCallback | null = null;
// Generate unique ID
function generateId(): string {
return Math.random().toString(36).substring(2, 10);
}
// Get all requests for a user
export function getUserRequests(userId: number): FetchRequest[] {
return [...fetchRequests.values()].filter(r => r.userId === userId);
// Initialize ytdlp module
export async function initYtdlp(config: {
enabled: boolean;
command: string;
ffmpegCommand: string;
musicDir: string;
fastQueueConcurrent: number;
slowQueueInterval: number;
allowPlaylists: boolean;
}): Promise<YtdlpStatus> {
featureEnabled = config.enabled;
ytdlpCommand = config.command;
ffmpegCommand = config.ffmpegCommand;
musicDir = config.musicDir;
fastQueueConcurrent = config.fastQueueConcurrent;
slowQueueInterval = config.slowQueueInterval;
allowPlaylists = config.allowPlaylists;
if (!featureEnabled) {
console.log("[ytdlp] Feature disabled in config");
return { available: false, enabled: false, version: null, ffmpeg: false };
}
// Check yt-dlp availability
try {
ytdlpVersion = await runCommand(ytdlpCommand, ["--version"]);
ytdlpAvailable = true;
console.log(`[ytdlp] Found yt-dlp version: ${ytdlpVersion.trim()}`);
} catch (e) {
console.error(`[ytdlp] yt-dlp not found (command: ${ytdlpCommand})`);
ytdlpAvailable = false;
featureEnabled = false;
}
// Check ffmpeg availability
try {
await runCommand(ffmpegCommand, ["-version"]);
ffmpegAvailable = true;
console.log("[ytdlp] ffmpeg available");
} catch (e) {
console.warn("[ytdlp] ffmpeg not found - audio extraction may fail");
ffmpegAvailable = false;
}
// Start slow queue processor
if (featureEnabled) {
startSlowQueueProcessor();
}
return getStatus();
}
// Get request by ID
export function getRequest(id: string): FetchRequest | undefined {
return fetchRequests.get(id);
// Run a command and return stdout
function runCommand(cmd: string, args: string[]): Promise<string> {
const fullCmd = `${cmd} ${args.join(" ")}`;
console.log(`[ytdlp] Running: ${fullCmd}`);
return new Promise((resolve, reject) => {
const proc = spawn(cmd, args, { shell: true });
let stdout = "";
let stderr = "";
proc.stdout.on("data", (data) => { stdout += data; });
proc.stderr.on("data", (data) => { stderr += data; });
proc.on("close", (code) => {
console.log(`[ytdlp] Command exited with code ${code}`);
if (code === 0) resolve(stdout);
else reject(new Error(stderr || `Exit code ${code}`));
});
proc.on("error", reject);
});
}
// Create a new fetch request
export function createFetchRequest(url: string, userId: number): FetchRequest {
const request: FetchRequest = {
// Get current status
export function getStatus(): YtdlpStatus {
return {
available: ytdlpAvailable,
enabled: featureEnabled,
version: ytdlpVersion,
ffmpeg: ffmpegAvailable
};
}
// Check if feature is enabled and available
export function isAvailable(): boolean {
return featureEnabled && ytdlpAvailable;
}
// Set progress callback
export function setProgressCallback(callback: ProgressCallback): void {
onProgress = callback;
}
// Get all queue items
export function getQueues(): { fastQueue: QueueItem[]; slowQueue: QueueItem[]; slowQueueNextIn: number } {
const now = Date.now();
const nextIn = Math.max(0, Math.floor((lastSlowDownload + slowQueueInterval * 1000 - now) / 1000));
return {
fastQueue: [...fastQueue],
slowQueue: [...slowQueue],
slowQueueNextIn: nextIn
};
}
// Get queue items for a specific user
export function getUserQueues(userId: number): { fastQueue: QueueItem[]; slowQueue: QueueItem[]; slowQueueNextIn: number } {
const queues = getQueues();
return {
fastQueue: queues.fastQueue.filter(i => i.userId === userId),
slowQueue: queues.slowQueue.filter(i => i.userId === userId),
slowQueueNextIn: queues.slowQueueNextIn
};
}
// Check URL and detect if it's a playlist
export async function checkUrl(url: string): Promise<PlaylistInfo | SingleVideoInfo> {
const args = ["--flat-playlist", "--dump-json", "--no-warnings", url];
const output = await runCommand(ytdlpCommand, args);
// Parse JSON lines
const lines = output.trim().split("\n").filter(l => l);
if (lines.length === 0) {
throw new Error("No video found");
}
if (lines.length === 1) {
const data = JSON.parse(lines[0]);
if (data._type === "playlist") {
// It's a playlist with entries
const items = (data.entries || []).map((e: any) => ({
id: generateId(),
url: e.url || e.webpage_url || `https://youtube.com/watch?v=${e.id}`,
title: e.title || "Unknown"
}));
return {
type: "playlist",
title: data.title || "Playlist",
count: items.length,
items,
requiresConfirmation: true
};
} else {
// Single video
return {
type: "single",
id: generateId(),
title: data.title || "Unknown",
url
};
}
} else {
// Multiple JSON lines = playlist
const items = lines.map(line => {
const data = JSON.parse(line);
return {
id: generateId(),
url: data.url || data.webpage_url || url,
title: data.title || "Unknown"
};
});
return {
type: "playlist",
title: "Playlist",
count: items.length,
items,
requiresConfirmation: true
};
}
}
// Add single video to fast queue
export function addToFastQueue(url: string, title: string, userId: number): QueueItem {
const item: QueueItem = {
id: generateId(),
url,
status: "pending",
progress: 0,
createdAt: Date.now(),
title,
userId,
status: "queued",
progress: 0,
queueType: "fast",
createdAt: Date.now()
};
fetchRequests.set(request.id, request);
return request;
fastQueue.push(item);
processNextFast();
return item;
}
// Update request status
export function updateRequest(id: string, updates: Partial<FetchRequest>): void {
const request = fetchRequests.get(id);
if (request) {
Object.assign(request, updates);
// Add items to slow queue (for playlists)
export function addToSlowQueue(items: { url: string; title: string }[], userId: number): QueueItem[] {
const queueItems: QueueItem[] = items.map(item => ({
id: generateId(),
url: item.url,
title: item.title,
userId,
status: "queued" as const,
progress: 0,
queueType: "slow" as const,
createdAt: Date.now()
}));
slowQueue.push(...queueItems);
return queueItems;
}
// Process next item in fast queue
function processNextFast(): void {
if (activeDownloads >= fastQueueConcurrent) return;
const item = fastQueue.find(i => i.status === "queued");
if (!item) return;
activeDownloads++;
downloadItem(item).finally(() => {
activeDownloads--;
processNextFast();
});
}
// Start slow queue processor
function startSlowQueueProcessor(): void {
if (slowQueueTimer) return;
const processNext = () => {
const item = slowQueue.find(i => i.status === "queued");
if (item) {
lastSlowDownload = Date.now();
downloadItem(item).finally(() => {
slowQueueTimer = setTimeout(processNext, slowQueueInterval * 1000);
});
} else {
slowQueueTimer = setTimeout(processNext, 5000); // Check again in 5s
}
};
// Start immediately if there are items
const hasQueued = slowQueue.some(i => i.status === "queued");
if (hasQueued) {
processNext();
} else {
slowQueueTimer = setTimeout(processNext, 5000);
}
}
// Remove completed/failed requests older than given age (ms)
export function cleanupOldRequests(maxAge: number = 3600000): void {
// Download a single item
async function downloadItem(item: QueueItem): Promise<void> {
item.status = "downloading";
item.progress = 0;
notifyProgress(item);
console.log(`[ytdlp] Starting download: ${item.title} (${item.url})`);
try {
const outputTemplate = join(musicDir, "%(title)s.%(ext)s");
const args = [
"-x",
"--audio-format", "mp3",
"-o", outputTemplate,
"--progress",
"--newline",
"--no-warnings",
item.url
];
const fullCmd = `${ytdlpCommand} ${args.join(" ")}`;
console.log(`[ytdlp] Running: ${fullCmd}`);
await new Promise<void>((resolve, reject) => {
const proc = spawn(ytdlpCommand, args, { shell: true });
proc.stdout.on("data", (data) => {
const line = data.toString();
console.log(`[ytdlp] ${line.trim()}`);
// Parse progress from yt-dlp output
const match = line.match(/(\d+\.?\d*)%/);
if (match) {
item.progress = parseFloat(match[1]);
notifyProgress(item);
}
});
proc.stderr.on("data", (data) => {
console.error(`[ytdlp] stderr: ${data}`);
});
proc.on("close", (code) => {
console.log(`[ytdlp] Download finished with code ${code}`);
if (code === 0) resolve();
else reject(new Error(`yt-dlp exited with code ${code}`));
});
proc.on("error", reject);
});
console.log(`[ytdlp] Complete: ${item.title}`);
item.status = "complete";
item.progress = 100;
item.completedAt = Date.now();
notifyProgress(item);
// Remove from queue after delay
setTimeout(() => removeFromQueue(item), 5000);
} catch (e: any) {
item.status = "error";
item.error = e.message || "Download failed";
notifyProgress(item);
// Remove from queue after delay
setTimeout(() => removeFromQueue(item), 10000);
}
}
// Remove item from queue
function removeFromQueue(item: QueueItem): void {
if (item.queueType === "fast") {
const idx = fastQueue.findIndex(i => i.id === item.id);
if (idx !== -1) fastQueue.splice(idx, 1);
} else {
const idx = slowQueue.findIndex(i => i.id === item.id);
if (idx !== -1) slowQueue.splice(idx, 1);
}
}
// Notify progress callback
function notifyProgress(item: QueueItem): void {
if (onProgress) {
onProgress(item);
}
}
// Cleanup old completed/failed items
export function cleanupOldItems(maxAge: number = 3600000): void {
const now = Date.now();
for (const [id, request] of fetchRequests) {
if (request.status === "complete" || request.status === "error") {
if (now - request.createdAt > maxAge) {
fetchRequests.delete(id);
const cleanup = (queue: QueueItem[]) => {
for (let i = queue.length - 1; i >= 0; i--) {
const item = queue[i];
if ((item.status === "complete" || item.status === "error") &&
now - item.createdAt > maxAge) {
queue.splice(i, 1);
}
}
}
}
// TODO: Implement actual yt-dlp download
// This will:
// 1. Spawn yt-dlp process with URL
// 2. Parse progress output
// 3. Update request status
// 4. Move completed file to music directory
// 5. Trigger library rescan
export async function startDownload(request: FetchRequest, musicDir: string): Promise<void> {
// Stub - to be implemented
console.log(`[ytdlp] Would download: ${request.url} to ${musicDir}`);
updateRequest(request.id, { status: "error", error: "Not implemented yet" });
};
cleanup(fastQueue);
cleanup(slowQueue);
}