Download Everything, YT-DLP and CURL - DownLoadAll. Need lyfe00011 Levanter Bot

dla.js

// dla.js is a complement to LevanterBot: https://github.com/lyfe00011/levanter
// Copyright (C) 2025 Weskerty
//
// Este programa se distribuye bajo los términos de la Licencia Pública General Affero de GNU (AGPLv3).
// Usted puede usarlo, modificarlo y redistribuirlo bajo esa licencia.
// Este software se proporciona SIN GARANTÍA alguna.
// Licencia completa: https://www.gnu.org/licenses/agpl-3.0.html


// Este script carga todo a RAM, no funciona filestream
// v19. Cargar Cookies desde Archivo o Texto. 

const fs = require('fs').promises;
const path = require('path');
const os = require('os');
const { promisify } = require('util');
const { exec: execCallback } = require('child_process');
const { bot, isUrl } = require('../lib');
require('dotenv').config();

const exec = promisify(execCallback);

const FILE_TYPES = {
  video: {
    extensions: new Set(['mp4', 'mkv', 'avi', 'webm', 'mov', 'flv', 'm4v']),
    mimetype: 'video/mp4',
  },
  image: {
    extensions: new Set(['jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp', 'tiff', 'svg']),
    mimetype: 'image/jpeg',
  },
  document: {
    extensions: new Set(['pdf', 'epub', 'docx', 'txt', 'apk', 'apks', 'zip', 'rar', 'iso', 'ini', 'cbr', 'cbz', 'torrent', 'json', 'xml', 'html', 'css', 'js', 'csv', 'xls', 'xlsx', 'ppt', 'pptx']),
    mimetypes: new Map([
      ['pdf', 'application/pdf'],
      ['epub', 'application/epub+zip'],
      ['docx', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'],
      ['txt', 'text/plain'],
      ['apk', 'application/vnd.android.package-archive'],
      ['apks', 'application/vnd.android.package-archive'],
      ['zip', 'application/zip'],
      ['rar', 'application/x-rar-compressed'],
      ['iso', 'application/x-iso9660-image'],
      ['ini', 'text/plain'],
      ['cbr', 'application/x-cbr'],
      ['cbz', 'application/x-cbz'],
      ['torrent', 'application/x-bittorrent'],
      ['json', 'application/json'],
      ['xml', 'application/xml'],
      ['html', 'text/html'],
      ['css', 'text/css'],
      ['js', 'application/javascript'],
      ['csv', 'text/csv'],
      ['xls', 'application/vnd.ms-excel'],
      ['xlsx', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'],
      ['ppt', 'application/vnd.ms-powerpoint'],
      ['pptx', 'application/vnd.openxmlformats-officedocument.presentationml.presentation'],
    ]),
    defaultMimetype: 'application/octet-stream',
  },
  audio: {
    extensions: new Set(['mp3', 'wav', 'ogg', 'flac', 'm4a', 'aac', 'wma']),
    mimetype: 'audio/mpeg',
  },
};


function getFileDetails(filePath) {
  const ext = path.extname(filePath).slice(1).toLowerCase();

  for (const [category, typeInfo] of Object.entries(FILE_TYPES)) {
    if (typeInfo.extensions.has(ext)) {
      return {
        category,
        mimetype: category === 'document'
          ? typeInfo.mimetypes.get(ext) || typeInfo.defaultMimetype
          : typeInfo.mimetype,
      };
    }
  }

  return {
    category: 'document',
    mimetype: FILE_TYPES.document.defaultMimetype,
  };
}

class DownloadQueue {
  constructor(maxConcurrent = 2) {
    this.queue = [];
    this.activeDownloads = 0;
    this.maxConcurrent = maxConcurrent;
  }

  async add(task) {
    return new Promise((resolve, reject) => {
      this.queue.push({ task, resolve, reject });
      this.processNext();
    });
  }

  async processNext() {
    if (this.activeDownloads >= this.maxConcurrent || this.queue.length === 0) {
      return;
    }

    this.activeDownloads++;
    const { task, resolve, reject } = this.queue.shift();

    try {
      const result = await task();
      resolve(result);
    } catch (error) {
      reject(error);
    } finally {
      this.activeDownloads--;
      this.processNext();
    }
  }
}

class MediaDownloader {
  constructor() {
    this.config = {
      tempDir: process.env.TEMP_DOWNLOAD_DIR || path.join(process.cwd(), 'tmp'),
      maxFileSize: (parseInt(process.env.MAX_UPLOAD, 10) * 1048576) || 1500000000,
      ytDlpPath: path.join(process.cwd(), 'media', 'bin'),
      maxConcurrent: parseInt(process.env.MAXSOLICITUD, 10) || 2,
      playlistLimit: parseInt(process.env.PLAYLIST_LIMIT, 10) || 10,
      cookies: process.env.COOKIES || null 
    };

    this.downloadQueue = new DownloadQueue(this.config.maxConcurrent);
    this.ytDlpCommand = null;

    this.ytDlpBinaries = new Map([
      ['win32-x64', 'yt-dlp.exe'],
      ['win32-ia32', 'yt-dlp_x86.exe'],
      ['darwin', 'yt-dlp_macos'],
      ['linux-x64', 'yt-dlp_linux'],
      ['linux-arm64', 'yt-dlp_linux_aarch64'],
      ['linux-arm', 'yt-dlp_linux_armv7l'],
      ['default', 'yt-dlp'],
    ]);

    this.formats = {
      video: process.env.YTDLP_VIDEO || '-f "sd/18/bestvideo[height<=720][vcodec*=h264]+bestaudio[acodec*=aac]/bestvideo[height<=720][vcodec*=h264]+bestaudio[acodec*=mp4a]/bestvideo[height<=720][vcodec*=h264]+bestaudio/bestvideo[height<=720]+bestaudio/bestvideo[vcodec*=h264]+bestaudio/bestvideo+bestaudio/best" --sponsorblock-mark all',
      audio: '-f "ba/best" -x --audio-format mp3 --audio-quality 0',
      playlist: '--yes-playlist',
      noPlaylist: '--no-playlist'
    };

    this.commonFlags = [
      '--restrict-filenames',
      '--extractor-retries 3',
      '--fragment-retries 3',
      '--compat-options no-youtube-unavailable-videos',
      '--ignore-errors',
      '--no-abort-on-error'
    ].join(' ');
  }

  buildCookiesFlag() {
    const cookiesPath = path.join(this.config.ytDlpPath, 'yt-dlp.cookies.txt');
    
    try {
      require('fs').accessSync(cookiesPath, require('fs').constants.F_OK);
      return `--cookies "${cookiesPath}"`;
    } catch {
      return this.config.cookies ? `--cookies "${this.config.cookies}"` : '';
    }
  }

  async safeExecute(command, silentError = false) {
    try {
      const result = await exec(command);
      return result;
    } catch (error) {
      if (!silentError) {
        console.error(`Command: ${command}`);
        console.error(`Error: ${error.message}`);
        if (error.stdout) console.error(`Stdout: ${error.stdout}`);
        if (error.stderr) console.error(`Stderr: ${error.stderr}`);
      }
      throw error;
    }
  }

  async isYtDlpAvailable() {
    try {
      await exec('yt-dlp --version', { stdio: 'ignore' });
      return true;
    } catch {
      return false;
    }
  }

  detectYtDlpBinaryName() {
    const platform = os.platform();
    const arch = os.arch();
    const key = `${platform}-${arch}`;
    return this.ytDlpBinaries.get(key) || this.ytDlpBinaries.get('default');
  }

  async ensureDirectories() {
    await Promise.all([
      fs.mkdir(this.config.tempDir, { recursive: true }),
      fs.mkdir(this.config.ytDlpPath, { recursive: true }),
    ]);
  }

  async detectYtDlpBinary(message) {
    if (this.ytDlpCommand) {
      return this.ytDlpCommand;
    }

    if (await this.isYtDlpAvailable()) {
      this.ytDlpCommand = 'nice -n 7 yt-dlp';
      return this.ytDlpCommand;
    }

    const fileName = this.detectYtDlpBinaryName();
    const filePath = path.join(this.config.ytDlpPath, fileName);
    
    try {
      await fs.access(filePath);
      this.ytDlpCommand = `nice -n 7 ${filePath}`;
      return this.ytDlpCommand;
    } catch {
      if (message) {
        this.ytDlpCommand = await this.downloadYtDlp(message);
        return this.ytDlpCommand;
      }
      return null;
    }
  }

  async downloadYtDlp(message) {
    await this.ensureDirectories();
    const fileName = this.detectYtDlpBinaryName();
    const downloadUrl = `https://github.com/yt-dlp/yt-dlp/releases/latest/download/${fileName}`;
    const filePath = path.join(this.config.ytDlpPath, fileName);

    try {
      await this.safeExecute(`curl -L -o "${filePath}" "${downloadUrl}"`);
      
      if (os.platform() !== 'win32') {
        await fs.chmod(filePath, '755');
      }
      
      return `nice -n 7 ${filePath}`;
    } catch (error) {
      const fetch = (await import('node-fetch')).default;
      
      const response = await fetch(downloadUrl);
      if (!response.ok) throw new Error(`Download failed: ${response.statusText}`);

      const buffer = Buffer.from(await response.arrayBuffer());
      await fs.writeFile(filePath, buffer);

      if (os.platform() !== 'win32') {
        await fs.chmod(filePath, '755');
      }

      return `nice -n 7 ${filePath}`;
    }
  }

  async processDownloadedFile(message, filePath, originalFileName, outputDir) {
    const { mimetype, category } = getFileDetails(filePath);
    const fileBuffer = await fs.readFile(filePath);

    await message.send(
      fileBuffer,
      { fileName: originalFileName, mimetype, quoted: message.data },
      category
    );

    await fs.unlink(filePath).catch(() => {});
  }

  async updateYtDlp(message, errorMsg = null) {
    try {
      const ytDlpPath = await this.detectYtDlpBinary(message);
      const updateCommand = `${ytDlpPath} --update-to nightly`;
      
      const result = await this.safeExecute(updateCommand);
      
      const updateOutput = result.stdout || result.stderr || 'yt-dlp actualizado exitosamente';
      const finalMsg = errorMsg ? `Stderr: ${errorMsg}\n\n${updateOutput}` : updateOutput;
      await message.send(finalMsg, { quoted: message.data });
      
      return true;
    } catch (error) {
      console.error(`Update error: ${error.message}`);
      const errorOutput = error.stdout || error.stderr || `Error al actualizar: ${error.message}`;
      const finalMsg = errorMsg ? `Stderr: ${errorMsg}\n\n${errorOutput}` : errorOutput;
      await message.send(finalMsg, { quoted: message.data });
      return false;
    }
  }

async uploadCookies(message, cookieText = null) {
  try {
    const quotedMessage = message.reply_message;
    let cookieContent = null;

    if (cookieText) {
      cookieContent = cookieText;
    } 
    else if (quotedMessage) {
      const mediaBuffer = await quotedMessage.downloadMediaMessage();
      if (!mediaBuffer) {
        await message.send('Error al descargar el archivo de cookies', { quoted: message.data });
        return;
      }
      cookieContent = mediaBuffer.toString();
    } 
    else {
      await message.send('Las cookies deben ser texto o citar un archivo de cookies', { quoted: message.data });
      return;
    }

    await this.ensureDirectories();
    const cookiesPath = path.join(this.config.ytDlpPath, 'yt-dlp.cookies.txt');
    await fs.writeFile(cookiesPath, cookieContent);
    
    await message.send('Cookies subida', { quoted: message.data });
  } catch (error) {
    console.error(`Cookie upload error: ${error.message}`);
    await message.send(`Error subiendo cookies: ${error.message}`, { quoted: message.data });
  }
}
  async downloadWithYtDlp(message, urls, customOptions = '', enablePlaylist = false) {
    return this.downloadQueue.add(async () => {
      const ytDlpPath = await this.detectYtDlpBinary(message);
      const sessionId = `yt-dlp_${Date.now()}`;
      const outputDir = path.join(this.config.tempDir, sessionId);
      const cookiesFlag = this.buildCookiesFlag();

      await this.ensureDirectories();
      await fs.mkdir(outputDir, { recursive: true });

      for (const url of urls) {
        const outputTemplate = path.join(outputDir, '%(title).20s.%(ext)s');
        const playlistFlag = enablePlaylist ? this.formats.playlist : this.formats.noPlaylist;
        const playlistItemsFlag = enablePlaylist ? `--playlist-items 1:${this.config.playlistLimit}` : '';
        
        const command = [
          ytDlpPath,
          `--max-filesize ${this.config.maxFileSize}`,
          this.commonFlags,
          playlistFlag,
          playlistItemsFlag,
          cookiesFlag,
          customOptions,
          `-o "${outputTemplate}"`,
          `"${url}"`
        ].filter(Boolean).join(' ');

        try {
          await this.safeExecute(command);
          
          const files = await fs.readdir(outputDir);
          for (const file of files) {
            try {
              await this.processDownloadedFile(
                message,
                path.join(outputDir, file),
                file,
                outputDir
              );
            } catch (processError) {
              console.error(`Failed to process file ${file}: ${processError.message}`);
            }
          }
        } catch (error) {
          const errorMsg = error.stderr || error.message || 'Error desconocido';
          await this.updateYtDlp(message, errorMsg);
        }
      }

      await fs.rm(outputDir, { recursive: true, force: true }).catch(() => {});
    });
  }

  async searchAndDownload(message, searchQuery, isVideo = false) {
    return this.downloadQueue.add(async () => {
      const sessionId = `yt-dlp_${Date.now()}`;
      const outputDir = path.join(this.config.tempDir, sessionId);
      const cookiesFlag = this.buildCookiesFlag(); 
      
      await this.ensureDirectories();
      await fs.mkdir(outputDir, { recursive: true });

      const outputTemplate = path.join(outputDir, '%(title).20s.%(ext)s');
      const ytDlpPath = await this.detectYtDlpBinary(message);
      
      const formatOptions = isVideo ? this.formats.video : this.formats.audio;
      
      const searchSources = [
        { source: 'ytsearch', name: 'YouTube' },
        ...(isVideo ? [] : [
            { source: 'scsearch', name: 'SoundCloud' },
            { source: 'nicosearch', name: 'NicoNico' }
        ])
      ];

      let success = false;

      for (const { source, name } of searchSources) {
        if (success) break;

        try {
          const command = [
            ytDlpPath,
            `--max-filesize ${this.config.maxFileSize}`,
            this.commonFlags,
            '--playlist-items 1',
            formatOptions,
            cookiesFlag,
            `-o "${outputTemplate}"`,
            `"${source}10:${searchQuery}"`
          ].filter(Boolean).join(' ');
          
          await this.safeExecute(command);

          const files = await fs.readdir(outputDir);
          if (files.length > 0) {
            await Promise.all(
              files.map(file => this.processDownloadedFile(
                message,
                path.join(outputDir, file),
                file,
                outputDir
              ))
            );
            success = true;
            break;
          }
        } catch (error) {
          const errorMsg = error.stderr || error.message || 'Error desconocido';
          await this.updateYtDlp(message, errorMsg);
          
          await fs.rm(outputDir, { recursive: true, force: true }).catch(() => {});
          await fs.mkdir(outputDir, { recursive: true });
        }
      }

      if (!success) {
        console.error(`No results found for ${searchQuery}`);
      }

      await fs.rm(outputDir, { recursive: true, force: true }).catch(() => {});
    });
  }
}

const mediaDownloader = new MediaDownloader();

bot(
  {
    pattern: 'dla ?(.*)',
    fromMe: true,
    desc: 'Download All Media Web Site.',
    type: 'download',
  },
  async (message, match) => {
    const input = match.trim() || message.reply_message?.text || '';
    
    if (!input) {
      await message.send(
        '> 🎶Search and Download Song:\n`dla` <query>\n' +
        '> 🎥Search and Download Video:\n`dla vd` <query>\n' +
        '> ⬇️Download All Media: \n`dla` <url> _YT-DLP FLAGS_ \n' +
        '> 🎵Download All Audio from Playlist: \n`dla mp3` <url> \n' +
        '> 🍪Cookies: youtu.be/KUk9nEf00_U' +
        '> 🌐More Info:\ngithub.com/yt-dlp/yt-dlp/blob/master/README.md#usage-and-options',
        { quoted: message.data }
      );
      return;
    }

    try {
      const args = input.match(/[^\s"]+|"([^"]*)"/g)?.map(arg => 
        arg.startsWith('"') && arg.endsWith('"') ? arg.slice(1, -1) : arg
      ) || [];

      const command = args[0];
      const remainingArgs = args.slice(1);
      const urls = remainingArgs.filter(arg => isUrl(arg));
      
if (command === 'cookies') {
  const cookiesIndex = input.toLowerCase().indexOf('cookies');
  if (cookiesIndex !== -1) {
    const cookieText = input.substring(cookiesIndex + 'cookies'.length).trim();
    await mediaDownloader.uploadCookies(message, cookieText || null);
  } else {
    await mediaDownloader.uploadCookies(message, null);
  }
  return;
}
      
      if (!urls.length) {
        if (input.includes('://')) {
          const inputUrl = input.match(/(https?:\/\/[^\s]+)/)?.[1];
          if (inputUrl) {
            const options = input.replace(inputUrl, '').trim();
            await mediaDownloader.downloadWithYtDlp(message, [inputUrl], `${mediaDownloader.formats.video} ${options}`);
            return;
          }
        }
        if (command === 'vd') {
          await mediaDownloader.searchAndDownload(message, remainingArgs.join(' '), true);
        } else {
          await mediaDownloader.searchAndDownload(message, input, false);
        }
        return;
      }

      switch (command) {
        case 'mp3':
          if (urls.length) {
            const options = remainingArgs
              .filter(arg => !isUrl(arg))
              .join(' ');
            await mediaDownloader.downloadWithYtDlp(
              message, 
              urls, 
              `${mediaDownloader.formats.audio} ${options}`,
              true
            );
          }
          break;

        default:
          const options = remainingArgs
            .filter(arg => !isUrl(arg))
            .join(' ');
          await mediaDownloader.downloadWithYtDlp(
            message, 
            urls, 
            `${mediaDownloader.formats.video} ${options}`
          );
          break;
      }
    } catch (error) {
      console.error(`Error in dla command: ${error.message}`);
    }
  }
);

module.exports = { mediaDownloader };

I can't download YouTube shorts

YT-DLP-DLA Plugin is a complement to BaseBot:
Levanter: lyfe00011/levanter
Or Mystic (Old Base)

It showing can't read properties of mp4

…

On Tue, Dec 31, 2024, 2:06 AM laWiskaPY ***@***.***> wrote: ***@***.**** commented on this gist. ------------------------------ I can't download YouTube shorts What error does it show? Because it works perfectly for me. image.png (view on web) <https://gist.github.com/user-attachments/assets/738a2883-00bc-4085-a26c-ceac932e2d7e> — Reply to this email directly, view it on GitHub <https://gist.github.com/weskerty/ce6a4ea2c4b0a73889cae8431911734d#gistcomment-5369375> or unsubscribe <https://github.com/notifications/unsubscribe-auth/BIQPSKCTROCFWYPK3VM3LB32IHU2DBFKMF2HI4TJMJ2XIZLTSKBKK5TBNR2WLJDUOJ2WLJDOMFWWLO3UNBZGKYLEL5YGC4TUNFRWS4DBNZ2F6YLDORUXM2LUPGBKK5TBNR2WLJDHNFZXJJDOMFWWLK3UNBZGKYLEL52HS4DFVRZXKYTKMVRXIX3UPFYGLK2HNFZXIQ3PNVWWK3TUUZ2G64DJMNZZDAVEOR4XAZNEM5UXG5FFOZQWY5LFVEYTGNBSGQ4TMOJVU52HE2LHM5SXFJTDOJSWC5DF> . You are receiving this email because you commented on the thread. Triage notifications on the go with GitHub Mobile for iOS <https://apps.apple.com/app/apple-store/id1477376905?ct=notification-email&mt=8&pt=524675> or Android <https://play.google.com/store/apps/details?id=com.github.android&referrer=utm_campaign%3Dnotification-email%26utm_medium%3Demail%26utm_source%3Dgithub> .

It showing can't read properties of mp4

Does this happen with all shorts?
What are you running it on? What server? What Linux version?

I can't download YouTube shorts

What error does it show? Because it works perfectly for me.

Well, that's because you are using termux right?

Well, that's because you are using termux right?

Yes, I use Termux.
If you use an external server that doesn't support ffmpeg, there's nothing you can do, this is not a problem with the bot, the script or yt-dlp, it's a problem with your server.