Download Everything, YT-DLP and CURL - DownLoadAll.

dla.js

// MR. De la Comunidad para la Comunidad. Prohibida su Venta.	
// El Software se proporciona bajo los términos de la Licencia MIT, excepto que usted no puede:	
// 1. Vender, revender o arrendar el Software.	
// 2. Cobrar a otros por el acceso, la distribución o cualquier otro uso comercial del Software.	
// 3. Usar el Software como parte de un producto comercial o una oferta de servicio.
// v12 Eliminado dla curl/wget. Limite Nice. (es posible que falle en windows, requiere borrar nice)

const fs = require('fs').promises;
const path = require('path');
const os = require('os');
const { promisify } = require('util');
const { exec: execCallback } = require('child_process');
const { bot, isUrl } = require('../lib');
require('dotenv').config();

const exec = promisify(execCallback);

const FILE_TYPES = {
  video: {
    extensions: new Set(['mp4', 'mkv', 'avi', 'webm', 'mov', 'flv', 'm4v']),
    mimetype: 'video/mp4',
  },
  image: {
    extensions: new Set(['jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp', 'tiff', 'svg']),
    mimetype: 'image/jpeg',
  },
  document: {
    extensions: new Set(['pdf', 'epub', 'docx', 'txt', 'apk', 'apks', 'zip', 'rar', 'iso', 'ini', 'cbr', 'cbz', 'torrent', 'json', 'xml', 'html', 'css', 'js', 'csv', 'xls', 'xlsx', 'ppt', 'pptx']),
    mimetypes: new Map([
      ['pdf', 'application/pdf'],
      ['epub', 'application/epub+zip'],
      ['docx', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'],
      ['txt', 'text/plain'],
      ['apk', 'application/vnd.android.package-archive'],
      ['apks', 'application/vnd.android.package-archive'],
      ['zip', 'application/zip'],
      ['rar', 'application/x-rar-compressed'],
      ['iso', 'application/x-iso9660-image'],
      ['ini', 'text/plain'],
      ['cbr', 'application/x-cbr'],
      ['cbz', 'application/x-cbz'],
      ['torrent', 'application/x-bittorrent'],
      ['json', 'application/json'],
      ['xml', 'application/xml'],
      ['html', 'text/html'],
      ['css', 'text/css'],
      ['js', 'application/javascript'],
      ['csv', 'text/csv'],
      ['xls', 'application/vnd.ms-excel'],
      ['xlsx', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'],
      ['ppt', 'application/vnd.ms-powerpoint'],
      ['pptx', 'application/vnd.openxmlformats-officedocument.presentationml.presentation'],
    ]),
    defaultMimetype: 'application/octet-stream',
  },
  audio: {
    extensions: new Set(['mp3', 'wav', 'ogg', 'flac', 'm4a', 'aac', 'wma']),
    mimetype: 'audio/mpeg',
  },
};

function getFileDetails(filePath) {
  const ext = path.extname(filePath).slice(1).toLowerCase();

  for (const [category, typeInfo] of Object.entries(FILE_TYPES)) {
    if (typeInfo.extensions.has(ext)) {
      return {
        category,
        mimetype: category === 'document'
          ? typeInfo.mimetypes.get(ext) || typeInfo.defaultMimetype
          : typeInfo.mimetype,
      };
    }
  }

  return {
    category: 'document',
    mimetype: FILE_TYPES.document.defaultMimetype,
  };
}

class DownloadQueue {
  constructor(maxConcurrent = 2) {
    this.queue = [];
    this.activeDownloads = 0;
    this.maxConcurrent = maxConcurrent;
  }

  async add(task) {
    return new Promise((resolve, reject) => {
      this.queue.push({ task, resolve, reject });
      this.processNext();
    });
  }

  async processNext() {
    if (this.activeDownloads >= this.maxConcurrent || this.queue.length === 0) {
      return;
    }

    this.activeDownloads++;
    const { task, resolve, reject } = this.queue.shift();

    try {
      const result = await task();
      resolve(result);
    } catch (error) {
      reject(error);
    } finally {
      this.activeDownloads--;
      this.processNext();
    }
  }
}

class MediaDownloader {
  constructor() {
    this.config = {
      tempDir: process.env.TEMP_DOWNLOAD_DIR || path.join(process.cwd(), 'tmp'),
      maxFileSize: (parseInt(process.env.MAX_UPLOAD, 10) * 1048576) || 1500000000,
      ytDlpPath: path.join(process.cwd(), 'media', 'bin'),
      maxConcurrent: parseInt(process.env.MAXSOLICITUD, 10) || 2,
      playlistLimit: parseInt(process.env.PLAYLIST_LIMIT, 10) || 10 // limite playlist mp3
    };

    this.downloadQueue = new DownloadQueue(this.config.maxConcurrent);

    this.ytDlpBinaries = new Map([
      ['win32-x64', 'yt-dlp.exe'],
      ['win32-ia32', 'yt-dlp_x86.exe'],
      ['darwin', 'yt-dlp_macos'],
      ['linux-x64', 'yt-dlp_linux'],
      ['linux-arm64', 'yt-dlp_linux_aarch64'],
      ['linux-arm', 'yt-dlp_linux_armv7l'],
      ['default', 'yt-dlp'],
    ]);
  }

  generateSafeFileName(originalName) {
    const ext = path.extname(originalName);
    const timestamp = Date.now();
    return `download_${timestamp}${ext}`;
  }

  async safeExecute(command, silentError = false) {
    try {
      const result = await exec(command);
      return result;
    } catch (error) {
      if (!silentError) {
        console.error(`Execution failed: ${error.message}`);
      }
      throw new Error('Execution failed');
    }
  }

  async isYtDlpAvailable() {
    try {
      await exec('yt-dlp --version', { stdio: 'ignore' });
      return true;
    } catch {
      return false;
    }
  }

  detectYtDlpBinaryName() {
    const platform = os.platform();
    const arch = os.arch();
    const key = `${platform}-${arch}`;
    return this.ytDlpBinaries.get(key) || this.ytDlpBinaries.get('default');
  }

  async ensureDirectories() {
    await Promise.all([
      fs.mkdir(this.config.tempDir, { recursive: true }),
      fs.mkdir(this.config.ytDlpPath, { recursive: true }),
    ]);
  }

  async detectYtDlpBinary(message) {
    if (await this.isYtDlpAvailable()) {
      return 'nice -n 7 yt-dlp';
    }

    const fileName = this.detectYtDlpBinaryName();
    const filePath = path.join(this.config.ytDlpPath, fileName);
    
    try {
      await fs.access(filePath);
      return `nice -n 7 ${filePath}`;
    } catch {
      return message ? await this.downloadYtDlp(message) : null;
    }
  }

  async downloadYtDlp(message) {
    await this.ensureDirectories();
    const fileName = this.detectYtDlpBinaryName();
    const downloadUrl = `https://github.com/yt-dlp/yt-dlp/releases/latest/download/${fileName}`;
    const filePath = path.join(this.config.ytDlpPath, fileName);

    try {
      await this.safeExecute(`curl -L -o "${filePath}" "${downloadUrl}"`);
      
      if (os.platform() !== 'win32') {
        await fs.chmod(filePath, '755');
      }
      
      return `nice -n 7 ${filePath}`;
    } catch (error) {
      // node-fetch fallback
      const fetch = (await import('node-fetch')).default;
      
      const response = await fetch(downloadUrl);
      if (!response.ok) throw new Error(`Download failed: ${response.statusText}`);

      const buffer = Buffer.from(await response.arrayBuffer());
      await fs.writeFile(filePath, buffer);

      if (os.platform() !== 'win32') {
        await fs.chmod(filePath, '755');
      }

      return `nice -n 7 ${filePath}`;
    }
  }

  async processDownloadedFile(message, filePath, originalFileName, outputDir) {
    const { mimetype, category } = getFileDetails(filePath);
    const fileBuffer = await fs.readFile(filePath);
    const safeFileName = this.generateSafeFileName(originalFileName);

    await message.send(
      fileBuffer,
      { fileName: safeFileName, mimetype, quoted: message.data },
      category
    );

    await fs.unlink(filePath).catch(() => {});
  }

  async downloadWithYtDlp(message, urls, options = '', enablePlaylist = false) {
    return this.downloadQueue.add(async () => {
      const ytDlpPath = await this.detectYtDlpBinary(message);
      const sessionId = `yt-dlp_${Date.now()}`;
      const outputDir = path.join(this.config.tempDir, sessionId);

      await this.ensureDirectories();
      await fs.mkdir(outputDir, { recursive: true });

      for (const url of urls) {
        const safePattern = path.join(outputDir, '%(title)s.%(ext)s');
        
        let playlistFlag = enablePlaylist ? '--yes-playlist' : '--no-playlist';
        let playlistItemsFlag = '';
        
        if (enablePlaylist) {
          playlistItemsFlag = `--playlist-items 1:${this.config.playlistLimit}`;
        }
        
        const baseCommand = `${ytDlpPath} --max-filesize ${this.config.maxFileSize} -o "${safePattern}" ${playlistFlag} ${playlistItemsFlag} "${url}"`;
        const downloadCommand = `${baseCommand} ${options}`;

        let downloadSuccess = false;
        
        try {
          await this.safeExecute(downloadCommand, true);
          downloadSuccess = true;
        } catch (error) {
          console.error(`Failed with full options: ${error.message}`);
          
          try {
            await this.safeExecute(baseCommand, true);
            downloadSuccess = true;
          } catch (fallbackError) {
            console.error(`Fallback also failed: ${fallbackError.message}`);
            
            try {
              await this.updateYtDlp(message, true);
            } catch (updateError) {
              console.error(`Update failed: ${updateError.message}`);
            }
          }
        }

        if (downloadSuccess) {
          try {
            const files = await fs.readdir(outputDir);
            for (const file of files) {
              try {
                await this.processDownloadedFile(
                  message,
                  path.join(outputDir, file),
                  file,
                  outputDir
                );
              } catch (processError) {
                console.error(`Failed to process file ${file}: ${processError.message}`);
              }
            }
          } catch (error) {
            console.error(`Failed to read output directory: ${error.message}`);
          }
        }
      }

      await fs.rm(outputDir, { recursive: true, force: true }).catch(() => {});
    });
  }

  async updateYtDlp(message, silent = false) {
    try {
      if (await this.isYtDlpAvailable()) {
        await this.safeExecute('pip install -U --pre "yt-dlp[default]" ');
        if (!silent) {
          await message.send('✅ yt-dlp Updated pip', { quoted: message.data });
        }
      } else {
        const filePath = await this.downloadYtDlp(message);
        if (!silent) {
          await message.send(`✅ yt-dlp Updated Bin _${filePath}_`, { quoted: message.data });
        }
      }
    } catch (error) {
      if (!silent) {
        await message.send(`✅ yt-dlp Update attempted`, { quoted: message.data });
      }
    }
  }

  async searchAndDownload(message, searchQuery, isVideo = false) {
    return this.downloadQueue.add(async () => {
      const sessionId = `yt-dlp_${Date.now()}`;
      const outputDir = path.join(this.config.tempDir, sessionId);
      
      await this.ensureDirectories();
      await fs.mkdir(outputDir, { recursive: true });

      const safePattern = path.join(outputDir, 'download.%(ext)s');
      const ytDlpPath = await this.detectYtDlpBinary(message);
      
      const searchSources = [
        { source: 'ytsearch', name: 'YouTube' },
        ...(isVideo ? [] : [
            { source: 'scsearch', name: 'SoundCloud' },
            { source: 'nicosearch', name: 'NicoNico' }
        ])
      ];

      let success = false;

      for (const { source, name } of searchSources) {
        if (success) break;

        try {
          const baseCommand = `${ytDlpPath} --max-filesize ${this.config.maxFileSize} --playlist-items 1 -o "${safePattern}"  --ignore-errors --no-abort-on-error`;
          const videoOptions = isVideo ? '-f "bestvideo[height<=720][ext=mp4][vcodec=h264]+bestaudio[acodec=aac]/best[height<=720][vcodec=h264]/best[ext=mp4]/best" --merge-output-format mp4 ' : '-x --audio-format mp3';
          const command = `${baseCommand} ${videoOptions} "${source}10:${searchQuery}"`;
          
          try {
            await this.safeExecute(command, true);
          } catch (error) {
            await this.safeExecute(`${baseCommand} "${source}10:${searchQuery}"`, true);
          }

          const files = await fs.readdir(outputDir);
          if (files.length > 0) {
            await Promise.all(
              files.map(file => this.processDownloadedFile(
                message,
                path.join(outputDir, file),
                file,
                outputDir
              ))
            );
            success = true;
            break;
          }
        } catch (error) {
          await fs.rm(outputDir, { recursive: true, force: true }).catch(() => {});
          await fs.mkdir(outputDir, { recursive: true });
        }
      }

      if (!success) {
        console.error(`No results found for ${searchQuery}`);
      }

      await fs.rm(outputDir, { recursive: true, force: true }).catch(() => {});
    });
  }
}

const mediaDownloader = new MediaDownloader();

bot(
  {
    pattern: 'dla ?(.*)',
    fromMe: true,
    desc: 'Download All Media Web Site.',
    type: 'download',
  },
  async (message, match) => {
    const input = match.trim() || message.reply_message?.text || '';
    
    if (!input) {
      await message.send(
        '> 🎶Search and Download Song:\n`dla` <query>\n' +
        '> 🎥Search and Download Video:\n`dla vd` <query>\n' +
        '> ⬇️Download All Media: \n`dla` <url> _YT-DLP FLAGS_ \n' +
        '> 🎵Download All Audio from Playlist: \n`dla mp3` <url> \n' +
        '> 🆙Fail Download? Update YT-DLP: \n`dla update` \n' +
        '> 🌐More Info:\ngithub.com/yt-dlp/yt-dlp/blob/master/README.md#usage-and-options',
        { quoted: message.data }
      );
      return;
    }

    try {
      const args = input.match(/[^\s"]+|"([^"]*)"/g)?.map(arg => 
        arg.startsWith('"') && arg.endsWith('"') ? arg.slice(1, -1) : arg
      ) || [];

      const command = args[0];
      const remainingArgs = args.slice(1);
      const urls = remainingArgs.filter(arg => isUrl(arg));
      
      if (!urls.length && command !== 'update') {
        if (input.includes('://')) {
          const inputUrl = input.match(/(https?:\/\/[^\s]+)/)?.[1];
          if (inputUrl) {
            const options = input.replace(inputUrl, '').trim();
            await mediaDownloader.downloadWithYtDlp(message, [inputUrl], `-f "bestvideo[height<=720][ext=mp4][vcodec=h264]+bestaudio[acodec=aac]/best[height<=720][vcodec=h264]/best[ext=mp4]/best" --merge-output-format mp4 ${options}`);
            return;
          }
        }
        if (command === 'vd') {
          await mediaDownloader.searchAndDownload(message, remainingArgs.join(' '), true);
        } else {
          await mediaDownloader.searchAndDownload(message, input, false);
        }
        return;
      }

      switch (command) {
        case 'update':
          await mediaDownloader.updateYtDlp(message);
          break;

        case 'mp3':
          if (urls.length) {
            const options = remainingArgs
              .filter(arg => !isUrl(arg))
              .join(' ');
            await mediaDownloader.downloadWithYtDlp(
              message, 
              urls, 
              `-x --audio-format mp3 ${options}`,
              true
            );
          }
          break;

        default:
          const options = remainingArgs
            .filter(arg => !isUrl(arg))
            .join(' ');
          await mediaDownloader.downloadWithYtDlp(
            message, 
            urls, 
            `-f "bestvideo[height<=720][ext=mp4][vcodec=h264]+bestaudio[acodec=aac]/best[height<=720][vcodec=h264]/best[ext=mp4]/best" --merge-output-format mp4 ${options}`
          );
          break;
      }
    } catch (error) {
      console.error(`Error in dla command: ${error.message}`);
    }
  }
);

module.exports = { mediaDownloader };

I can't download YouTube shorts

I can't download YouTube shorts

What error does it show?
Because it works perfectly for me.

It showing can't read properties of mp4

…

On Tue, Dec 31, 2024, 2:06 AM laWiskaPY ***@***.***> wrote: ***@***.**** commented on this gist. ------------------------------ I can't download YouTube shorts What error does it show? Because it works perfectly for me. image.png (view on web) <https://gist.github.com/user-attachments/assets/738a2883-00bc-4085-a26c-ceac932e2d7e> — Reply to this email directly, view it on GitHub <https://gist.github.com/weskerty/ce6a4ea2c4b0a73889cae8431911734d#gistcomment-5369375> or unsubscribe <https://github.com/notifications/unsubscribe-auth/BIQPSKCTROCFWYPK3VM3LB32IHU2DBFKMF2HI4TJMJ2XIZLTSKBKK5TBNR2WLJDUOJ2WLJDOMFWWLO3UNBZGKYLEL5YGC4TUNFRWS4DBNZ2F6YLDORUXM2LUPGBKK5TBNR2WLJDHNFZXJJDOMFWWLK3UNBZGKYLEL52HS4DFVRZXKYTKMVRXIX3UPFYGLK2HNFZXIQ3PNVWWK3TUUZ2G64DJMNZZDAVEOR4XAZNEM5UXG5FFOZQWY5LFVEYTGNBSGQ4TMOJVU52HE2LHM5SXFJTDOJSWC5DF> . You are receiving this email because you commented on the thread. Triage notifications on the go with GitHub Mobile for iOS <https://apps.apple.com/app/apple-store/id1477376905?ct=notification-email&mt=8&pt=524675> or Android <https://play.google.com/store/apps/details?id=com.github.android&referrer=utm_campaign%3Dnotification-email%26utm_medium%3Demail%26utm_source%3Dgithub> .

It showing can't read properties of mp4

Does this happen with all shorts?
What are you running it on? What server? What Linux version?

I can't download YouTube shorts

What error does it show? Because it works perfectly for me.

Well, that's because you are using termux right?

Well, that's because you are using termux right?

Yes, I use Termux.
If you use an external server that doesn't support ffmpeg, there's nothing you can do, this is not a problem with the bot, the script or yt-dlp, it's a problem with your server.

https://gist.github.com/weskerty/ce6a4ea2c4b0a73889cae8431911734d?permalink_comment_id=5415523#gistcomment-5415523

I created a custom plugin that can download node modules directly to the node modules folder
Since some panel don't have interactive shell for users to install a missing module when trying to try out a new plugin.

It's a private gist but I can share the url if anyone desire

I created a custom plugin that can download node modules directly to the node modules folder Since some panel don't have interactive shell for users to install a missing module when trying to try out a new plugin.

What plugin did you make that requires other node modules?
There is already an alternative to this.
The Linux plugin or CMD (my gist) where you just have to do
.cmd npm install <module> --force --no-save and that's it.

I saw your Gist. Are you talking about a bot external to Levanter?

here is the plugin, when you asked what type of plugin I made that would require one to need to install a custom module.
There's was a plugin I was making over 2 months ago before dropping it because of a reason.

So back to the reason, my plugin(an interactive multi users RPG game with economy where every players that will be connected and be able to interact with each other) required mongoose module which is not pre-installed.

I created a custom plugin that can download node modules directly to the node modules folder Since some panel don't have interactive shell for users to install a missing module when trying to try out a new plugin.

What plugin did you make that requires other node modules? There is already an alternative to this. The Linux plugin or CMD (my gist) where you just have to do .cmd npm install <module> --force --no-save and that's it.

I saw your Gist. Are you talking about a bot external to Levanter?

@weskerty
So i made the plugin.
And tbh, I didn't know that this existed before I created mine fr 🙂.

And I have a question
@weskerty
Is it possible to achieve the goal of making multiple users have a unique account for the game when invoking a command from the bot.

My reason for asking this is that levanter is a one user per bot policy and every available commands are for the users only, but commands can be made available with the zushi command.

But even when using zushi, the bot will react as if you are the one invoking a command in a group when others use the set commands.

Does WhatsApp have a unique way to recognize different users in a group so that the bot will be able to know that Participant A is different from Participant B in a group

Does WhatsApp have a unique way to recognize different users in a group so that the bot will be able to know that Participant A is different from Participant B in a group

Yes. The Jids.
But I haven't seen how it works in Levante yet.
Do you want everyone to log in to YT or other services?