Arcanist Replay Downloader

index.js

import fs from 'fs';
import https from 'https';

const cutOffDate = '2024-07-15';

const dictionaryFilePath = './replays.json';
const replayFilesOutPath = './replays';

const discordUrl = 'https://discord.com/api/v9/channels/760349009192943656/messages?limit=100';

async function get100Messages(before) {
    while (true) {
        try {
            const url = before ? `${discordUrl}&before=${before}` : discordUrl;

            const response = await fetch(url, {
                'headers': {'authorization': ''},
                'body': null,
                'method': 'GET'
            });
            const messages = await response.json();
            return messages.map(({content, timestamp, id, attachments}) => ({
                content,
                timestamp,
                id,
                url: attachments?.[0]?.url,
                fileName: attachments?.[0]?.filename
            }));
        } catch (e) {
            console.log(e);
            await new Promise((resolve) => setTimeout(resolve, 60 * 1000));
        }
    }
}

async function downloadReplay(url) {
    return new Promise((resolve, reject) => {
        https.get(url, (response) => {
            if (response.statusCode !== 200) {
                reject(new Error(`Request failed with status code ${response.statusCode}`));
                return;
            }

            const chunks = [];
            response.on('data', (chunk) => {
                chunks.push(chunk);
            });
            response.on('end', () => {
                resolve(Buffer.concat(chunks));
            });
            response.on('error', (error) => {
                reject(new Error(`Error during response: ${error.message}`));
            });
        }).on('error', (error) => {
            reject(new Error(`Request error: ${error.message}`));
        });
    });
}

const replayDictionary = fs.existsSync(dictionaryFilePath) ? JSON.parse(await fs.promises.readFile(dictionaryFilePath, 'utf-8')) : {};
console.log(`${new Date().toISOString()}: ${dictionaryFilePath}: replay dictionary loaded with ${Object.keys(replayDictionary).length} entries`);

let before = null;
while (true) {
    const discordMessages = await get100Messages(before);
    console.log(`${new Date().toISOString()}: fetched 100 Discord messages, oldest timestamp is ${discordMessages[discordMessages.length - 1].timestamp}`);

    const validDiscordMessages = discordMessages.filter((discordMessage) => discordMessage.timestamp >= cutOffDate);

    if (validDiscordMessages.length < 100) {
        console.log(`${new Date().toISOString()}: no remaining messages, exiting`);
        break;
    }

    console.log(`${new Date().toISOString()}: downloading replay files...`);

    let amountOfEntriesAdded = 0;
    let amountOfEntriesSkipped = 0;
    let amountOfReplayFilesDownloaded = 0;

    if (!fs.existsSync(replayFilesOutPath)) {
        fs.mkdirSync(replayFilesOutPath, {recursive: true});
    }

    const downloadPromises = validDiscordMessages.map(async ({id, url, ...discordMessage}) => {
        if (replayDictionary[id] === undefined) {
            replayDictionary[id] = discordMessage;
            amountOfEntriesAdded += 1;
        } else {
            amountOfEntriesSkipped += 1;
        }

        const filePath = `${replayFilesOutPath}/${id}`;
        if (!fs.existsSync(filePath)) {
            const buffer = await downloadReplay(url);
            await fs.promises.writeFile(filePath, buffer);
            amountOfReplayFilesDownloaded += 1;
        }
    });

    await Promise.allSettled(downloadPromises);

    console.log(`${new Date().toISOString()}: ${amountOfReplayFilesDownloaded} replay files downloaded to ${replayFilesOutPath}/`);

    const oldestDiscordMessageFetched = validDiscordMessages[validDiscordMessages.length - 1];
    before = oldestDiscordMessageFetched.id;

    await fs.promises.writeFile(dictionaryFilePath, JSON.stringify(replayDictionary));
    console.log(`${new Date().toISOString()}: ${dictionaryFilePath}: ${amountOfEntriesAdded} new entries`);
}