aoe4 leaderboard scraper

index.js

const fs = require('fs');
const axios = require('axios');
const jsdom = require('jsdom');
const { JSDOM } = jsdom;

const TABLE_SELECTOR = 'table.table-auto'
const LINK_NEXT_SELECTOR = 'a[rel="next"]'

let page = 1
let data = []

main()

async function main() {
  while (true) {
    const url = `https://aoe4world.com/leaderboard/rm_solo?page=${page}`
    await axios
      .get(url)
      .then(function (response) {
        const dom = new JSDOM(response.data);
        const table = dom.window.document.querySelector(TABLE_SELECTOR);
        const rows = table.querySelectorAll('tbody tr')
        rows.forEach(row => {
          const cells = [...row.querySelectorAll('td')].map(e => e.textContent.trim())
          const [position, name, elo, winrate, games, lastGame, social] = cells
          data.push({
            position,
            name,
            elo: parseInt(elo),
            winrate: parseFloat(winrate),
            games: parseInt(games),
            lastGame,
            social
          })
        });

        fs.writeFileSync('data.json', JSON.stringify(data))

        const link = dom.window.document.querySelector(LINK_NEXT_SELECTOR);
        if (!link) {
          throw new Error('finished')
        }
        page = parseInt(link.href.split('=').at(-1))
      });
  }
}