Extract Wikipedia dumps

extract.sh

#!/bin/bash

# First download the page listing all the dumps, then extract, with some brittle xpath magic the list of dump files and curl them locally.
curl -sL https://dumps.wikimedia.org/commonswiki/20210420/ | \
  xidel --xpath '//li[./span[contains(@class, "title") and ./big/b/text() = "Articles, templates, media/file descriptions, and primary meta-pages."]]/ul/li/a/@href' - | \
  xargs -n 1 -i curl --progress-bar -OL -C - https://dumps.wikimedia.org/{}