This python script generates a file list for aria2 for downloading all the packages from all the repos using all the available mirrors

archlinux-mirror.py

#!/usr/bin/python3

import os
import shutil
import tempfile
import subprocess
from pprint import pprint
from Reflector import MirrorStatus

ms = MirrorStatus()
mirrors = ms.filter(ms.get_mirrors(), protocols="http, https, ftp")
mirrors = [mirror["url"] for mirror in mirrors]
print("we have {} mirrors".format(len(mirrors)))

#result = subprocess.run(["pacman", "-Sl"], stdout=subprocess.PIPE)
#lines = (line.split(" ") for line in result.stdout.decode().split("\n"))
#packages = [{"repo": line[0], "pacage":line[1], "version":line[2]} for line in lines if len(line) > 2 and line[0] in ms.REPOSITORIES]



tmpdir = tempfile.mkdtemp()
print(tmpdir)


REPO_DB_FILES = ('.db',
                 '.db.tar.gz',
                 '.db.tar.bz2',
                 '.db.tar.xz',
                 '.db.tar.Z',
                 '.db.tar')

REPOSITORIES = (
    'community',
    'community-staging',
    'community-testing',
    'core',
    'extra',
    'gnome-unstable',
    'kde-unstable',
    'multilib',
    'multilib-testing',
    'staging',
    'testing',
)
ARCHITECTURES = ['i686', 'x86_64']

db_urls = []
for repo in REPOSITORIES:
    for arch in ARCHITECTURES:
        db_urls.append("{}{}/os/{}/{}\n  dir={}/{}/os/{}".format(mirrors[10], repo, arch, repo+".db.tar.gz", tmpdir, repo, arch))

urllist = "\n".join(db_urls)
cmd = ["aria2c", "-d", tmpdir, "-Z", "-j10", "-c", "-i", "-"]
subprocess.run(cmd, input=urllist.encode())
#print(urllist)

packages = []

for repo in REPOSITORIES:
    for arch in ARCHITECTURES:
        temp_repo = tempfile.mkdtemp()
        subprocess.run(["bsdtar", "-xf", os.path.join(tmpdir, repo, "os", arch, repo+".db.tar.gz"), "-C", temp_repo])
        pkgs = os.listdir(temp_repo)
        for pkg in pkgs:
            with open(os.path.join(temp_repo, pkg, "desc")) as f:
                lines = f.readlines()
                for n, line in enumerate(lines):
                    if line.strip() == "%ARCH%":
                        pkg_arch = lines[n+1].strip()
                        break
            packages.append({"repo":repo, "arch": arch, "pkg": pkg, "pkg_arch": pkg_arch})
        shutil.rmtree(temp_repo)

shutil.rmtree(tmpdir)

print("and {} packages".format(len(packages)))

print("writing dl-list.txt")
with open("dl-list.txt", "w") as f:
    for repo in REPOSITORIES:
        for arch in ARCHITECTURES:
            for repo_file in (".db", ".db.tar.gz", ".abs.tar.gz", ".files", ".files.tar.gz"):        
                f.write("{}{}/os/{}/{}\n  dir={}/os/{}\n".format(mirrors[10], repo, arch, repo+repo_file,  repo, arch))
    for pkg in packages:
        repo, arch, pkg, pkg_arch = pkg["repo"], pkg["arch"], pkg["pkg"], pkg["pkg_arch"]
        pkg_line = "\t".join(["{}{}/os/{}/{}".format(mirror, repo, arch, pkg+"-"+pkg_arch+".pkg.tar.xz" ) for mirror in mirrors])
        f.write(pkg_line)
        opts = "\n  dir={}/os/{}\n".format(repo, arch)
        f.write(opts)
        pkg_line = "\t".join(["{}{}/os/{}/{}".format(mirror, repo, arch, pkg+"-"+pkg_arch+".pkg.tar.xz.sig" ) for mirror in mirrors])
        f.write(pkg_line)
        opts = "\n  dir={}/os/{}\n".format(repo, arch)
        f.write(opts)
        print(".", end="", flush=True)
    print("Done")