Crawl Gentoo mirror for .*CONTENTS.gz files and extract all the unique architecture quads, then use crossdev to build a full suite of cross compilers.

xcompiler.py

#!/usr/bin/env python3

import os
import re
import sys
import copy
import gzip
import urllib.request as request
from html.parser import HTMLParser
from urllib.parse import urlparse, urljoin

class Throbber:

    def __init__(self):
        self.data = ['-', '\\', '|', '/']
        self.idx = 0

    def __repr__(self):
        return self.__str__()
        
    def __str__(self):
        retv = self.data[self.idx]
        if self.idx+1 < len(self.data):
            self.idx += 1
        else:
            self.idx = 0
        return retv

class PageParser(HTMLParser):

    def __init__(self):
        HTMLParser.__init__(self)
        self.files = []
        self.paths = []

    def _filter(self, itm):
        if itm[0] == 'href' and itm[1].find('://') == -1 and not itm[1].startswith('?'):
            if itm[1].endswith('/'):
                self.paths.append(itm[1])
            else:
                self.files.append(itm[1])
        
    def handle_starttag(self, t, a):
        if t == 'a':
            for attr in a:
                self._filter(attr)

if __name__ == '__main__' and len(sys.argv) > 1:
    try:
        quads = []
        throb = Throbber()
        urls = [i for i in sys.argv[1:]]
        patt = re.compile('^.*\\./usr/bin/gcc.*[a-zA-Z0-9_\\-]*-gcc$')
        sys.stdout.write('Finding quads: ')
        sys.stdout.flush()
        while len(urls) > 0:
            n = urls.pop(0)
            sys.stdout.write(f'\rFinding quads: {throb} {len(quads)}')
            sys.stdout.flush()
            obj = PageParser()
            req = request.urlopen(n)
            obj.feed(req.read().decode('utf-8'))

            for fn in obj.files:
                fu = urljoin(n, fn)
                if fn.endswith('CONTENTS.gz') and fu.find('current-stage3') != -1 and fu.find('-openrc/') != -1:
                    tgt = urljoin(n, fn)
                    tgr = request.urlopen(tgt)
                    tgd = gzip.decompress(tgr.read()).decode('utf-8').split('\n')
                    for line in tgd:
                        if patt.match(line.strip()):
                            res = '-'.join(line.strip().split()[-1].split('/')[-1].split('-')[:-1])
                            if res not in quads:
                                quads.append(res)

            for pu in obj.paths:
                uri = urlparse(urljoin(n, pu)).geturl()
                if len(uri) > len(n) and uri != pu and uri.find('binpackages') == -1:
                    urls.append(uri)

        print(f'\rFinding quads: {throb} {len(quads)} - done')

        cmdln = 'crossdev --env "MAKEOPTS=\'-j36\'" -P \'-bk\' -t'
        quads.sort()
        for quad in quads:
            os.system(f'{cmdln} {quad}')
    except KeyboardInterrupt:
        sys.exit(1)