atmel pack scraping

atmel.js

const r2 = require('r2')
const $ = require('cheerio')
const unzipper = require('unzipper')
const fs = require('fs')
const BASE = "http://packs.download.atmel.com/"
const preamble = `
from collections import namedtuple, defaultdict


__all__ = ["devices", "devices_by_signature"]


AVRDevice = namedtuple("AVRDevice",
    ("name", "signature",
     "calibration_size", "fuses_size",
     "program_size", "program_page",
     "eeprom_size",  "eeprom_page"))

undefined = None # heh

devices = [

`
const postamble = `

]

devices_by_signature = defaultdict(lambda: None,
    ((device.signature, device) for device in devices))
`
async function packs() {
    const stuff = await r2(BASE).response
    const html = await stuff.text()
    return Array.from($.load(html)('.panel-heading > button')).map(x => $(x).attr('data-link'))
}
function toStr(stream, cb) {
    var string = ''
    stream.on('readable',function(buffer){
        let p
        while(p = stream.read()) {
            string += p.toString()
        }
    });
    
    stream.on('end',function(){
        cb(string)
    });
}
async function unpack(packname, to) {
    const url = `${BASE}${packname}`
    console.log("getting", url)
    const stuff = await r2(url).response
    await new Promise((resolve, reject) => {
        stuff.body.pipe(unzipper.Parse())
            .on('entry', entry => {
                if (entry.path.startsWith("atdf") && entry.type == "File") {
                    console.log("processing", entry.path)
                    toStr(entry, str => {
                        const e = $.load(str)
                        const devs = e('device')
                        for(const dev of Array.from(devs)) {
                            to.write(topy($(dev)))
                        }
                    })
                } else {
                    entry.autodrain()
                }
            })
            .on('error', reject)
            .on('finish', resolve)
    })
}

async function yes() {
    const packlist = await packs()
    output = fs.createWriteStream('avr.py')
    output.write(preamble)
    for(const pack of packlist) {
        output.write(`    # extracted from ${BASE}${pack}:\n`)
        await unpack(pack, output)
        output.write('\n')
    }
    output.write(postamble)
    output.close()
    console.log('done')
}
function topy(dev) {
    const props = {}
    for (const prop of Array.from(dev.find('property'))) {
        props[$(prop).attr('name')] = $(prop).attr('value')
    }
    function m(names) {
        names = Array.isArray(names) ? names : [names]
        let x
        for(const name of names) {
            x = dev.find(`memory-segment[name=${name}]`)
            if (x.length) break
        }
        function fu(q) { return q === undefined ? "None" : q }
        return ({
            size: fu(x.attr('size')),
            page: fu(x.attr('pagesize'))
        })
    }
    return `
    AVRDevice("${dev.attr('name')}",
        signature=(${props.SIGNATURE0}, ${props.SIGNATURE1}, ${props.SIGNATURE2}),
        calibration_size=${m('OSCCAL').size}, fuses_size=${m('FUSES').size},
        program_size=${m(['FLASH','PROGMEM']).size}, program_page=${m(['FLASH','PROGMEM']).page},
        eeprom_size=${m('EEPROM').size}, eeprom_page=${m('EEPROM').page}),
`
}
module.exports = yes
if (!module.parent) {
    yes()
}