Downloader for one game

downloader.js

const axios = require('axios').default
const http = require('http')
const https = require('https')
const path = require('path')
const fs = require('fs')
const Transform = require('stream').Transform
const promisePipe = require('promisepipe')
const mkdirp = require('mkdirp')
const SpaceProgress = require('./space-progress')
const spaceProgress = new SpaceProgress(process.stderr)
const { promisify } = require('util')
const readFile = promisify(fs.readFile)
const rename = promisify(fs.rename)

function sleep (ms) {
    return new Promise(res => setTimeout(res, ms))
}
class HaxeUnserialize {
    constructor (buf) {
        this.buf = buf
        this.pos = 0
        this.cache = []
        this.scache = []
    }
    readDigits () {
        let k = 0
        let s = false
        let fpos = this.pos
        while(true) {
            let c = this.buf.charCodeAt(this.pos)
            if(c != c) {
                break;
            }
            if(c == 45) {
                if(this.pos != fpos) {
                    break
                }
                s = true
                this.pos++
                continue
            }
            if(c < 48 || c > 57) {
                break
            }
            k = k * 10 + (c - 48)
            this.pos++
        }
        if(s) {
            k *= -1;
        }
        return k
    }
    unserializeObject (o) {
        while(true) {
            if(this.pos >= this.length) {
                throw new Error("Invalid object")
            }
            if(this.buf.charCodeAt(this.pos) == 103) {
                break
            }
            var k = this.unserialize()
            if(typeof(k) != "string") {
                throw Error("Invalid object key");
            }
            var v = this.unserialize()
            o[k] = v
        }
        this.pos++
    }
    unserialize () {
        let c = this.buf.charCodeAt(this.pos++)
        switch (c) {
        case 82:
            let n = this.readDigits()
            if(n < 0 || n >= this.scache.length) {
                throw new Error("Invalid string reference")
            }
            return this.scache[n]
        case 97: // a
            let a = []
            this.cache.push(a)
            while(true) {
                let c = this.buf.charCodeAt(this.pos)
                if(c == 104) { // h
                    this.pos++
                    break
                }
                if(c == 117) { // u
                    this.pos++
                    let n1 = this.readDigits()
                    a[a.length + n1 - 1] = null
                } else {
                    a.push(this.unserialize())
                }
            }
            return a
        case 102:
            return false
        case 105:
            return this.readDigits()
        case 111:
            let o2 = {}
            this.cache.push(o2)
            this.unserializeObject(o2)
            return o2
        case 116:
            return true
        case 121:
            let len1 = this.readDigits()
            if(this.buf.charCodeAt(this.pos++) != 58 || this.length - this.pos < len1) {
                throw new Error("Invalid string length")
            }
            let s2 = this.buf.substr(this.pos, len1)
            this.pos += len1
            s2 = decodeURIComponent(s2.split("+").join(" "))
            this.scache.push(s2)
            return s2
        default:
            console.log('unknown token', c, String.fromCharCode(c))
            throw new Error('unknown token:' + c)
        }
    }
}

class MultiTask {
    constructor (count) {
        this.count = count
        this.queue = []
        this.running = 0
        this.waiting = []
    }
    append (promiseFactory) {
        return new Promise((res, rej) => {
            this.queue.push({
                fac: promiseFactory,
                res,
                rej
            })
            this.runner()
        })
    }
    async runner () {
        if (this.running < this.count) {
            const { fac, res, rej } = this.queue.shift()
            try {
                this.running++
                await fac()
                res()
            } catch (e) {
                console.error('Error while run', e)
                rej(e)
            } finally {
                this.running--
            }
            this.afterRun()
        }
    }
    afterRun () {
        if (this.queue.length > 0) {
            this.runner()
        }
        if (this.queue.length === 0) {
            const waiting = this.waiting
            this.waiting = []
            for (let res of waiting) {
                res()
            }
        }
    }
    waitAll () {
        return new Promise(res => {
            if (this.queue.length === 0) {
                return res()
            }
            this.waiting.push(res)
        })
    }
}

class VirtualBar {
    constructor (pool, node, debug) {
        this.pool = pool
        this.index = node.index
        this.debug = debug

        this.terminate = () => null
    }
    tick (count, tokens) {
        this._b.tick(count, tokens)
    }
    update (...args) {
        this._b.update(...args)
    }
    set total (v) {
        this._b.total = v
    }
    get total () {
        return this._b.total
    }
    get _b () {
        return this.pool.bars[this.index].bar
    }
}
class ProgressPool {
    constructor (count) {
        this.test = 0

        this.num = 0
        this.count = count
        this.bars = []

        for (let i = 0; i < count; i++) {
            this.bars.push({
                free: true,
                index: i,
                vir: null,
                bar: spaceProgress.newBar('', {
                    total: 1,
                    width: 1,
                    clear: true
                })
            })
        }
    }
    getBar (fmt, options) {
        options = options || {}

        let cur = null
        for (let i of this.bars) {
            if (i.free) {
                cur = i
                break
            }
        }
        if (cur === null) {
            return null
        }
        cur.free = false
        const bar = cur.bar
        bar.fmt = fmt
        bar.curr = options.curr || 0
        bar.total = options.total
        bar.width = options.width
        bar.chars = {
            complete   : options.complete || '=',
            incomplete : options.incomplete || '-',
            head       : options.head || (options.complete || '=')
        }
        bar.tokens = {}
        bar.render()
        const vir = new VirtualBar(this, cur, fmt)
        cur.vir = vir
        vir.terminate = () => this.terminate(vir)
        this.num++
        return vir
    }
    transfer (from, to) {
        const fromBar = from.bar
        const toBar = to.bar
        const fromVir = from.vir
        const props = ['fmt', 'curr', 'total', 'width', 'clear', 'chars', 'callback', 'tokens']

        for (let key of props) {
            toBar[key] = fromBar[key]
        }
        fromBar.fmt = ''
        to.free = from.free
        from.free = true
        if (fromVir) fromVir.index = to.index
        to.vir = fromVir

        fromBar.render()
        toBar.render()
    }
    terminate (virtualBar) {
        let index = virtualBar.index

        for (let i = index + 1; i < this.count; i++) {
            const to = this.bars[i - 1]
            const from = this.bars[i]
            this.transfer(from, to)
        }

        const last = this.bars[this.bars.length - 1]
        last.free = true
        last.bar.fmt = ''
        last.vir = null

        for (let i = index; i < this.count; i++) {
            const node = this.bars[i]
            node.bar.render()
        }

        this.num--
        virtualBar.index = -1
    }
}
class ProgressTransform extends Transform {
    constructor (bar) {
        super()
        this.bar = bar
    }
    _transform (data, encoding, callback) {
        const bar = this.bar
        bar.tick(data.length)
        callback(null, data)
    }
}

class Downloader {
    /**
     * 
     * @param {string} base 
     * @param {string} dst 
     * @param {number} count 
     */
    constructor (base, dst, count) {
        this.base = base
        this.dst = dst
        this.progressPool = new ProgressPool(count + 1)
        this.overall = null
        this._httpAgent = new http.Agent({ keepAlive: true })
        this._httpsAgent = new https.Agent({ keepAlive: true })
        this._fileCount = 0
        this._downloadCount = 0
        this._multi = new MultiTask(count)
    }
    getSrc (name) {
        return `${this.base}${name}`
    }
    getDst (name) {
        return path.join(this.dst, name)
    }
    async readDst (name) {
        return await readFile(this.getDst(name), 'r')
    }
    download (...args) {
        return this._multi.append(() => this._download(...args))
    }
    async _download (file, dst) {
        const d = this.getDst(dst)
        const tmpName = d + '.tmp'
        mkdirp(path.dirname(d))
        const bar = this.progressPool.getBar(`       [:bar] :percent ${file}`, {
            complete: '=',
            incomplete: ' ',
            width: 30,
            total: 1
        })
        try {
            if (bar === null) {
                console.log('fuck', (new Error).stack)
                return
            }
            bar.tick(0)
            const resp = await axios({
                method: 'GET',
                url: this.getSrc(file),
                responseType: 'stream',
                headers: {
                    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
                },
                httpAgent: this._httpAgent,
                httpsAgent: this._httpsAgent
            })
            const length = parseInt(resp.headers['content-length'])
            bar.total = length + 1
            bar.tick(1)

            await promisePipe(resp.data, new ProgressTransform(bar), fs.createWriteStream(tmpName))
            await rename(tmpName, d)
            bar.tick(bar.total)
        } catch (e) {
            console.error(e.stack)
            throw new Error(`Error downloading ${file}` + e)
        } finally {
            this.downloadCount++
            bar && bar.terminate()
        }
    }
    async downloadList (list) {
        let tasks = list.map(i => this.download(i[0], i[1]))
        this.fileCount += tasks.length
        return Promise.all(tasks)
    }
    async downloadJs () {
        const html = await readFile(this.getDst('./index.html'))
        const re = /<script type="text\/javascript" src="(\.\/.*?)"><\/script>/g
        let ret = null
        let list = []
        while (ret = re.exec(html)) {
            const p = ret[1]
            list.push([path.relative('.', p), p])
        }
        await this.downloadList(list)
    }
    async start () {
        this.overall = this.progressPool.getBar('Total: [:bar] :file/:count files', {
            total: 1000,
            width: 50
        })

        await this.downloadList([
            ['index.html', './index.html'],
            ['manifest/default.json', './manifest/default.json'],
        ])
        this.downloadJs()

        const manifest = JSON.parse(await readFile(this.getDst('./manifest/default.json')))
        const assets = new HaxeUnserialize(manifest.assets).unserialize()

        let assetCount = 0
        for (let path of this.paths(assets)) {
            this.download(path, `./${path}`)
            assetCount++
        }
        this.fileCount += assetCount
        await this._multi.waitAll()
    }
    *paths (assets) {
        for (let item of assets) {
            if (item.type === 'FONT') {
                yield item.id
                yield item.id.replace(/.ttf$/, '.woff')
            } else {
                if (item.path) {
                    yield item.path
                }
                if (item.pathGroup) {
                    for (let p of item.pathGroup) {
                        yield p
                    }
                }
            }
        }
    }
    get fileCount () {
        return this._fileCount
    }
    set fileCount (v) {
        this._fileCount = v
        this._renderOverall()
    }
    get downloadCount () {
        return this._downloadCount
    }
    set downloadCount (v) {
        this._downloadCount = v
        this._renderOverall()
    }
    _renderOverall () {
        this.overall.update(this._downloadCount / this._fileCount, {
            file: this._downloadCount,
            count: this._fileCount
        })
    }
}

async function main (base) {
    const PCount = 10
    const downloader = new Downloader(base, './game/', PCount)
    await downloader.start()
}
if (process.argv.length === 3) {
    main(process.argv[2]).catch(e => console.error(e))
} else {
    console.log('usage: node downloader.js <base_url>')
}

package.json

{
  "name": "dd",
  "version": "1.0.0",
  "description": "",
  "main": "downloader.js",
  "dependencies": {
    "axios": "^0.18.0",
    "mkdirp": "^0.5.1",
    "progress": "^2.0.0",
    "promisepipe": "^3.0.0"
  },
  "devDependencies": {},
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1"
  },
  "author": "",
  "license": "MIT"
}

space-progress.js

// from https://github.com/pitaj/multi-progress/blob/master/multi-progress.js
const ProgressBar = require('progress')

class LineBar {
    constructor (index, bar) {
        this.index = index
        this.bar = bar
    }
}

class SpaceProgress {
    constructor (stream) {
        this.stream = stream || process.stderr

        this.cursor = 0
        this.bars = []
        this.terminates = 0
        if (!this.stream.isTTY) {
            this.newBar = () => ({
                tick(){},
                update(){},
                terminate(){},
                render(){},
                interrupt(){}
            })
        }
    }
    newBar (schema, options) {
        options.stream = this.stream
        let bar = new ProgressBar(schema, {
            ...options,
            renderThrottle: 0
        })
        this.bars.push(bar)
        let index = this.bars.length - 1

        this.move(index)
        this.stream.write('\n')
        this.cursor++

        bar.otick = bar.tick
        bar.oterminate = bar.terminate
        bar.oupdate = bar.update
        bar.ointerrupt = bar.interrupt
        bar.orender = bar.render
        bar.tick = (value, options) => {
            this.tick(index, value, options)
        }
        bar.terminate = () => {
            this.terminates++
            if (this.terminates === this.bars.length) {
                this.terminate()
            }
        }
        bar.update = (value, options) => {
            this.update(index, value, options)
        }
        bar.interrupt = (message) => {
            this.interrupt(index, message)
        }
        bar.render = (count, tokens) => {
            this.render(index, count, tokens)
        }
    
        return bar
    }
    terminate () {
        this.move(this.bars.length)
        this.stream.clearLine()
        this.stream.cursorTo(0)
    }
    move (index) {
        this.stream.moveCursor(0, index - this.cursor)
        this.cursor = index
    }
    moveBack () {
        this.move(this.bars.length - 1)
        this.stream.cursorTo(0)
    }
    tick (index, value, options) {
        const bar = this.bars[index]
        if (bar) {
            this.move(index)
            bar.renderThrottleTimeout = true
            bar.otick(value, options)
            bar.render()
            this.moveBack()
        }
    }
    update (index, value, options) {
        const bar = this.bars[index]
        if (bar) {
          this.move(index)
          bar.oupdate(value, options)
          this.moveBack()
        }
    }
    interrupt (index, message) {
        const bar = this.bars[index]
        if (bar) {
            this.move(index)
            bar.ointerrupt(message)
            this.moveBack()
        }
    }
    render (index, count, tokens) {
        const bar = this.bars[index]
        if (bar) {
            this.move(index)
            bar.orender(count, tokens)
            this.moveBack()
        }
    }
}
module.exports = SpaceProgress