Download series of files in parallel with progress

a-demo.md

Steps

• Download these files to a folder
• Go to http://downloader.soundcloud.ruud.ninja/
• Authorise via SoundCloud login
• Paste gettracklinks.browser.js into the browser console
• Paste the clipboard into alltracks.json
• Run node getallfiles.js

But there's a quicker way!

• Hit tab until you reach the first link
• Hit enter to show the save file dialog, and enter again to save it
• Repeat: tab, enter, enter until all files downloaded

So this was just for fun to learn how to buffer the progress of multiple downloads in the terminal!

Bugs

1. Tracks with the same display name on SoundCloud will overwrite each other. To fix: extract filename= from the content-disposition header and append to filename to make it unique.

alltracks.json

[
  {
    "name": "Improv_1",
    "href": "https://api.soundcloud.com/tracks/313794512/download?oauth_token=xxxxxx"
  },
  {
    "name": "Improv_2",
    "href": "https://api.soundcloud.com/tracks/313794510/download?oauth_token=xxxxxx"
  },
  {
    "name": "Improv_3",
    "href": "https://api.soundcloud.com/tracks/313794508/download?oauth_token=xxxxxx"
  },
  {
    "name": "Improv_4",
    "href": "https://api.soundcloud.com/tracks/313794506/download?oauth_token=xxxxxx"
  },
  {
    "name": "Improv_5",
    "href": "https://api.soundcloud.com/tracks/313794504/download?oauth_token=xxxxxx"
  },
  {
    "name": "Improv_6",
    "href": "https://api.soundcloud.com/tracks/313794497/download?oauth_token=xxxxxx"
  },
  {
    "name": "Improv_7",
    "href": "https://api.soundcloud.com/tracks/313794496/download?oauth_token=xxxxxx"
  },
  {
    "name": "Lowwood Improvisation",
    "href": "https://api.soundcloud.com/tracks/255446481/download?oauth_token=xxxxxx"
  },
  {
    "name": "Catweazle Interrupted Improv",
    "href": "https://api.soundcloud.com/tracks/245492111/download?oauth_token=xxxxxx"
  },
  {
    "name": "Catweazle Interrupted Improv",
    "href": "https://api.soundcloud.com/tracks/245489053/download?oauth_token=xxxxxx"
  },
  {
    "name": "Politics In The Afternoon - Instrumental 2",
    "href": "https://api.soundcloud.com/tracks/243561184/download?oauth_token=xxxxxx"
  },
  {
    "name": "Politics In The Afternoon - Instrumental",
    "href": "https://api.soundcloud.com/tracks/243542853/download?oauth_token=xxxxxx"
  },
  {
    "name": "Catweazle 2",
    "href": "https://api.soundcloud.com/tracks/184069376/download?oauth_token=xxxxxx"
  },
  {
    "name": "Cat Weazle 1",
    "href": "https://api.soundcloud.com/tracks/181105187/download?oauth_token=xxxxxx"
  },
  {
    "name": "Face The Crowd",
    "href": "https://api.soundcloud.com/tracks/171818909/download?oauth_token=xxxxxx"
  },
  {
    "name": "Evening Bells",
    "href": "https://api.soundcloud.com/tracks/168357146/download?oauth_token=xxxxxx"
  },
  {
    "name": "That's alright, don't worry",
    "href": "https://api.soundcloud.com/tracks/168356731/download?oauth_token=xxxxxx"
  },
  {
    "name": "Tuesday 08:11 PM",
    "href": "https://api.soundcloud.com/tracks/165933824/download?oauth_token=xxxxxx"
  },
  {
    "name": "Patatap 1",
    "href": "https://api.soundcloud.com/tracks/162296286/download?oauth_token=xxxxxx"
  },
  {
    "name": "Sunday 08:36 PM",
    "href": "https://api.soundcloud.com/tracks/159542139/download?oauth_token=xxxxxx"
  },
  {
    "name": "Sunday Improv #1",
    "href": "https://api.soundcloud.com/tracks/63395930/download?oauth_token=xxxxxx"
  }
]

getallfiles.js

const fs = require('fs');
const fetch = require('node-fetch');

const files = require('./allfiles.json');

const readline = require('readline');

const fileLogs = [];
const wheel = ['|', '/', '-', '\\'];
const wheelLength = wheel.length;
let hasLogged = false;
let lastLogLength = 0;
const updateLogs = function () {
    readline.moveCursor(process.stdout, 0, -lastLogLength);
    readline.clearLine(process.stdout, 0);
    const longestFileName = fileLogs.reduce((length, file) => {
        return Math.max(length, file.filename.length);
    }, 0);
    const text = fileLogs
        .map(file => {
            const status = file.done ? 'done' : wheel[file.progress % wheelLength];
            const paddingLength = longestFileName - file.filename.length;
            const padding = Array(paddingLength > 0 ? paddingLength : 0).join('.');
            return `=> ${file.filename}${padding}...${status}`;
        });
    lastLogLength = text.length;
    process.stdout.write(text.join('\n') + '\n');
};

let start = Date.now();
let INTERVAL_MS = 100;
const updateThrottle = function () {
    const now = Date.now();
    if (now - start > INTERVAL_MS) {
        start = now;
        updateLogs();
    }
};

files.forEach(file => {
    fetch(file.href)
        .then(res => {
            const filetype = res.headers.get('x-amz-meta-file-type');
            const filename = `${file.name}.${filetype}`;
            const dest = `tracks/${filename}`;
            const fileLog = {filename, progress: 0, done: false};
            fileLogs.push(fileLog);
            const destStream = fs.createWriteStream(dest);
            const pipe = res.body.pipe(destStream);
            res.body
                .on('data', () => {
                    fileLog.progress += 1;
                    updateThrottle();
                })
                .on('end', () => {
                    fileLog.done = true;
                    updateThrottle();
                });
        });
});

gettracklinks.browser.js

// http://downloader.soundcloud.ruud.ninja/
var alltracks = $('#tracks a').map(function () {
  var $el = $(this);
  return {
      name: $el.text(),
      href: $el.attr('href')
  }
});
// Put into clipboard.
copy(alltracks.get());
// Save into allfiles.json.

package.json

{
  "name": "soundcloud",
  "version": "1.0.0",
  "description": "",
  "main": "getallfiles.js",
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1"
  },
  "keywords": [],
  "author": "Henry Blyth <[email protected]> (http://henryblyth.com/)",
  "license": "ISC",
  "dependencies": {
    "node-fetch": "^1.7.2"
  }
}