scrapes loan data from occ

scrape-occ.js

const fetch = require('node-fetch');
const fs = require('fs');

const startDate = new Date(2021, 0);
const endDate = new Date();
const msPerDay = 86400000;
const fetchDelayMS = 1000;

function pad(s, d) {
    while (s.length < d) {
        s = '0' + s;
    }
    return s;
}

function scrapeURL (date) {
    const day = date.getDate();
    const month = date.getMonth() + 1;
    const year = date.getFullYear();
    return `https://marketdata.theocc.com/stock-loan-bal-by-security?dailyDate=${pad(month + '', 2)}/${pad(day + '', 2)}/${pad(year + '', 2)}&format=csv`
}

function csvName (date) {
    const day = date.getDate();
    const month = date.getMonth() + 1;
    const year = date.getFullYear();
    return `./data/${pad(year + '', 2)}_${pad(month + '', 2)}_${pad(day + '', 2)}.csv`;
}

function sleep (ms) {
    return new Promise((resolve) => {
        setTimeout(resolve, ms);
    });
}

(async () => {
    for (let d = +startDate; d < +endDate; d += msPerDay) {
        const date = new Date(d);
        const url = scrapeURL(date);
        try {
            const res = await fetch(url);
            const report = await res.text();
            if (report.startsWith('No record')) {
                console.log('No data for: ', date);
            } else {
                await fs.promises.writeFile(csvName(date), report);
                console.log(`Fetched data for ${date}`);
            }
        } catch (err) {
            console.error(`Error on data for date ${date}`);
            console.error(err);
        }
        await sleep(fetchDelayMS);
    }
})()