Script to fetch all companies and their information from https://github.com/mesaugat/tech-companies-in-nepal and push it to Airtable.

tcin-airtable.js

/**
 * Script to fetch all companies and their information from https://github.com/mesaugat/tech-companies-in-nepal and push it to Airtable.
 *
 * Some inspiration from https://github.com/poteto/hww-api
 *
 * I know the script doesn't look good but hey, it works!
 *
 * Requires node > 7.6
 * Run: node --harmony <filename>
 */

const async = require('async');
const fetch = require('node-fetch');
const Airtable = require('airtable');

const RAW_README_URL = 'https://raw.githubusercontent.com/mesaugat/tech-companies-in-nepal/master/README.md';

const base = new Airtable({apiKey: 'API_KEY'}).base('AIRTABLE_BASE');
const companiesTable = base('Companies');
const locationsTable = base('Locations');

/**
 * Find location.
 *
 * @param  {String} name
 * @return {Promise}
 */
function findLocation(name) {
  return new Promise((resolve, reject) => {
    locationsTable.select({
      filterByFormula: `{Name} = "${name}"`
    })
    .eachPage(resolve, reject);
  });
}

/**
 * Create a location.
 *
 * @param  {String} name
 * @return {Promise}
 */
function createLocation(name) {
  return new Promise((resolve, reject) => {
    locationsTable.create({
      'Name': name
    }, (err, record) => {
      if (err) return reject(err);

      console.info(`Location created: ${record.getId()} | ${record.get('Name')}`);
      return resolve(record.getId());
    })
  });
}

/**
 * Parse and get companies list from README.
 *
 * @return {array}
 */
async function getCompanyList() {
  const response = await fetch(RAW_README_URL);
  const text = await response.text();

  const matches = text.match(/\[.+\]\(.+\) \| .+ \| .+/g);

  const companies = matches.slice(1).map(match => {
    let details = match.split('|');

    let [,name] = details[0].trim().match(/\[(.*)\]/);
    let [,website] = details[0].trim().match(/\((http.*)\)/);
    let locations = details[1].trim().split(', ');
    let description = details[2].trim();

    return {name, website, locations, description};
  });

  return companies;
}

/**
 * Get all companies and add the information to Airtable.
 */
getCompanyList().then(companies => {
  async.eachSeries(companies, (company, callback) => {
    // Get locations of a company and add it to locations table
    Promise.all(
      company.locations.map(location => {
        // Find or create location
        return findLocation(location)
          .then(records => {
            if (records.length) {
              return records.shift().getId();
            }

            return createLocation(location);
          });
      })
    ).then(locationIds => {
      companiesTable.create({
        'Name': company.name,
        'Website': company.website,
        'Location': locationIds,
        'Description': company.description
      },
      (err, record) => {
        if (err) {
          console.error(err);

          return;
        }

        console.info(`Company created: ${record.getId()} | ${record.get('Company')}`);
        callback(record.get('Name'));
      });
    });
  }, err => {
    console.error(err);
  });
});