CyTube metadata updater, processing external datasets into custom JSON is the tricky part!

_updater.js

'use strict';

const _ = require('lodash');
const Promise = require('bluebird');
const data = require('./format.json')
const opts = require('./opts');

const config = require('yaml').parse(require('fs').readFileSync('config.yaml', 'utf-8'));

const knex = require('knex')({
  client: 'mysql',
  connection: config.mysql  
});

function getById(processedRow) {
  return knex('channels')
    .where('name', processedRow.id)
    .first()
    .then(channel => {
      return _.merge({}, processedRow, {
        id: channel.id,
        name: processedRow.id
      }) 
    });
};

function parseGenres(rows) {
  return _.map(rows, row => _.merge({}, row, { genres: row.genres.split(',') }));
};

function createGroupInserts(rows) {
  return _.map(rows, row => knex('channel_data')
    .insert({
      channel_id: row.id,
      key: 'groups',
      value: JSON.stringify(row.genres)
    }));
};

function fetchOpts(row) {
  return knex('channel_data')
    .where('channel_id', row.id)
    .where('key', 'opts')
    .first()
    .then((data) => {
      return _.merge({}, row, { opts: data.value });
    });
};

function mergeExternalId(rows) {
  return _.map(rows, row => _.merge({}, row, {
    opts: _.merge(opts, JSON.parse(row.opts), { externalid: row.external_id })
  }));
};

function createOptsUpdates(rows) {
  return _.map(rows, row => knex('channel_data')
    .where('channel_id', row.id)
    .where('key', 'opts')
    .update({
      value: JSON.stringify(row.opts)
    }));
}

const groupInserts = Promise.all(_.map(data, getById))
  .then(parseGenres)
  .then(createGroupInserts);

const optsUpdates = Promise.all(_.map(data, getById))
  .then(rows => Promise.all(_.map(rows, fetchOpts)))
  .then(mergeExternalId)
  .then(createOptsUpdates);

Promise
  .all(groupInserts)
  .all(optsUpdates)
  .then((data) => {
    console.log('All done!');
    knex.destroy();
  });

config.yaml

mysql:
  server: '127.0.0.1'
  port: 3306
  database: 'root'
  user: 'root'
  password: ''

format.json

[
   {
      "id":"internalid",
      "external_id":"9c0b8f7e-9f27-4d82-9f0c-ba276387ba60",
      "genres":"Comedy,Drama" // comma-separated
   }
]

opts.js

'use strict';

module.exports = {
    allow_voteskip: true,      // Allow users to voteskip
    voteskip_ratio: 0.5,       // Ratio of skip votes:non-afk users needed to skip the video
    afk_timeout: 600,          // Number of seconds before a user is automatically marked afk
    pagetitle: "", // Title of the browser tab
    maxlength: 0,              // Maximum length (in seconds) of a video queued
    externalcss: "",           // Link to external stylesheet
    externaljs: "",            // Link to external script
    externalid: "",            // An external identifier
    chat_antiflood: false,     // Throttle chat messages
    chat_antiflood_params: {
        burst: 4,              // Number of messages to allow with no throttling
        sustained: 1,          // Throttle rate (messages/second)
        cooldown: 4            // Number of seconds with no messages before burst is reset
    },
    show_public: false,        // List the channel on the index page
    enable_link_regex: true,   // Use the built-in link filter
    password: false,           // Channel password (false -> no password required for entry)
    allow_dupes: false,        // Allow duplicate videos on the playlist
    torbanned: false,          // Block connections from Tor exit nodes
    block_anonymous_users: false, //Only allow connections from registered users.
    allow_ascii_control: false,// Allow ASCII control characters (\x00-\x1f)
    playlist_max_per_user: 0,  // Maximum number of playlist items per user
    new_user_chat_delay: 0,      // Minimum account/IP age to chat
    new_user_chat_link_delay: 0, // Minimum account/IP age to post links
    playlist_max_duration_per_user: 0 // Maximum total playlist time per user
};

package.json

{
  "name": "updater",
  "version": "1.0.0",
  "description": "",
  "main": "index.js",
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1"
  },
  "author": "",
  "license": "ISC",
  "dependencies": {
    "bluebird": "^3.7.2",
    "knex": "^1.0.4",
    "lodash": "^4.17.21",
    "mysql": "^2.18.1",
    "yaml": "^1.10.2"
  }
}