Script for generating text & html content from API Endpoint in Live Streams

get_devo.js

import axios from "axios";
import striptags from "striptags";
import { decode } from 'html-entities';
import fs from 'fs';
import QRCode from 'qrcode';
import path from "path";
import inquirer from 'inquirer';
import DatePrompt from "inquirer-date-prompt";
import dayjs from 'dayjs';
import isYesterday from 'dayjs/plugin/isYesterday.js';
import utc from 'dayjs/plugin/utc.js';
import localizedFormat from 'dayjs/plugin/localizedFormat.js';
import { getDevoFromAPI } from "./APIs/DevoAPI.js";
import { YouTubeAPI } from "./APIs/YouTubeAPI.js";

dayjs.extend(utc);
dayjs.extend(localizedFormat)

const GAPI_KEY = '#############-#########################';

// name the needed directories
const outputDir = './outputs';
const imagesDir = `${outputDir}/images`;
const textDir = `${outputDir}/text`;
const htmlDir = `${outputDir}/html`;
const YouTubeAPIClient = new YouTubeAPI();
const qrConfig = {
    color: {
        dark: '#2279CB',  // Blue dots
        light: '#0000' // Transparent background
    },
    width: 800,
    height: 800
}

// make the needed directories
makeDirIfNotExists(outputDir);
makeDirIfNotExists(textDir);
makeDirIfNotExists(htmlDir);

/**
 * generateContent is the main function for the script.
 * Using today's date fetch the devotional data from the devo API.
 * Then store each devo attribute in text format, html,
 * download any detected images, and generate a QR code to link to ODB.
 */
async function generateContent() {
    try {
        console.time('get devo');
        inquirer.registerPrompt("date", DatePrompt);
        let selectedDate = await propmtForDate('selectedDate', 'Select Devotional Date: ');
        selectedDate = dayjs(selectedDate).format('MM-DD-YYYY');
        console.log(selectedDate);

        // create resources for linking to most recent YouTube video
        // The video will always be the most recent in the "Devotional Streams" playlist
        // the dates passed in are used as the startDate and endDate for a YouTube Analytics report
        // const endDate = dayjs();
        await generateYouTubeLinks(dayjs('2023-01-01', 'YYYY-MM-DD').format('YYYY-MM-DD'), dayjs().format('YYYY-MM-DD'));


        const devo = await getDevoFromAPI(selectedDate);

        let REQUIRES_YESTERDAY = false;

        // this handy little section checks for existing devo data from yesterday
        // if found it calls a function to move all of yesterday's devo files
        // into corresponding yesterday folders in each html, images, and text dirs
        if (fs.existsSync(`${textDir}/date.txt`)) {

            dayjs.extend(isYesterday);
            // console.log('found data from yesterday?');

            // TODO: ensure the data from yesterday corresponds
            // to the day prior to the user's requested date
            const dateFile = await fs.readFileSync(`${textDir}/date.txt`, 'utf8');
            const DATA_FROM_YESTERDAY_FOUND = dayjs.utc(+dateFile).isYesterday();
            console.log(dayjs.utc(+dateFile).format('MM-DD-YYYY'), DATA_FROM_YESTERDAY_FOUND);

            if (DATA_FROM_YESTERDAY_FOUND) {
                console.log("moving yesterdays data");
                moveYesterdaysFiles();
            } else {
                REQUIRES_YESTERDAY = true;
            }
            REQUIRES_YESTERDAY = dayjs(+dateFile).isBefore(selectedDate) && !DATA_FROM_YESTERDAY_FOUND;
        } else {
            REQUIRES_YESTERDAY = true;
        }

        await generateFiles(selectedDate, devo);

        if (REQUIRES_YESTERDAY) {
            // script hasn't run before, so get yesterday's devo data from API
            let yesterday = dayjs(selectedDate, 'MM-DD-YYYY').subtract(1, 'day').format('MM-DD-YYYY');
            console.log('Yesterday: ' + yesterday);
            const yesterdayDevo = await getDevoFromAPI(yesterday);
            await generateFiles(yesterday, yesterdayDevo, '/yesterday')
        }
        console.timeEnd('get devo');
    } catch (err) {
        console.warn('Unable to generate devo outputs!');
        console.log(err);
    }
}

/**
 * Update the latest_stream.html file with yesterday's video embed.
 * 
 * @param {string} startDate 
 * @param {string} endDate 
 */
async function generateYouTubeLinks(startDate, endDate) {
    const channel = await YouTubeAPIClient.getChannel();

    YouTubeAPIClient.getPlaylist(undefined, 'Devotional Streams').then(async devoStreamPlaylist => {
        let playlist = devoStreamPlaylist;
        if (Array.isArray(devoStreamPlaylist)) {
            playlist = devoStreamPlaylist[0];
        }
        const { id: playlistId } = playlist;
        const videos = await YouTubeAPIClient.getPlaylistItems(playlistId);

        // sort the videos in playlist descending by pulishtedAt
        const sortedVideos = videos.sort(function (a, b) {
            return new Date(b.snippet.publishedAt) - new Date(a.snippet.publishedAt);
        });
        const mostRecentVideoFromPlaylist = sortedVideos[0];
        const { videoId } = mostRecentVideoFromPlaylist.contentDetails;
        const mostRecentVideo = await YouTubeAPIClient.getVideos(videoId);

        const video = mostRecentVideo[0];

        // analytics output does not contain references to videoId
        const theAnalytics = await YouTubeAPIClient.getAnalytics(videoId, video.snippet.channelId, startDate, endDate);


        // make QR Code
        const videoURL = `https://www.youtube.com/watch?v=${videoId}&list=${playlistId}`;
        makeDirIfNotExists(imagesDir);


        QRCode.toFile(`${imagesDir}/latest_youtube_stream.png`, videoURL, qrConfig).then(() => { console.log('Latest YouTube Stream QR Code Generated') });

        // make html embed file
        fs.writeFile(`${htmlDir}/latest_stream.html`, `${formatHTML(video.player.embedHtml.replace('//www.', 'https://www.').replace(videoId, `${videoId}?&autoplay=1`))}`, () => {
            console.log('Latest YouTube Stream Embed HTML Generated')
        });
    });
}

/**
 * Use a devotional object to extract its attributes into individual files for each attribute
 * @param {string} date 
 * @param {object} devo 
 * @param {string} path 
 */
async function generateFiles(date, devo, path) {
    let theDate = dayjs(date, 'MM-DD-YYYY');
    const month = theDate.get('month') + 1;
    const day = theDate.get('date');
    const year = theDate.get('year');
    const attributes = Object.keys(devo);
    // const values = Object.values(devo);

    // loop over each piece of devotional data
    // and store it in html, text, or image folder
    for (let i = 0; i < attributes.length; i++) {
        const attribute = attributes[i];
        const value = `${devo[attribute]}`;

        // remove all html tags and decode characters
        const pureString = decode(striptags(value));

        // find any images and save them
        checkForAndSaveImages(pureString, attribute, path);

        // write the html and text files
        const html = formatHTML(value);
        if (path) {
            makeDirIfNotExists(`${htmlDir}${path}`);
            makeDirIfNotExists(`${textDir}${path}`);
            makeDirIfNotExists(`${imagesDir}${path}`);
        }
        if (html) {
            fs.writeFile(`${htmlDir}${path ? path : ''}/${attribute}.html`, `${formatHTML(value)}`, () => {
                // console.log(`HTML Written: ${attribute}`)
            });
        }

        if (pureString) {
            fs.writeFile(`${textDir}${path ? path : ''}/${attribute}.txt`, `${pureString}`, () => {
                // console.log(`Text Written: ${attribute}`)
            });
        }
    }
    fs.writeFile(`${textDir}${path ? path : ''}/date-string.txt`, `${theDate.format('LL')}`, () => {
        console.log('Date written')
    });
    createDevoQRCode(month, day, year, path);
}

/**
 * This little function moves all of the existing output files from
 * the html, text, and images dirs into their respective yesterday
 * dirs for easy access on the stream
 */
function moveYesterdaysFiles() {
    const yesterdayHTMLDir = `${htmlDir}/yesterday`;
    const yesterdayTextDir = `${textDir}/yesterday`;
    const yesterdayImagesDir = `${imagesDir}/yesterday`;
    makeDirIfNotExists(yesterdayHTMLDir);
    makeDirIfNotExists(yesterdayTextDir);
    makeDirIfNotExists(yesterdayImagesDir);
    fs.readdirSync(htmlDir).forEach(file => {
        fs.rename(`${htmlDir}/${file}`, `${yesterdayHTMLDir}/${file}`, () => { })
    });
    fs.readdirSync(textDir).forEach(file => {
        fs.rename(`${textDir}/${file}`, `${yesterdayTextDir}/${file}`, () => { })
    });
    fs.readdirSync(imagesDir).forEach(file => {
        fs.rename(`${imagesDir}/${file}`, `${yesterdayImagesDir}/${file}`, () => { })
    });
}

/**
 * Take a given piece of devo data and if it is a link to an image then download the image
 * @param {string} url
 * @param {string} attribute 
 */
async function checkForAndSaveImages(url, attribute, savePath) {
    makeDirIfNotExists(imagesDir);
    const imageRegex = /(?:https?|ftp):\/\/[\S]*\.(?:png|jpe?g|gif|svg|webp)(?:\?\S+=\S*(?:&\S+=\S*)*)?/g;
    const IS_IMAGE_LINK = url.match(imageRegex);
    if (IS_IMAGE_LINK) {

        // get the images file name and image format type (ie png, jpg, jpeg, svg)
        const urlArr = url.split('/');
        const originalFilename = urlArr[urlArr.length - 1];
        const fileArr = originalFilename.split('.');
        const fileType = fileArr[fileArr.length - 1];
        const CAN_BE_SAVED = attribute && fileType;

        if (CAN_BE_SAVED) {
            if (savePath) {
                makeDirIfNotExists(`${imagesDir}${savePath}`);
            }
            const filePath = path.resolve('./', `${imagesDir}${savePath ? savePath : ''}`, `${attribute}.${fileType}`);
            const writer = fs.createWriteStream(filePath);

            // get the image data
            const response = await axios.get(url, {
                responseType: 'stream'
            });

            // save the image file
            response.data.pipe(writer);
        }
    }
}

/**
 * Create and save a QR code to the images directory
 * @param {number} month 
 * @param {number} day 
 * @param {number} year 
 * @param {string} path 
 */
async function createDevoQRCode(month, day, year, path) {
    makeDirIfNotExists(imagesDir);
    if (path) {
        makeDirIfNotExists(`${imagesDir}${path}`);
    }
    // TODO: add tracking code to watch usage!
    const url = `https://odb.org/US/${year}/${('0' + month).slice(-2)}/${('0' + day).slice(-2)}`;
    // property id 357539023
    // stream id 4691043667
    // measurement id G-XKYR6QM6X5
    // http://www.example.com/?utm_source=exampleblog&utm_medium=referral&utm_campaign=summer-sale

    await QRCode.toFile(`${imagesDir}${path ? path : ''}/devo_qr.png`, url, qrConfig);
}

/**
 * Format and save the content in a valid html document.
 * This is critical because it sets the meta charset tag to properly parse the content.
 * @param {string} content 
 * @returns 
 */
function formatHTML(content) {
    if (content) {
        const head = '<html lang="en" data-color-mode="auto" data-light-theme="light" data-dark-theme="dark" data-a11y-animated-images="system" data-turbo-loaded=""><head><meta charset="utf-8"><title>Embed utf-8 html content</title></head><body>'
        const end = '</body></html>';
        const final = `${head}${content}${end}`;
        return final;
    }
}

/**
 * Check for a named directory and create the dir if it does not exist.
 * @param {string} dir 
 */
function makeDirIfNotExists(dir) {
    if (!fs.existsSync(dir)) {
        fs.mkdirSync(dir);
    }
}

/**
 * Create a date prompt with Inquirer
 * @param {string} name 
 * @param {string} message 
 * @returns { date } result from Inquirer prompt
 */
async function propmtForDate(name, message) {
    const startDatePrompt = {
        type: 'date',
        name,
        message,
        locale: 'en-US',
        format: { month: "2-digit", day: "2-digit", year: "numeric", hour: undefined, minute: undefined }
    };
    const result = await inquirer.prompt([startDatePrompt]);
    return result[name];
}

generateContent();

package.json

{
  "name": "stream-scripts",
  "version": "0.0.1",
  "description": "Script for generating text & html content from API Endpoint in Live Streams",
  "main": "get_devo.js",
  "type": "module",
  "directories": {
    "test": "test"
  },
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1"
  },
  "author": "Charlie Bleisch",
  "license": "ISC",
  "dependencies": {
    "axios": "^1.3.4",
    "babel-polyfill": "^6.26.0",
    "branded-qr-code": "^1.3.0",
    "dayjs": "^1.11.7",
    "google-auth-library": "^8.7.0",
    "googleapis": "^112.0.0",
    "html-entities": "^2.3.3",
    "inquirer": "^9.1.4",
    "inquirer-date-prompt": "^3.0.0",
    "qr-code-styling": "^1.6.0-rc.1",
    "qr-with-logo": "^1.0.5",
    "qrcode": "^1.5.1",
    "qrcode-with-logos": "^1.0.3",
    "striptags": "^3.2.0"
  }
}