Script to Convert Storyblok Assets (PNG/JPG/JPEG) to Modern WebP Format

storyblok-webp-converter.js

'use strict'

const _ = require('lodash');
const axios = require('axios');
const fs = require('fs');
const path = require('path');
const FormData = require('form-data');

const StoryblokClient = require('storyblok-js-client');
const sharp = require('sharp');

const context = {
    oauthToken: '',
    accessToken: '',
    spaceId: '',
}

const beforeWebpAssetSize = {}
const afterWebpAssetSize = {}

/**
 * Returns a new Storyblok client authenticated with the given credentials
 * @returns {StoryblokClient} The Storyblok client authenticated
 */
const getStoryblokClient = () => {
    return new StoryblokClient(context)
}

/**
 * Returns whether the given asset has an image extension
 * @param {Object} asset The asset to check
 * @returns {Boolean} Whether the asset has an image extension
 */
const hasAssetImageExtension = (asset) => {
    const imageExtensions = ['jpg', 'jpeg', 'png'];
    const extension = asset.filename.split('.').pop();
    const isAlreadyWebp = extension === 'webp' || asset.content_type === 'image/webp'
    return imageExtensions.includes(extension) && !isAlreadyWebp;
}

/**
 * Converts the given image buffer to webp
 * @param {Buffer} imageBuffer The image buffer to convert to webp
 * @returns {Promise<Buffer>} The converted image buffer
 */
const convertToWebp = async (imageBuffer) => {
    const webpBuffer = await sharp(imageBuffer)
        .webp()
        .toBuffer();

    return webpBuffer;
};

/**
 * Fetches the image buffer from the given URL
 * @param {String} imageUrl The URL of the image to fetch
 * @returns {Promise<Buffer>} The image buffer
 */
const getImageBufferFromUrl = async (imageUrl) => {
    const response = await axios.get(imageUrl, {
        responseType: 'arraybuffer',
    });

    return Buffer.from(response.data, 'binary');
};

/**
 * Saves the given image buffer to a file in the medias folder
 * @param {Buffer} imageBuffer The image buffer to save
 * @param {String} filename The name of the file to save the image as
 * @returns {Promise<String>} The absolute path of the saved .webp image
 */
const saveConvertedAsset = async (assetId, imageBuffer) => {
    const absolutePath = path.resolve(`./medias/${assetId}.webp`)
    fs.promises.writeFile(absolutePath, imageBuffer);
    afterWebpAssetSize[assetId] = imageBuffer.length
    return absolutePath
};

/**
 * Converts the given asset image to webp
 * @param {Object} asset The asset to convert to webp
 * @returns {Promise<String>} The absolute path of the converted .webp image
 */
const covertAssetImageToWebp = async (asset) => {
    if (!hasAssetImageExtension(asset)) {
        return null
    }

    try {
        console.log('> Converting image %s to webp', asset.filename)
        beforeWebpAssetSize[asset.id] = asset.content_length

        const imageBuffer = await getImageBufferFromUrl(asset.filename);
        const resizedImageBuffer = await convertToWebp(imageBuffer);
        const webpFilePath = await saveConvertedAsset(asset.id, resizedImageBuffer);
        return webpFilePath
    } catch (error) {
        return null
    }
}

/**
 * Requests the upload params for the given asset ID
 * @param {Number} assetId The storyblok asset ID to request upload params for
 * @returns {Promise<Object>} The upload params for the given asset ID
 */
const requestAssetUploadParams = async (assetId) => {
    return getStoryblokClient().post(`spaces/${context.spaceId}/assets`, {
        id: assetId,
        filename: `${assetId}.webp`,
        validate_upload: '1',
    }).then(response => {
        return {
            fields: response.data.fields,
            post_url: response.data.post_url,
            pretty_url: response.data.pretty_url,
            public_url: response.data.public_url,
            id: response.data.id,
            locked: response.data.locked,
        }
    }).catch(error => {
        console.error(error)
        return null
    })
}

/**
 * Submits the given form to the given URL
 * @param {String} post_url The url to send the form to via post
 * @param {FormData} form The form to submit
 * @returns {Promise<Boolean>} Whether the form was submitted successfully
 */
const submitAssetUpload = (post_url, form) => {
    return new Promise((resolve, reject) => {
        form.submit(post_url, (error, response) => {
            if (error) {
                reject(false)
            } else {
                resolve(true)
            }
        })
    })
}

/**
 * Refreshes the given asset upload after form submission.
 * @param {Object} params The params to refresh the asset upload
 * @returns {Promise<String>} The public URL of the uploaded asset, or `false`
 */
const refreshAssetUpload = async (params) => {
    return getStoryblokClient()
        .get(`spaces/${context.spaceId}/assets/${params.id}/finish_upload`)
        .then(() => params.public_url)
        .catch(() => false);
}

/**
 * Uploads the given .webp image to Storyblok and refreshes the cache
 * @param {String} absolutePath The absolute path of the .webp image to upload
 */
const uploadConvertedAsset = async (absolutePath) => {
    const form = new FormData();

    const assetId = absolutePath.split('/').pop().split('.').shift()
    const uploadParams = await requestAssetUploadParams(assetId)

    for (let key in uploadParams.fields) {
        form.append(key, uploadParams.fields[key]);
    }

    form.append('file', fs.createReadStream(absolutePath));
    const isSubmitted = await submitAssetUpload(uploadParams.post_url, form)

    if (isSubmitted) {
        const publicUrl = await refreshAssetUpload(uploadParams)
        const newParams = await requestAssetUploadParams(assetId)
    } else {
        console.error('> Failed to submit %s', absolutePath)
    }
}

/**
 * Converts the images in the given story to webp
 * @param {Array<Object>} assets The list of assets to convert to WebP
 * @returns {Promise<Array<String>>} The absolute paths of the converted .webp images
 */
const convertAssetsToWebp = async (assets) => {
    const results = []

    for (const assetImage of assets) {
        const path = await covertAssetImageToWebp(assetImage)
        results.push(path)
    }

    const pathsToUpload = results.filter(_.isString)
    await Promise.all(pathsToUpload.map(uploadConvertedAsset))

    for (const uploadedPath of pathsToUpload) {
        fs.unlinkSync(uploadedPath)
    }
}

/**
 * Converts the given bytes to mb
 * @param {Number} bytes The bytes to convert to mb
 * @returns {String} The bytes converted to mb
 */
const bytesToMb = (bytes) => {
    return (bytes / 1024 / 1024).toFixed(2);
}

/**
 * Returns a promise that resolves after the given number of milliseconds
 * @param {Number} ms The number of milliseconds to delay
 * @returns {Promise} A promise that resolves after the given number of milliseconds
 */
const pleaseWait = (ms) => {
    return new Promise(resolve => setTimeout(resolve, ms));
}

/**
 * Prints the benchmark of the conversion at ending
 */
const printBenchmark = () => {
    const totalFiles = Object.keys(beforeWebpAssetSize).length
    const beforeTotal = Object.values(beforeWebpAssetSize).reduce((a, b) => a + b, 0)
    const afterTotal = Object.values(afterWebpAssetSize).reduce((a, b) => a + b, 0)
    const percent = Math.round((afterTotal / beforeTotal) * 100)

    console.log('====================================================')
    console.log('> [Benchmark] Total Size Before: %d mb(s)', bytesToMb(beforeTotal))
    console.log('> [Benchmark] Total Size After : %d mb(s)', bytesToMb(afterTotal))
    console.log('> [Benchmark] File Reduction Average: %d%%', percent)
    console.log('> [Benchmark] Total of Number of Files/Assets', totalFiles)
    console.log('====================================================')
}

const convertAllAssets = async () => {
    let currentPage = 1
    let totalPages = 2

    const client = getStoryblokClient()
    const PER_PAGE = 6

    while (currentPage <= totalPages) {
        const response = await client.get(`spaces/${context.spaceId}/assets`, {
            per_page: PER_PAGE,
            page: currentPage
        })

        totalPages = (response.total / PER_PAGE)

        const assets = response.data.assets
        await convertAssetsToWebp(assets) // Convert all assets to webp

        console.log('> Page %d of %d completed', currentPage, totalPages)
        console.log(' ')
        printBenchmark()
        console.log(' ')

        currentPage++
    }

    return (currentPage * PER_PAGE)
}

/**
 * Begins the conversion of the given config (spaceId, accessToken, and oauthToken).
 * @param {Object} config The config to begin the conversion
 */
const beginConversion = async (config) => {
    context.spaceId = config.spaceId
    context.accessToken = config.accessToken
    context.oauthToken = config.oauthToken

    await convertAllAssets()
    printBenchmark()
}

module.exports = beginConversion.bind(this)

I developed the script to convert 21,750 assets from the website https://lupa.uol.com.br/ into the modern and efficient 'WebP' format introduced by Google. This conversion resulted in approximately 80% bandwidth savings, as the WebP format is more streamlined and compact.