Nuxt Magpie module

magpie.js

const fs = require('fs')
const { URL } = require('url')
const { join } = require('path')
const axios = require('axios')
const consola = require('consola')

const defaults = {
  path: '/_images', // dir where downloaded images will be stored
  extensions: ['jpg', 'jpeg', 'gif', 'png', 'webp'],
  baseUrl: '' // cms url
  // TODO: add option to allow keeping the original folder structure
}

export default function Magpie(moduleOptions) {
  const options = { ...defaults, ...moduleOptions }
  const baseDir = join(this.options.generate.dir, options.path)

  this.nuxt.hook('generate:distCopied', () => {
    if (!fs.existsSync(baseDir)) fs.mkdirSync(baseDir)
  })

  this.nuxt.hook('generate:page', async (page) => {
    const urls = []
    const test = new RegExp('(http(s?):)([/|.|\\w|\\s|-])*\.(?:' + options.extensions.join('|') + ')', 'g')
    const matches = page.html.matchAll(test)
    for (const match of matches) {
      const baseUrl = URL(moduleOptions.baseUrl)
      const url = URL(match[0])
      if (baseUrl.hostname === url.hostname && !urls.find(u => u.href === url.href)) {
        urls.push(url)
      }
    }
    if (!urls.length) return
    consola.info(`${page.route}: Magpie is replacing ${urls.length} images with local copies`)
    return await replaceRemoteImages(page.html, urls).then(html => page.html = html)
  })

  async function replaceRemoteImages(html, urls) {
    await Promise.all(urls.map(async (url) => {
      const ext = '.' + url.pathname.split('.').pop()
      const name = slugify(url.pathname.split(ext).join('')) + ext
      const imgPath = join(baseDir, name)
      return saveRemoteImage(url.href, imgPath)
        .then(() => {
          html = html.split(url.href).join(options.path + '/' + name)
        })
        .catch(e => consola.error(e))
    }))
    return html
  }
}

function saveRemoteImage(url, path) {
  return axios({
    url,
    responseType: 'stream'
  }).then(
    response =>
      new Promise((resolve, reject) => {
        response.data
          .pipe(fs.createWriteStream(path))
          .on('finish', () => resolve())
          .on('error', e => reject(e))
      })
  )
}

// https://gist.github.com/codeguy/6684588
function slugify(text) {
  return text
    .toString()
    .toLowerCase()
    .normalize('NFD')
    .trim()
    .replace('/', '')
    .replace(/\s+/g, '-')
    .replace(/[^\w\-]+/g, '-')
    .replace(/\-\-+/g, '-')
}

This looks awesome! Any guidance on how to insert this into nuxt?

Hey! this has not been revised for nuxt v2.13 which introduces full static mode and js payloads for each route, but for the previous versions you just put it in a file inside the modules dir and add it in the nuxt config file like this:

  buildModules: [
    ['~/modules/magpie.js', { baseUrl: PUT_YOUR_CMS_URL_HERE }]
  ],

@emiliobondioli
Thank you very much for this!
Would be amazing to have this for 2.13. Will you update it?

@J05HI sure! Right now the generated page is parsed correctly, but I'm still trying to figure the best way to replace the image urls in the payload saved during full static generation (mentioned here).

Thank you! I've managed to create a little module adapted for v2.13 nuxt-image-extractor, it's working fine, the only missing thing is payload replacement.
Please if you have any idea on this, I'll be glad to help!

I can think of 2 options:

1. Hooking into nuxt export process and replacing the urls in the payload before it is saved, for this we'd need an additional hook since from what I can see there is no export hook which gives me the payload for each route. I've tried adding it in the nuxt source files (somewhere around here) and it is working fine, albeit a bit hackish because the payload has already been stringified at that point. I've asked the team about adding that hook but I've received no answer so far.
2. Hooking into export:routeCreated to get the payload.js for each route, read it, replace image urls and save it again. This would be the best approach right now because it doesn't require changes to nuxt source, but is also way less efficient because it adds a read and write operation for each generated route.

Well, I'm going to explore the second option... less efficient is better than nothing, at least in my case 😄

If you want, I'd be happy to see what solution you came up with!
As a heads up, each time nuxt runs a full static generation, a new dir with the version number is created, so to get the correct path for each route you can do:

  this.nuxt.hook('export:routeCreated', async ({ route, path, errors }) => {
    const routePath = join(this.options.generate.dir, this.options.generate.staticAssets.versionBase, route)
    const payloadPath = join(routePath, 'payload.js')
    // ex. payloadPath = /dist/_nuxt/static/1592921605/about/payload.js

Thanks for the payload path information, it helped me. After a hard time on RegEx and string manipulation... I've managed to complete a working version. However, it needs more testing to be production ready.