Gatsby + S3 no trailing slash redirect patch

README.md

Read more here: gatsbyjs/gatsby#19543 (comment)

These scripts should be paired with: https://www.gatsbyjs.org/packages/gatsby-plugin-remove-trailing-slashes/

patch-trailing-slash-locally.sh

#!/bin/sh

# This is the first of the two scripts that are being used to work around
# the limitations of conventional filesystems where two filesystem nodes
# can't share the same name regardless of whether they're of the same type
# or not (folder / file).

# S3 is not a conventional filesystem, it's an object storage where filename
# is the key and the value is the object and therefore this limitation does
# not apply to S3.

# However, since we need to build the application locally first, we need to
# append an extension to the cloned files locally and then after uploading
# all files to S3, we run the 2nd script that will remove the file extension.


# This script will find all index.html files that are nested at least 2 levels deep and copy them
# to the grandparent folder while renaming them to match their original parent folder name and
# appending .collision extension because most filesystem don't allow two filesystem nodes to share
# the same name regardless of their type (folder / file). However, S3 is not a typical filesystem
# and allows such a configuration where folder and file share the same name and that's what we're
# utilizing here. So the order of steps is as follows:
#
# 1. Deployment artifact is built
# 2. patch-trailing-slash-locally.sh is ran which creates clones with a .collision.html extension to
#    prevent conflicts (this script)
# 3. public folder is deployed
# 4. patch-trailing-slash-remotely.sh is ran which removes the .collision.html extension

echo "Cloning nested index.html files and copying them to the grandparent folder"
find ./public -mindepth 2 -name "index.html" | sed 's/^\(.*\)\/index\.html$/\1/' | grep -v 404 | xargs -L1 -t -I {} cp "{}/index.html" "{}.collision.html"
echo "Done"

patch-trailing-slash-remotely.js

// Loosely based on https://gist.github.com/cmawhorter/b706e45ba88e43bc379c

// Script to work around discrepancies in s3 <=> os file and directory names.
// It allows you to host your static website on s3 without trailing slashes.
// e.g. example.com/products and example.com/products/mugs

const http = require('http')
const https = require('https')

const AWS = require('aws-sdk')
const async = require('async')

// Resource limits
http.globalAgent.maxSockets = 72
https.globalAgent.maxSockets = 72

// Script input assertions
if (!process.env.S3_BUCKET) throw new Error('Must configure S3_BUCKET env var')
if (!process.env.AWS_ACCESS_KEY_ID) throw new Error('Must configure AWS_ACCESS_KEY_ID env var')
if (!process.env.AWS_SECRET_ACCESS_KEY)
  throw new Error('Must configure AWS_SECRET_ACCESS_KEY env var')

// AWS SDK configuration
const options = {
  bucket: process.env.S3_BUCKET,
  region: process.env.AWS_DEFAULT_REGION,
  key: process.env.AWS_ACCESS_KEY_ID,
  secret: process.env.AWS_SECRET_ACCESS_KEY,
  bucketPrefix: '',
}

AWS.config.update({
  accessKeyId: options.key,
  secretAccessKey: options.secret,
  region: options.region,
})

// AWS SDK error handling
AWS.events.on('httpError', function onHttpError() {
  if (this.response.error && this.response.error.code === 'UnknownEndpoint') {
    this.response.error.retryable = true
  }
})

const s3 = new AWS.S3()

// This function will determine the mime type based on the file extension
function determineMimeType(prefix) {
  const ext = prefix.split('.').pop()
  switch (ext.toLowerCase()) {
    default:
    case 'html':
    case 'aspx':
      return 'text/html'
  }
}

// S3 SDK does not offer a method to rename files hence we had to bake our own
// This function will remove the .collision.html extension from the name of the
// S3 object stored under the prefix path and configure its mime type correctly
function renameObject(prefix, callback) {
  const renamedPrefix = prefix.replace('.collision.html', '')
  const mimeType = determineMimeType(prefix)

  console.log('renameObject %s -> %s; mime = %s', prefix, renamedPrefix, mimeType)

  const params = {
    Bucket: options.bucket,
    CopySource: `${options.bucket}/${prefix}`,
    Key: renamedPrefix,
    MetadataDirective: 'REPLACE',
  }

  if (mimeType) {
    params.ContentType = mimeType
  }

  s3.copyObject(params, function onCopyObject(err, data) {
    if (err) return callback(err)

    console.log('\t-> copied (%s)', prefix)

    s3.deleteObject(
      {
        Bucket: options.bucket,
        Key: prefix,
      },
      function onDeleteObject(err, data) {
        if (err) return callback(err)
        console.log('\t-> removed (%s)', prefix)
        callback()
      }
    )
  })
}

// This function returns true if the prefix ends with our magic file extension
function collisionsOnly(prefix) {
  return prefix
    .split('/')
    .pop()
    .endsWith('.collision.html')
}

function removeMagicExtension(prefix) {
  return async.apply(renameObject, prefix)
}

const allKeys = []

const listAllKeys = function listAllKeys(marker, cb) {
  s3.listObjects({ Bucket: options.bucket, Marker: marker }, function onListObjects(err, data) {
    if (err) {
      return cb(err)
    }

    Array.prototype.push.apply(
      allKeys,
      data.Contents.map(function onMapContents(el) {
        return el.Key
      })
    )

    if (data.IsTruncated) {
      listAllKeys(data.Contents.slice(-1)[0].Key, cb)
    } else {
      cb()
    }
  })
}

// Traverse the entire tree of files and remove the magic extension
// from all matching files.
listAllKeys(options.bucketPrefix, function onListAllKeys(err) {
  if (err) {
    process.exit(1)
  }

  async.parallelLimit(allKeys.filter(collisionsOnly).map(removeMagicExtension), 12, function onDone(
    err
  ) {
    if (err) throw err
    console.log('Done')
    process.exit()
  })
})