Atomic Write Helper for the Firebase Realtime Database

PendingUpdate.js

/*! Copyright (c) 2017 Samuel Jones | MIT License | github.com/samthecodingman */
/* Contains contributions by @benweier and @rhalff */

/**
 * A Reference represents a specific location in the Database and can be used
 * for reading or writing data to that Database location.
 * @typedef fbReference
 * @type {firebase.database.Reference}
 */

/**
 * Defines a helper class to combine multiple operations into a single
 * "all-or-nothing" database update.
 * @type {Object}
 */
class PendingUpdate {
  /**
   * Creates a new PendingUpdate instance.
   * @param  {Object} db - the firebase database instance to use
   * @param  {boolean} suppressChecks - if true, skip path checks
   */
  constructor (db, suppressChecks) {
    // Nifty snippet from github.com/benweier/connect-session-firebase [MIT]
    if (db.ref) {
      this.db = db
    } else if (db.database) {
      this.db = db.database()
    } else {
      throw new Error('Invalid Firebase reference')
    }
    this.data = {}
    this.suppressChecks = Boolean(suppressChecks)
  }

  /**
   * Removes all entries that are children of the given location.
   * @param  {(string|fbReference)} ref - path/reference to operation location
   * @return {PendingUpdate} for chaining
   */
  remove (ref) {
    if (typeof ref === 'string') ref = this.db.ref(ref)
    let path = _getRefPath(ref)
    this.data = this.data.filter((entryPath) => !entryPath.startsWith(path))
    return this
  }

  /**
   * Adds a SET operation to the update that will remove the data at the given
   * location.
   * 
   * Unless configured otherwise, the operation will be ignored unless the
   * operation occurs on a non-root key.
   * @param  {(string|fbReference)} ref - path/reference to operation location
   * @param  {boolean} [allowRootKey] - skips the path check if truthy
   * @return {PendingUpdate} for chaining
   */
  addRemove (ref, allowRootKey) {
    if (typeof ref === 'string') ref = this.db.ref(ref)
    let path = _getRefPath(ref)
    if (this.suppressChecks || allowRootKey || (path && path.lastIndexOf('/') > 0)) {
      this.data[path] = null
    }
    return this
  }

  /**
   * Adds a SET operation to the update
   * 
   * Unless configured otherwise, the operation will be ignored unless the
   * operation occurs on a non-root key.
   * @param  {(string|fbReference)} ref - path/reference to operation location
   * @param  {*} data - the data to set
   * @param  {boolean} [allowRootKey] - skips the path check if truthy
   * @return {PendingUpdate} for chaining
   */
  addSet (ref, data, allowRootKey) {
    if (typeof ref === 'string') ref = this.db.ref(ref)
    let path = _getRefPath(ref)
    if (this.suppressChecks || allowRootKey || (path && path.lastIndexOf('/') > 0)) {
      this.data[path] = data
    }
    return this
  }

  /**
   * Adds an UPDATE operation to the update
   * 
   * Unless configured otherwise, the operation will be ignored unless the
   * operation occurs on a non-root key.
   * @param  {(string|fbReference)} ref - path/reference to operation location
   * @param  {*} data - the data to set
   * @param  {boolean} [allowRootKey] - skips the path check if truthy
   * @return {PendingUpdate} for chaining
   */
  addUpdate (ref, data, allowRootKey) {
    if (typeof ref === 'string') ref = this.db.ref(ref)
    if ((isArrayOrObject(data) && (
      (isObject(data) && !isEmptyObject(data)) ||
      (Array.isArray(data) && (data.length !== 0))
    ))) {
      Object.keys(data).forEach((key) => {
        this.addSet(ref.child(key), data[key], allowRootKey)
      });
      return this
    } else {
      return this.addSet(ref.child(key), data[key], allowRootKey)
    }
  }

  /**
   * Adds individual SET operations for each value in the given object.
   *
   * Unless configured otherwise, the operation will be ignored unless the
   * operation occurs on a non-root key.
   * @param  {(string|fbReference)} ref - path/reference to operation location
   * @param  {*} data - the data to set
   * @param  {boolean} [allowRootKey] - skips the path check if truthy
   * @return {PendingUpdate} for chaining
   */
  addRecursiveSet (ref, data, allowRootKey) {
    if (typeof ref === 'string') ref = this.db.ref(ref)
    let path = _getRefPath(ref)
    if (this.suppressChecks || allowRootKey || (path && path.lastIndexOf('/') > 0)) {
      objectToPathMap(data, this.data, [path])
    } else {
      Object.keys(data).forEach((key) => {
        if ((isArrayOrObject(data[key]) && (
          (isObject(data[key]) && !isEmptyObject(data[key])) ||
          (Array.isArray(data[key]) && (data[key].length !== 0))
        ))) {
          return this.addRecursiveSet(ref.child(key), data[key], allowRootKey)
        } else {
          return this.addSet(ref.child(key), data[key], allowRootKey)
        }
      })
    }

    return this
  }

  /**
   * Commits the changes to the database.
   * @return {Promise} resolves when the write has completed.
   */
  commit () {
    return this.db.ref().update(this.data).then(() => this.data.length)
  }
}

module.exports = PendingUpdate

/**
 * Returns a string path of the given reference relative to the database root.
 * @param  {fbReference} ref - the Reference object
 * @return {string} - the string path
*/
function _getRefPath (ref) {
  return ref.toString().substring(ref.root.toString().length)
}

/*! Imported from rhalff/dot-object */
/*! Copyright (c) 2013 Rob Halff | MIT License */
/* eslint-disable require-jsdoc */

function isArrayOrObject (val) {
  return Object(val) === val
}

function isObject (val) {
  return Object.prototype.toString.call(val) === '[object Object]'
}

function isEmptyObject (val) {
  return Object.keys(val).length === 0
}

/**
 *
 * Convert object to path/value pair
 *
 * @param {Object} obj source object
 * @param {Object} tgt target object
 * @param {Array} path path array (internal)
 * @param {Boolean} keepArray indicates if arrays should be preserved
 * @return {Object} reference to tgt
 */
function objectToPathMap (obj, tgt, path, keepArray) {
  tgt = tgt || {}
  path = path || []
  Object.keys(obj).forEach((key) => {
    if ((isArrayOrObject(obj[key]) && (
      (isObject(obj[key]) && !isEmptyObject(obj[key])) ||
      (Array.isArray(obj[key]) && (!keepArray && (obj[key].length !== 0)))
    ))) {
      return objectToPathMap(obj[key], tgt, path.concat(key), keepArray)
    } else {
      tgt[path.concat(key).join('/')] = obj[key]
    }
  })
  return tgt
}

All-or-Nothing / Atomic Write Helper for the Firebase Realtime Database

Purpose

Sometimes you need to update many locations in the Firebase Realtime Database at once. Aside from manually throwing paths into an update() operation, this isn't overly intuitive. It also hurts readability depending on what you are trying to do with the update. To fix this, I devised this helper class that allows you to add set() and remove() operations that will either all succeed at once or all fail. Furthermore, it also allows you do a recursive set() that truly merges the changes of every sub-property rather than replacing the child properties as objects (see Issue #134 of firebase/firebase-js-sdk for more information).

This is particularly useful for lengthy data clean-up tasks or large data merges that should complete as a single operation. An example of this helper class in use is at the bottom of this documentation comment or in the documentation comment of adminTaskFunction.js.

Initialization

Imports

Import the required modules:

const admin = require('firebase-admin') // (should also work with client just fine)
const PendingUpdate = require('./PendingUpdate')
admin.initializeApp(...)

Constructor

• db is the firebase database instance to be used for the update.
• suppressChecks determines if the built-in data safety checks (see below) should be used. Passing in true will disable the checks and improve performance merginally but opens up the possibility of corrupting your database.
  Default: false.

let pendingUpdate = new PendingUpdate(db, suppressChecks)

Operations

Each operation will return the handle to it's own PendingUpdate instance. This allows you to chain operations if desired.

The first argument of each of the following methods is the location of the operation. They can be any of the following:

• a truthy string representing the path, this will be passed through this.db.ref(string)
• a firebase.database.Reference
• an admin.database.Reference

The last argument of each of the following methods is optional and specifies whether the root-access protections should be lifted for that operation (see Data Safety Features below).

Set

To add a set() operation to the batch:

pendingUpdate.addSet('/some/new/key', someData)
// or
pendingUpdate.addSet(admin.database().ref('/some/new/key'), someData)

Remove

To add a remove() operation to the batch:

pendingUpdate.addRemove('/some/path/to/key')

which is equivalent to:

pendingUpdate.addSet('/some/path/to/key', null)

Update

To add a traditional update() operation to the batch:

pendingUpdate.addUpdate('/some/new/key', { date: Date.now(), data: { nested: 'name' } })

which is equivalent to:

pendingUpdate.addSet('/some/path/to/key/date', Date.now())
pendingUpdate.addSet('/some/path/to/key/data', { nested: 'name' })

Note: This operation adds individual set() operations for each top-level enumerable property in the value, replicating the behaviour of update() from the client and admin javascript SDKs. If you want a true merge, see the Merge operation below.

Merge

The traditional update() operation only creates set() operations for the top level enumerable properties as discussed in Issue #134 of firebase/firebase-js-sdk. This variant will add set operations for each enumerable property and their sub-properties recursively.

pendingUpdate.addRecursiveSet('/some/new/key', { date: Date.now(), data: { nested: 'name' } })

which is equivalent to:

pendingUpdate
  .addSet('/some/path/to/key/date', Date.now())
  .addSet('/some/path/to/key/data/nested', 'name')

Commit Changes

Nothing will change in your database until you call commit(). Don't forget to call it!

let returnedPromise = pendingUpdate.commit() // returns a Promise

This will call a multi-location update using the given database instance that will change all your data atomically in an all-or-nothing fashion.

• If successful, returnedPromise will resolve with the number of paths changed.
• If anything fails (permissions, data validation, etc), no changes will be made and the promise will be rejected with the associated error.

Note: Once a PendingUpdate has been committed, you should consider it's state invalid and cease further use of that PendingUpdate instance.

Data Safety Features

Disclaimer

These protections are not going to be infallible. But they should at least help protect your database somewhat. I take no responsibility for lost data.

Introduction

When you create a new instance of PendingUpdate or use one of it's methods, the last optional argument determines if the helper class will silently ignore operations affecting the top-level keys in the root of your database.

To create a protected PendingUpdate instance:

let pendingUpdate = new PendingUpdate(admin.database())
// OR let pendingUpdate = new PendingUpdate(admin.database(), false)

pendingUpdate.addRemove('/users') // silently ignored

To create an unprotected PendingUpdate instance:

let pendingUpdate = new PendingUpdate(admin.database(), true)
pendingUpdate.addRemove('/users') // added to pending operations

To create an protected PendingUpdate instance, but override the protection for a single call:

let pendingUpdate = new PendingUpdate(admin.database())
pendingUpdate
  .addRemove('/data') // ignored
  .addRemove('/users', true) // added to pending operations

The protections prevent usage like so:

pendingUpdate.addRemove('/') // attempts to remove entire db, silently ignored

Protection Example: Attempting to delete /users

By default, calling addRemove() for the path /users will result in nothing happening.

let pendingUpdate = new PendingUpdate(admin.database()) // protected mode
pendingUpdate
  .addRemove('/users') // silently ignored
  .commit() // commits nothing

To override this behaviour for the entire PendingUpdate instance, supply true as the second argument of the constructor.

let pendingUpdate = new PendingUpdate(admin.database(), true) // unprotected mode
pendingUpdate
  .addRemove('/users') // adds pending delete operation
  .commit() // commits the delete of the entire /users tree

To override this on a per-call basis, initialize the PendingUpdate instance as normal, but provide true as the last parameter of the call to addRemove().

let pendingUpdate = new PendingUpdate(admin.database()) // protected mode
pendingUpdate
  .addRemove('/data') // silently ignored
  .addRemove('/users', true) // unprotected call, adds pending delete operation
  .commit() // commits the delete of the entire /users tree

Example

const functions = require('firebase-functions')
const admin = require('firebase-admin')
const PendingUpdate = require('PendingUpdate')
const md5 = require('md5')
admin.initializeApp(functions.config().firebase)

exports.dataCleanupForDeletedUser = functions.auth.user().onDelete((event) => {
  let deletedUserRecord = event.data
  let deletedUserID = deletedUserRecord.uid
  let deletedUserEmails = deletedUserRecord.providerData.reduce((emails, userInfo) => {
    if (userInfo.email) emails.push(userInfo.email)
    return emails
  }, deletedUserRecord.email ? [deletedUserRecord.email] : [])
  
  let db = admin.database()
  
  let pendingUpdate = new PendingUpdate()
  pendingUpdate.addRemove(`/users/${deletedUserID}`)
  
  deletedUserEmails.forEach((email) => {
    let hashedEmail = getMD5EmailHash(email)
    pendingUpdate.addRemove(`/emailUserMap/${hashedEmail}`)
  })
  
  if (deletedUserRecord.phoneNumber) {
    pendingUpdate.addRemove(`/phoneUserMap/${phoneNumber}`)
  }
  
  return pendingUpdate.commit().then(() => {
    console.log(`Successfully deleted user data for @${deletedUserID}`)
  }, (err) => {
    console.error(`Failed to delete user data for @${deletedUserID}.`, err)
  })
})

/**
 * Gets a Gravatar-compatible MD5 hash of the given email address.
 * @param  {String} email - the email to hash
 * @return {String}       - the hashed email
 */
function getMD5EmailHash (email) {
  return email == null ? email : md5(String(email).trim().toLowerCase())
}