GCloud Backup script

README-gcloud-backup-scripts.md

Backups

The Gist has turned into: https://github.com/TvrboPro/TvrboBackup

backup.js

#!/usr/bin/env node

// npm install gcloud-backup-toolset shelljs bluebird

const config = require('./config.js');
const gCloudFileName = process.cwd() + '/gcloud.json';
const gCloudConfig = require(gCloudFileName);
const { cd, exec, ls } = require('shelljs');

global.Promise = require("bluebird");
const { storage } = require('google-cloud');
const { tar, bzip2, aesEncrypt, log, makeRandomFolder, cleanRandomFolders } = require('./lib');

const gcs = storage({
  projectId: gCloudConfig.project_id,
  keyFilename: gCloudFileName,
  promise: global.Promise
});
const bucket = gcs.bucket(config.gCloudBucket);

// YOUR CODE GOES HERE

function myDataHandling(tmpDir) {
  cd(tmpDir);

  log("DUMPING THE DATABASE");
  exec(`mongodump --db ${config.databaseName}`);
}

// MAIN CODE

async function doBackup() {
  const tmpDir = makeRandomFolder();
  log(`The working folder is ${tmpDir}`);
  log("STARTING BACKUP", gCloudConfig.project_id, config.backupName);

  // prepare
  myDataHandling(tmpDir);

  // package
  cd(tmpDir);
  log("PACKAGING");
  const dateSuffix = new Date().toJSON().replace(/:/g, '-').replace(/\.[0-9]{3}Z$/, '');
  const tarFile = tar(".", `backup-${config.backupName}-${dateSuffix}.tar`);

  log("COMPRESSING", tarFile);
  const bzipFile = bzip2(tarFile);

  var uploadedFile;
  if (config.aesKey) {
    log("ENCRYPTING", bzipFile);
    uploadedFile = bzipFile + ".aes";
    aesEncrypt(bzipFile, uploadedFile, config.aesKey);
  }
  else {
    uploadedFile = bzipFile;
  }

  // upload
  log("UPLOADING", tmpDir + "/" + uploadedFile);
  const data = await bucket.upload(tmpDir + "/" + uploadedFile, { destination: `${config.bucketSubdir}/${uploadedFile}` });
  var file = data[0];

  if(file && file.metadata && file.metadata.selfLink) {
    log("DONE", file && file.metadata && file.metadata.selfLink);
  }
  else {
    log("Could not complete the operation", file);
  }
  cleanRandomFolders();
}

try {
  doBackup();
}
catch (err) {
  console.error("Unable to complete the backup:", err.message);
  console.error(err);
}

config.js

module.exports = {
  gCloudBucket: "tvrbo-back-dora-black",
  bucketSubdir: "mongodb-backup",
  aesKey: "YOUR KEY GOES HERE", // optional
  databaseName: 'db-name',
  backupName: 'weekly',
  backupCopies: 10  // how many rotations to keep
};

lib.js

const { exec , rm, mkdir } = require('shelljs');
const { existsSync } = require('fs');
const { extname, basename } = require('path');

// PACKAGING / ENCRYPTION

function tar(inFiles, outFile) {
    if(typeof(inFiles) == "object")
        exec(`tar cvf ${outFile} ${inFiles.join(' ')}`);
    else
        exec(`tar cvf ${outFile} ${inFiles}`);
        
    return outFile;
}

function unTar(inFile) {
    exec(`tar xvf ${inFile}`);
    return inFile;
}

function bzip2(file) {
    exec(`bzip2 -z -f -v --best ${file}`);
    return file + ".bz2";
}

function bunzip2(file) {
    const ext = extname(file);
    const newpath = basename(file, ext);
    exec(`bunzip2 ${file}`);
    return newpath;
}

function aesEncrypt(inFile, outFile, pass) {
    exec(`openssl enc -e -aes-256-cbc -pass pass:${pass} -in ${inFile} -out ${outFile}`);
    return outFile;
}

function aesDecrypt(inFile, outFile, pass) {
    exec(`openssl enc -d -aes-256-cbc -pass pass:${pass} -in ${inFile} -out ${outFile}`);
    return outFile;
}

// TEMP STORAGE

var deletionPending = [];

function cleanRandomFolders() {
    deletionPending.forEach(path => {
        console.log("Clean up: removing path [" + path + "]");
        rm("-Rf", path);
    });
}

function makeRandomFolder() {
  var outPath;
  const sysTmpPath = process.env.TMPDIR ||
      process.env.TMP ||
      process.env.TEMP ||
      (process.platform === "win32" ? "c:\\windows\\temp\\" : "/tmp/")
  
  do {
    outPath = Date.now() + Math.random() + '';
  } while(existsSync(sysTmpPath + outPath));

  mkdir('-p', sysTmpPath + outPath);
  deletionPending.push(sysTmpPath + outPath);

  return sysTmpPath + outPath;
}

function log(...args) {
    console.log(new Date().toJSON(), "|", ...args, "\n");
}

module.exports = {
    aesEncrypt,
    aesDecrypt,
    tar,
    unTar,
    bzip2,
    bunzip2,
    cleanRandomFolders,
    makeRandomFolder,
    log
};

package.json

{
  "name": "backup",
  "version": "1.0.0",
  "description": "",
  "main": "backup.js",
  "dependencies": {
    "bluebird": "^3.5.0",
    "google-cloud": "^0.55.0",
    "shelljs": "^0.7.7"
  },
  "devDependencies": {
    "eslint": "^3.19.0"
  },
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1"
  },
  "keywords": [],
  "author": "",
  "license": "ISC"
}

rotate.js

#!/usr/bin/env node

// npm install @google-cloud/storage bluebird

const config = require('./config.js');
const gCloudFileName = process.cwd() + '/gcloud.json';
const gConfig = require(gCloudFileName);

const { log } = require('./lib');
const { storage } = require('google-cloud');
global.Promise = require("bluebird");

var gcs = storage({
  projectId: gConfig.project_id,
  keyFilename: gCloudFileName
});

log("STARTING ROTATION", gConfig.project_id, config.backupName);

const bucket = gcs.bucket(config.gCloudBucket);

// Remove old backups
listFiles(bucket, config.bucketSubdir)
.then(files => {
  log(`GOT ${files.length} FILES`);

  if (!files || files.length < config.backupCopies) return;
  
  files = files.sort((file1, file2) => file2.metadata.timeCreated.localeCompare(file1.metadata.timeCreated));
  files = files.filter(file => parseInt(file.metadata.size) > 0)
  
  const cleanable = files.slice(config.backupCopies);

  log(`CLEANING THE OLDEST ${cleanable.length}`);
  return Promise.map(cleanable, file => {
    log('Cleaning', file.name);
    return bucket.file(file.name).delete();
  })
})
.then(result => {
  log("DONE", gConfig.project_id, config.backupName);
})
.catch(err => {
  log("ERROR: UNABLE TO ROTATE", err);
});

// AUX

function listFiles(bucket, prefix) {
  return new Promise((resolve, reject) => {
    var list = [];

    function readResponse(err, files, nextQuery, apiResponse) {
      if (err) return reject(err);
      list = list.concat(files);
      if (nextQuery) bucket.getFiles(nextQuery, readResponse);
      else resolve(list);
    }
    
    bucket.getFiles({
      maxResults: 50,
      prefix: prefix
    }, readResponse);
  });
}