Last active
July 11, 2018 17:06
-
-
Save DazWilkin/883dd885bfe4b301323220c406cb5944 to your computer and use it in GitHub Desktop.
Google Cloud Storage "exploder"
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/* globals exports, require */ | |
/* jshint node: true */ | |
/* jshint strict: false */ | |
/* jshint esversion: 6 */ | |
"use strict"; | |
const crc32 = require("fast-crc32c"); | |
const gcs = require('@google-cloud/storage')(); | |
const stream = require("stream"); | |
const unzipper = require("unzipper"); | |
const bucketName = { | |
src: "[[YOUR-ROOT]]-receive", | |
dst: "[[YOUR-ROOT]]-explode" | |
}; | |
exports.processZip = function(event, callback) { | |
const file = event.data; | |
console.log(`Processing Zip: ${file.name}`); | |
var srcBucket = gcs.bucket(bucketName.src); | |
var dstBucket = gcs.bucket(bucketName.dst); | |
var gcsSrcObject = srcBucket.file(file.name); | |
var prefix = (new Date()).getTime(); | |
gcsSrcObject.createReadStream() | |
.pipe(unzipper.Parse()) | |
.on("entry", function(entry) { | |
var filePath = entry.path; | |
var type = entry.type; | |
var size = entry.size; | |
console.log(`Found ${type}: ${filePath}`); | |
var gcsDstObject = dstBucket.file(`${prefix}/${filePath}`); | |
entry | |
.pipe(gcsDstObject.createWriteStream()) | |
.on('error', function(err) { | |
console.log(`File Error: ${err}`); | |
}) | |
.on('finish', function() { | |
console.log("File Extracted"); | |
}); | |
}) | |
.promise() | |
.then(() => { | |
console.log("Zip Processed"); | |
callback(); | |
},(err) => { | |
console.log(`Zip Error: ${err}`); | |
}); | |
}; |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment