Skip to content

Instantly share code, notes, and snippets.

@pepetox
Created January 11, 2018 19:55
Show Gist options
  • Select an option

  • Save pepetox/bef8b865d9a5c6aae460fde72e76ce2e to your computer and use it in GitHub Desktop.

Select an option

Save pepetox/bef8b865d9a5c6aae460fde72e76ce2e to your computer and use it in GitHub Desktop.
Google Cloud Function to publish data in pub/sub and from pub/sub to Big query
//gcloud beta functions deploy --trigger-topic data --stage-bucket XXXXXXX-ml-mlengine --project XXXXXX subscribeDataFlow
//gcloud beta functions deploy --trigger-http --stage-bucket XXXXXXX-ml-mlengine --project XXXXXX publishData
var pubsub = require('@google-cloud/pubsub')();
const bigquery = require('@google-cloud/bigquery')();
exports.publishData = function(req, res) {
var topic = pubsub.topic('data');
var publisher = topic.publisher();
var key = req.body.key || "test10";
var number = req.body.number || 100;
for (var i = number - 1; i >= 0; i--) {
var data = new Buffer(JSON.stringify({"key": key, "value": key+'--'+i}));
publisher.publish(data).then(function(messageId) {
console.log("correcto: "+messageId);
});
}
res.status(200).json({"result":"ok", "key": key, "number": number});
}
exports.subscribeDataFlow = function subscribe(event, callback) {
// The Cloud Pub/Sub Message object.
const pubsubMessage = event.data;
console.log('Procesado --> '+ Buffer.from(pubsubMessage.data, 'base64').toString());
var rawData = Buffer.from(pubsubMessage.data, 'base64').toString();
var data = JSON.parse(rawData);
var myDataset = bigquery.dataset('dataflow_example');
var tabla_uno = myDataset.table('dataflow_example_gcf');
// Import data into a table.
tabla_uno.insert({
key: data.key+"-gcf",
value: data.value
}, function(err, job) {});
callback();
};
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment