Uploading images to Cloud Storage via the web and identifying them with Google Cloud Vision API

README.md

Uploading images to Cloud Storage via the web and identifying them with Google Cloud Vision API

If you're trying to do this, you came to the right place!

Watch this code work in real time: https://twitter.com/CodingDoug/status/945035556555186176

Setup

These instructions assume that you already have a Firebase project, and billing is enabled. Billing is required to use the Vision API.

1. Create a project workspace with the Fireabse CLI with Hosting and Functions enabled (with TypeScript).

2. Allow global read and write access to Realtime Database and Storage. (But don't do this in production!!)

3. Copy index.html and drop.js to your public web content folder.

4. Copy index.ts to the functions/src folder.

5. npm install firebase-functions firebase-admin @google-cloud/vision

6. Deploy Hosting and Functions with firebase deploy

7. Navigate to the site URL you're given.

8. Drop a file containing an image (PNG, JPG, GIF, etc)

9. Watch magic happen.

What's happening here?

1. User drags an image to the browser.

2. A drop event triggers an upload of that file to Cloud Storage.

3. When the upload is compete, the client pushes a record into Realtime Database with the image download URL.

4. A Cloud Function also triggers when the upload is complete.

• It calls the Google Cloud Vision API with a reference to the file in storage.
• It writes the results of the API (descriptive labels) to the database.

5. The UI in the browser is automatically updated with the results of the API.

Helpful documentation

• https://developer.mozilla.org/en-US/docs/Web/API/HTML_Drag_and_Drop_API
• https://firebase.google.com/docs/storage/web/upload-files
• https://firebase.google.com/docs/functions/gcp-storage-events
• https://firebase.google.com/docs/database/web/read-and-write
• https://cloud.google.com/nodejs/docs/reference/vision/latest/

drop.js

// Copyright 2017 Google LLC.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

'use strict'

document.addEventListener("DOMContentLoaded", function(event) {
    const dropEl = document.getElementById('drop')
    const logEl = document.getElementById('log')
    const templateEl = document.getElementById('template')
    templateEl.remove()
    templateEl.style.display = "block"

    // Set up realtime display of image upload and vision data
    const imagesRef = firebase.database().ref('images')
    function updateEntry(snapshot) {
        const data = snapshot.val()
        let entryEl = document.getElementById(snapshot.key)
        if (!entryEl) {
            entryEl = templateEl.cloneNode(true)
            entryEl.id = snapshot.key
            logEl.insertBefore(entryEl, logEl.firstChild)
        }
        if (data.url) {
            const imgEl = entryEl.querySelector("img")
            imgEl.src = data.url
        }
        const urlEl = entryEl.querySelector(".url")
        urlEl.href = data.url
        urlEl.textContent = data.url
        const labelsEl = entryEl.querySelector(".labels")
        labelsEl.textContent = data.labels
    }
    imagesRef.on('child_added', updateEntry)
    imagesRef.on('child_changed', updateEntry)

    // Set up a drop target to upload images to Cloud Storage
    dropEl.addEventListener('drop', event => {
        event.preventDefault()
        dropEl.classList.remove('green')
        const files = event.dataTransfer.files
        for (let i = 0; i < files.length; i++) {
            const file = files[i]
            const newImageRef = imagesRef.push({ seasons: 'greetings' })
            const storagePath = '/images/' + newImageRef.key + file.name.substr(file.name.lastIndexOf('.'))
            firebase.storage().ref(storagePath).put(file)
            .then(storageSnapshot => {
                return storageSnapshot.ref.getDownloadURL()
            })
            .then(url => {
                return newImageRef.update({ url: url, seasons: null })
            })
            .catch(error => {
                console.error(error)
                entryEl.remove()
            })
        }
    })
    dropEl.addEventListener('dragover', event => {
        event.preventDefault()
    })
    dropEl.addEventListener('dragenter', event => {
        event.preventDefault()
        dropEl.classList.add('green')
    })
    dropEl.addEventListener('dragleave', event => {
        event.preventDefault()
        dropEl.classList.remove('green')
    })
})

index.html

<!DOCTYPE html>
<html>
  <head>
    <meta charset="utf-8">
    <title>What's that thing?</title>
    <script defer src="/__/firebase/5.5.3/firebase-app.js"></script>
    <script defer src="/__/firebase/5.5.3/firebase-database.js"></script>
    <script defer src="/__/firebase/5.5.3/firebase-storage.js"></script>
    <script defer src="/__/firebase/init.js"></script>
    <script src="drop.js"></script>
    <style>
      #drop * { pointer-events: none }
      .green { background-color: lightgreen }
    </style>
  </head>
  <body style="font-family: Roboto, sans-serif; font-size: 16px; width: 2000px">
    <div id="drop" style="border: 1px solid red; padding: 8px 24px 8px 24px; display: inline-block; margin-bottom: 4px;">
      <span>Drop images here 🎅🎄🎁🤶</span>
    </div>
    <div id="log">
      <div id="template" class="entry" style="margin-bottom: 2px; display: none">
        <img style="width: 50px; height: 50px; object-fit: cover" src="https://mir-s3-cdn-cf.behance.net/project_modules/disp/585d0331234507.564a1d239ac5e.gif"/>
        <div style="display: inline-block; vertical-align: top">
          <table style="border: 0px">
            <tr><td style="text-align: right">URL: </td><td><a class="url" href="https://asdf/" style="text-overflow: ellipsis; overflow: hidden; white-space: nowrap">http://howdyhowdyhowdyhowdyhowdyhowdyhowdyhowdyhowdyhowdyhowdyhowdyhowdyhowdyhowdyhowdyhowdyhowdyhowdy/howdyhowdyhowdyhowdyhowdyhowdyhowdyhowdyhowdyhowdyhowdyhowdyhowdyhowdyhowdyhowdyhowdyhowdyhowdyhowdyhowdyhowdyhowdy</a></td></tr>
            <tr><td style="text-align: right">Labels: </td><td class="labels" style="color: green"></td></tr>
          </table>
        </div>
      </div>
    </div>
  </body>
</html>

index.ts

// Copyright 2017 Google LLC.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

import * as functions from 'firebase-functions'
import * as admin from 'firebase-admin'
import * as path from 'path'
const { ImageAnnotatorClient } = require('@google-cloud/vision')

admin.initializeApp()

export const onUploadImage = functions.storage.object().onFinalize(async object => {
    const parts = path.parse(object.name)

    if (parts.root !== '' || parts.dir !== 'images') {
        return null
    }

    // Pass the Cloud Storage URL directly to the Cloud Vision API
    const client = new ImageAnnotatorClient()
    const inStorage = `gs://${object.bucket}/${object.name}`

    const [ results ] = await client.labelDetection(inStorage)
    const labels = results.labelAnnotations
    // Update the database with a comma-separated list of labels
    await admin.database().ref(`images/${parts.name}`).update({
        labels: labels.map(label => label.description).join(', ')
    })
})