Google App Script for handling github notifications

email-filter.js

// email-filter.js
// Set this up by visiting https://script.google.com/home
// see below for 
// CONFIGURATION
// begin things you can change or should know about:

// list the teams you care about here:
// the script will search email messages for these specific
// mentions
var myTeams = [ "@github/pe-security-workflows" ]

// I find that these tags can get rather long & wordy
// so, here you can customize how labels get shortened.
//
// the script parses out github/ from team names and repos
// so, team mentions for @github/foo will appear as @foo
// and repository tags for [github/foo] will appear as [foo]
//
// example: "@long-team-name-here": "@shorter-name"
// "[extremely-long-repo-name]": "[shorter]"
var abbreviations = {
  "@pe-security-workflows": "@pe-sec-work",
  "@compliance-code-reviewers": "@compliance-rev",
  "@pe-security-workflows-reviewers": "@pe-sec-work-rev",
  "@experience-engineering-code": "@ee-code",
  "@experience-engineering": "@ee",
  "@engineering": "@eng",
  "@pe-coding-workflows": "@pe-code",
  "[github]": "[gh]",
  "[experience-engineering-code]": "[ee-code]",
  "[experience-engineering]": "[ee]",
  "[dependency-graph-api]": "[dep-graph]",
  "review": "rev",
  "mention": "@phillmv",
  "team": "tm"
}

// this variable controls whether the tags the script
// introduces are prefixed with the string "t:".
//
// This is useful if you're just trying shit out, and,
// if you have an existing filtering system whose labels
// overlap with the labels this one will autogenerate, you
// don't want to end up polluting your existing tags
var addTestPrefixToLabels = false;

// used for debugging
var debug = false;

// END CONFIGURATION

// When replacing this script with a newer version,
// delete everything below this line.
// ----------------------------------
// ----------------------------------
//
// GITHUB EMAIL FILTERING SCRIPT
// Google Apps Script that parses GitHub notifications and
// auto generates Gmail labels based on repo names, team mentions,
// pull request reviews, etc, with some configurable settings.
//
// by @phillmv, with some code from @mislav's labler.js
//
// ------------
// INSTALLATION
// ------------
//
// 1. Visit https://script.google.com/home while logged in
// to your GitHub Google account.
//
// 2. Create a New Script.
//
// 3. Copy and paste the following script into the Code.gs
// file that materializes within the Google Apps Script editor.
//
// 4. Go through the CONFIGURATION section below, and change
// the three variables to your taste and liking:
//  a) myTeams
//  b) abbreviations
//  c) addTestPrefixToLabels
//
// 5. Click the little floppy button to save it. I named
// mine "email-filter".
//
// 6. In the toolbar, next to the debug button, there is
// a dropdown with named functions. Select the "main" function.
//
// 7. Hit the Play button / Click Run > Run Function > main
//
// 8. This will take a while, depending on how many unread
// emails you have. Easily ten, twenty minutes; it may also time
// out, cos Google enforced script execution time lengths.
//
// If things don't seem to work properly, consult @phillmv
//
// 9. Open up your gmail. Go to Settings > Labels
// https://mail.google.com/mail/#settings/labels
// and scroll down till you find "been-processed-hide-this-label".
//
// Hide the label.
//
// 10. See if that everything works! I like to assign colours to
// diff labels.
//
// 11. If things are working, it's time to configure a trigger.
// Go back to the Apps Script project tab.
//
// Click Edit > Current project's triggers
//
// Set up to Run "main" every 10 or 15 minutes.
//
// You're done!
//
// ------------
// HOW IT WORKS
// ------------
//
// This script iterates over unread, github notification emails
// since the last time the script was invoked,
//
// looks at the first and last email in the thread,
//
// examines the email's Reason header to deduce why you're
// receiving it,
//
// parses out from the Message-ID the specific repository OR
// if it's a discussion, the team name being mentioned,
//
// then scans the body of the message for team mentions listed in
// the `myTeams` variable below, and if it's a PR tries to parse
// which team got tagged for a review.
//
// FINALLY, it takes all this information and tags the thread
// with information like:
// [github], @experience-engineering-code, pr



// end things that you can change or should know about

// From within Gmail's UI, you should hide this label
// when it pops up.
var beenProcessedLabel = "been-processed-hide-this-label";
var emailSearchQuery = "is:unread [email protected]";
var relevantHeadersRE = "(X-GitHub-Reason|Message-ID):";
var requestedReviewRE = /requested review from (@\S+)/;
var emailSearchCacheKey = "v6";

var teamSearchRE = RegExp(myTeams.map(function(t) { return "("+t+")" }).join("|"));

// i.e. <github/your-repo-here/{issue,pull,discussion}/1234
var messageIDDecomposeRE = /<github\/([^\/]+)\/([^\/]+)\/\S+@/

function main() {
  var threads,
    lastRunAt,
    offset = 0,
    perPage = 50,
    query = emailSearchQuery;

  lastRunAt = cache.getLastRun()

  if (lastRunAt) {
    lastRunAtDate = new Date(lastRunAt * 1000)
  } else {
    // if we can't detect a date, let's just process
    // all email from the last 45 days
    lastRunAtDate = new Date(new Date().setDate(new Date().getDate() - 45));
    lastRunAt = parseInt(lastRunAtDate / 1000);
  }
  query += " after:" + lastRunAt;
  Logger.log(lastRunAtDate)

  do {
    var threads = fetchUnreadMail(query, offset, perPage);
    Logger.log("Looking at " + threads.length + " threads");

    forEach(threads, function(thread, i) {
      var existingLabels = fetchAndCacheThreadLabels(thread);

      var labels = parseThread(thread, lastRunAtDate);
      // Logger.log("labels " + labels)

      addLabelsToThread(existingLabels, labels, thread);
    });

    offset += perPage;

  // used for debugging:
  // } while (offset == 0)
  } while (threads.length == perPage)

  // set cache for next run
  cache.setLastRun(new Date)
}

function parseThread(thread, lastRunAtDate) {
  var labels = new Set;
  var messages = thread.getMessages();
  // Logger.log(messages[0].getSubject())

  // right now, we only look at the very first message
  // most of the time,
  // getLabelsForMessage(labels, messages[0]);
  for(i = 0; i < messages.length; i++) {
    var msg = messages[i];
    if(msg.getDate() > lastRunAtDate) {
      getLabelsForMessage(labels, msg);
    }
  }
  return labels;
}

function getLabelsForMessage(labels, message) {
  var [headers, rawContent] = splitEmailMessage(message)
  // Logger.log(headers)

  var githubReason = headers["x-github-reason"]
  var messageID = headers["message-id"]

  applyMessageIDLabels(labels, messageID);
  switch(githubReason) {
    case "author":
      labels.add("mention");
      break;
    case "assigned":
      // labels.add("assigned");
      labels.add("mention");
      break;
    case "mention":
      labels.add("mention");
      break;
    case "team_mention":
      labels.add("team");
      applyTeamLabels(labels, messageID, rawContent);
      break;
    case "review_requested":
      labels.add("review")
      applyTeamLabels(labels, messageID, rawContent);
      break;
    case "security_alert":
      labels.add("security");
      break;
  }
}

function applyTeamLabels(labels, messageID, rawContent) {
  var teams = [],
    review_match = rawContent.match(requestedReviewRE),
    team_match = rawContent.match(teamSearchRE)

  // Logger.log(rawContent)

  if (review_match) {
    teams.push(review_match[1])
  }

  if (team_match) {
    teams.push(team_match[0])
  }

  forEach(teams, function(team_name) {
    [org, name] = team_name.split("/")
    if (name) {
      labels.add("@" + name)
    }
  })
}

// splits messageID to find repo name and whether
// it's a discussion or a PR
function applyMessageIDLabels(labels, messageID) {
  messageID_match = messageID.match(messageIDDecomposeRE);
  if(messageID_match == null) { return }

  [_, repo, type] = messageID_match

  if(!repo) { return }

  switch(type) {
    case "discussions":
      labels.add("@" + repo)
      break;
    case "issues":
      labels.add("["+repo+"]")
      break;
    case "pull":
      labels.add("["+repo+"]")
      labels.add("pr")
      break;
  }
}

function splitEmailMessage(message) {
  var headers = {},
    rawContent = message.getRawContent()
    rawHeaders = rawContent.split("\r\n\r\n", 2)[0]

  // re: headers we only care about two of them
  // so we find them and normalize their keys
  forEach(rawHeaders.split("\n"), function(line) {
    if(line.match(relevantHeadersRE)) {
      var match = line.match(/^(\S+):\s*(.*)/)
      if (match) {
        headers[match[1].toLowerCase()] = match[2]
      }
    }
  })

  return [headers, rawContent]
}

function fetchUnreadMail(query, offset, perPage) {
  var threads = GmailApp.search(query, offset, perPage)
  return threads;
}

// adding a label to a thread is very expensive
// and takes on the order of 0.1 to 0.3s per label
// going by the execution transcript.

// but looking up labels is very cheap:
// thread.getLabels takes 0.009s
// label.getName takes 0.003s

// over the lifetime of this script, the avg msg
// will already have a label, so we stand to save a lot
// of time
function addLabelsToThread(existingLabels, labels, thread) {
  forEach(labels.entries(), function(label) {
    var label_name = abbreviations[label] || label

    if(addTestPrefixToLabels) {
      label_name = "t:"+label;
    }

    if(!existingLabels.has(label_name)) {
      thread.addLabel(getLabel(label_name))
    }
  });

  // mark the thread as processed
  thread.addLabel(getLabel(beenProcessedLabel));
}

function fetchAndCacheThreadLabels(thread) {
  var labelCache = new Set,
    existingLabels = thread.getLabels();

  forEach(existingLabels, function(threadLabel) {
    labelCache.add(threadLabel.getName())
  });

  return labelCache;
}

// UTILS
function forEach(a, fn) {
  for (var i = 0; i < a.length; i++) if (fn(a[i], i) === false) break
}

function Set() {
  var store = {}

  return {
    add: function(item) {
      store[item] = true;
      return this;
    },
    entries: function() {
      return Object.keys(store);
    },
    has: function(item) {
      return store[item] || false;
    },
    toString: function() {
      return this.entries().toString();
    }
  }
}

cache = (function(){
  var userCache = CacheService.getUserCache(),
    cacheKey = emailSearchCacheKey,
    expiresIn = 60 * 60 * 2;

  return {
    getLastRun: function() {
      return userCache.get(cacheKey+"lastRunAt")
    }
  , setLastRun: function(date) {
      userCache.put(cacheKey+"lastRunAt", parseInt(date / 1000), expiresIn)
    }
  }
})()

getLabel = (function(){
  var labelIndex = null

  return function(name) {
    if (!labelIndex) {
      labelIndex = {}
      forEach(GmailApp.getUserLabels(), function(label){
        labelIndex[label.getName().toLowerCase()] = label
      })
    }

    var lower_name = name.toLowerCase();
    if(!labelIndex[lower_name]) {
      labelIndex[lower_name] = GmailApp.createLabel(name)
    }
    return labelIndex[lower_name]
  }
})()