Search twitter archive in a web worker

-twitter-archive-search-webworker.md

Twitter Archive Search in Web Worker

Twitter's default offline archive search is slow as heck. If you do it in a web worker, the search itself is actually pretty fast.

Instructions

• Download search.html and worker.js to the root directory of your twitter archive (the same directory as index.html)
• Open search.html in browser
• Open JS console (probably ctrl shift j)
• Type search("insert some regex here") and press enter
• Results will be output to the console

search.html

<script>
function createWebWorkerFromFunction(f) {
  var blobContents = ['(', f.toString(), ')();'];
  var blob = new Blob(blobContents, { type: 'application/javascript'});
  var blobUrl = URL.createObjectURL(blob);
  var worker = new Worker(blobUrl);
  URL.revokeObjectURL(blobUrl);
  return worker;
}

var worker = createWebWorkerFromFunction(function() {
  self.onmessage = function(e) {
    self.baseUrl = e.data.baseUrl;
    self.importScripts(self.baseUrl + 'worker.js');
  };
});

var baseUrl = (function() {
  var parts = document.location.href.split('/');
  parts[parts.length - 1] = '';
  return parts.join('/');
}());

worker.onmessage = function(e) {
  console.log(e.data.created_at + ' ' + e.data.text);
}

// Send the worker the path to load the script from
worker.postMessage({ baseUrl: baseUrl });

function search(searchTerm) {
  worker.postMessage(searchTerm);
};
</script>

worker.js

self.Grailbird = self.Grailbird || {};
self.Grailbird.data = self.Grailbird.data || {};

self.onmessage = function(e) {
  search(e.data);
};

importScripts(self.baseUrl + "data/js/tweet_index.js");

function getTweets(obj) {
  return self.Grailbird.data[obj.var_name] || (function() {
    importScripts(self.baseUrl + obj.file_name);
    return self.Grailbird.data[obj.var_name];
  }());
};

function search(regexString) {
  var regex = new RegExp(regexString, 'im');

  self.tweet_index.forEach(function(t) {
    getTweets(t).forEach(function(tweet) {
      if (tweet.text.match(regex)) {
        self.postMessage(tweet);
      }
    });
  });
};