mrm8488

module['exports'] = function highFive(hook) {

  // hook.io has a range of node modules available - see
  // https://hook.io/modules.
  // We use request (https://www.npmjs.com/package/request) for an easy way to
  // make the HTTP request.
  var request = require('request');

  // The parameters passed in via the slash command POST request.
  var params = hook.params;

mrm8488

ServiceWorker code to cache TensorFlow model shards.

One of the problems I have when testing giant TensorFlow models in TensorFlow.js is that they're huge (like 500 MB) and they take forever to download, every time I refresh the page. This is how I setup my ServiceWorker code so that at least in testing I only have to download the model once, and then it's saved in the cache for the next time.

mrm8488

mrm8488

#!/bin/sh
set -x

# == Swarm training (alpha release) ==

# Setup:
#
#   git clone https://github.com/shawwn/gpt-2
#   cd gpt-2
#   git checkout dev-shard

mrm8488

var mongoose = require('mongoose');
 
mongoose.connect('mongodb://localhost/test');
 
var db = mongoose.connection;
 
db.on('error', function() {
  return console.error.bind(console, 'connection error: ');
});
 

mrm8488

from torch.utils.data import IterableDataset

class CustomIterableDataset(IterableDataset):

    def __init__(self, filename):

        #Store the filename in object's memory
        self.filename = filename

        #And that's it, we no longer need to store the contents in the memory

mrm8488

#Creating the iterable dataset object
dataset     = CustomIterableDataset('path_to/somefile')
#Creating the dataloader
dataloader  = DataLoader(dataset, batch_size = 64)

for data in dataloader:
    #Data is a list containing 64 (=batch_size) consecutive lines of the file
    print(len(data)) #[64,]
    
    #We still need to separate the text and labels from each other and preprocess the text

mrm8488

class CustomIterableDatasetv1(IterableDataset):

    def __init__(self, filename):

        #Store the filename in object's memory
        self.filename = filename

        #And that's it, we no longer need to store the contents in the memory

    def preprocess(self, text):

mrm8488

dataset = CustomIterableDatasetv1('path_to/somefile')
dataloader = DataLoader(dataset, batch_size = 64)

for X, y in dataloader:
    print(len(X)) # 64
    print(y.shape) # (64,)

    ### Do something with X and y

    ###

mrm8488

class CustomIterableDatasetv2(IterableDataset):

    def __init__(self, filename_en, filename_gm):

        #Store the filenames in object's memory
        self.filename_en = filename_en
        self.filename_gm = filename_gm

        #And that's it, we no longer need to store the contents in the memory