giacaglia

def transform_entity(pb):
  from google.cloud.datastore.helpers import entity_from_protobuf
  entity = entity_from_protobuf(pb)
  retailer_id = entity.get('id', '')
  name = entity.get('name')
  category = entity.get(‘category’, ‘’)
  description = entity.get(‘description’, ‘’)
  image_link = entity.get(‘image’, ‘’)
  price = entity.get(‘price’, ‘0’)
  link = entity.get('url', '')

giacaglia

#!/bin/bash
python -m catalog_upload — project project_name — runner DataflowRunner \
    — staging_location $BUCKET/staging — temp_location $BUCKET/temp \
     — output $BUCKET/results/output — template_location $BUCKET/catalog_up \
     — requirements_file requirements.txt

giacaglia

def start_dataflow_pipeline(): 
  bucket = ‘fb_bucket’
  BODY = {
    “jobName”: ‘fb_
    catalog_upload’,
    “gcsPath”: “gs://{bucket}/catalog_up”.format(bucket=bucket),
    “environment”: {
      “tempLocation”: “gs://{bucket}/temp”.format(bucket=bucket),
      “zone”: “us-central1-f”
    }

giacaglia

{
 "Iris": 100,
 "MNIST": 100000,
 "Public SVNH": 1000000,
 "Champolion": 10000000,
 "ImageNet": 100000000,
 "HN transcriptor": 1000000000,
 "Seconds from birth to college graduation": 10000000000,
 "HN detector": 100000000000
}

giacaglia

import os
from datetime import datetime
import argparse
import torch.multiprocessing as mp
import torchvision
import torchvision.transforms as transforms
import torch
import torch.nn as nn
import torch.distributed as dist
from apex.parallel import DistributedDataParallel as DDP

giacaglia

class ConvNet(nn.Module):
    def __init__(self, num_classes=10):
        super(ConvNet, self).__init__()
        self.layer1 = nn.Sequential(
            nn.Conv2d(1, 16, kernel_size=5, stride=1, padding=2),
            nn.BatchNorm2d(16),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2, stride=2))
        self.layer2 = nn.Sequential(
            nn.Conv2d(16, 32, kernel_size=5, stride=1, padding=2),

giacaglia

def train(gpu, args):
	torch.manual_seed(0)
    model = ConvNet()
    model = nn.DataParallel(model)
    torch.cuda.set_device(gpu)
    model.cuda(gpu)
    batch_size = 100
    # define loss function (criterion) and optimizer
    criterion = nn.CrossEntropyLoss().cuda(gpu)
    optimizer = torch.optim.SGD(model.parameters(), 1e-4)

giacaglia

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-n', '--nodes', default=1,
                        type=int, metavar='N')
    parser.add_argument('-g', '--gpus', default=1, type=int,
                        help='number of gpus per node')
    parser.add_argument('-nr', '--nr', default=0, type=int,
                        help='ranking within the nodes')
    parser.add_argument('--epochs', default=2, type=int, 
                        metavar='N',

giacaglia

def train(gpu, args):
    ############################################################
    rank = args.nr * args.gpus + gpu	                          
    dist.init_process_group(                                   
    	backend='nccl',                                         
   		init_method='env://',                                   
    	world_size=args.world_size,                              
    	rank=rank                                               
    )                                                          
    ############################################################

giacaglia

python src/mnist-distributed.py -n 4 -g 8 -nr 0