tag_deepdress.map.py

import urllib.request
import json
from pymongo import MongoClient
import time
from multiprocessing import Pool

DEEP_DRESS_WGET = 'http://localhost/wget/?url='
DEEP_DRESS_PREDICT = 'http://localhost/predict/'

def wget_deep_dress(url):
  serviceurl = DEEP_DRESS_WGET + url
  response = urllib.request.urlopen(serviceurl)
  f = response.read()
  j = json.loads(f.decode('utf-8'))
  return(j['file_id'])

def predict_deep_dress(file_id, top_n):
  serviceurl = DEEP_DRESS_PREDICT + '?file_id=' + file_id + '&top_n=' + top_n
  response = urllib.request.urlopen(serviceurl)
  f = response.read()
  j = json.loads(f.decode('utf-8'))
  return(j)


client = MongoClient()
db = client.DeepDress
instagram = db.InstagramV4
matches = db.MatchesV4
# already tagged
ids = set([m['_id'] for m in matches.find()])

def match_and_save(i):
  try:
    if (i['_id'] not in ids):
      file_id = wget_deep_dress(i['image'])
      preds = predict_deep_dress(file_id, '3')
      result = matches.insert({'_id' : i['_id'], 'prediction' : preds})
      time.sleep(1) # Be nice
      return(1)
  except:
    return(0)


pool = Pool(processes=5) # Don't make this too high - rate limits
totals = pool.map(match_and_save, instagram.find())
print(sum(totals), len(totals))