Gensim Celery Issue Replication

Procfile

redis: redis-server
celery_worker: celery -A tasks worker -l info

requirements.txt

amqp==2.2.2
billiard==3.5.0.3
boto==2.48.0
boto3==1.7.0
botocore==1.10.0
bz2file==0.98
celery==4.1.0
certifi==2018.1.18
chardet==3.0.4
cymem==1.31.2
cytoolz==0.8.2
dill==0.2.7.1
docutils==0.14
en-core-web-sm==2.0.0
gensim==3.4.0
honcho==1.0.1
idna==2.6
jmespath==0.9.3
kombu==4.1.0
msgpack-numpy==0.4.1
msgpack-python==0.5.6
murmurhash==0.28.0
numpy==1.14.2
pathlib==1.0.1
plac==0.9.6
preshed==1.0.0
python-dateutil==2.6.1
pytz==2018.3
redis==2.10.6
regex==2017.4.5
requests==2.18.4
s3transfer==0.1.13
scipy==1.0.1
six==1.11.0
smart-open==1.5.7
spacy==2.0.11
termcolor==1.1.0
thinc==6.10.2
toolz==0.9.0
tqdm==4.20.0
ujson==1.35
urllib3==1.22
vine==1.1.4
wrapt==1.10.11

run.py

from tasks import most_sim

# run regular function
most_sim('cat')

# test task
r = most_sim('cat')

tasks.py

import gensim.downloader as gensim_api
from gensim.models import KeyedVectors
from celery import Celery

app = Celery('tasks', broker='redis://localhost:6379/0', backend='redis://localhost:6379/0')

# (Google News Vectors from gensim downloader) doesnt work ❌
# w2v_model_gn_api = gensim_api.load('word2vec-google-news-300')
# w2v_model_gn_api.init_sims(replace=True)

# (GloVe vectors from Gensim Downloader) doesnt work ❌
# w2v_model_glove_api = gensim_api.load('glove-wiki-gigaword-50')
# w2v_model_glove_api.init_sims(replace=True)

# (Google news Vectors as KeyedVectors from original source) works ✅
# wvpath = '/Users/pbaumgartner/data/GoogleNews-vectors-negative300.bin.gz'
# w2v_model_gn_kv = KeyedVectors.load_word2vec_format(wvpath, binary=True)
# w2v_model_gn_kv.init_sims(replace=True)

# (GloVe vectors as KeyedVectors from original source) works ✅
# glovewvpath = '/Users/pbaumgartner/data/glove.6B/gensim/glove.6B.50d.txt'
# w2v_model_glove_kv = KeyedVectors.load_word2vec_format(glovewvpath)
# w2v_model_glove_kv.init_sims(replace=True)

# (Glove Vectors from gensim downloader loaded into KeyedVectors) works ✅
# glovewvpath_gensim = '/Users/pbaumgartner/gensim-data/glove-wiki-gigaword-50/glove-wiki-gigaword-50.gz'
# w2v_model_glove_kv_gensim = KeyedVectors.load_word2vec_format(glovewvpath_gensim)
# w2v_model_glove_kv_gensim.init_sims(replace=True)

@app.task
def most_sim(word):
    print("hello")
    similarities = w2v_model_glove_kv.most_similar(word)
    return similarities