lambdaofgod

{% add --template=nocode.tpl to nbconvert command %}

{%- extends 'full.tpl' -%}
{% block input_group %}
{%- endblock input_group %}

lambdaofgod

import $ivy.`org.slf4j:slf4j-api:1.7.21`
import $ivy.`org.scalanlp:breeze_2.11:0.12`
import $ivy.`org.scalanlp:breeze-natives_2.11:0.12`

lambdaofgod

from bitarray import bitarray

# see https://nlp.stanford.edu/IR-book/html/htmledition/gamma-codes-1.html

def gamma_code(n):
  binary_n = format(n, 'b')
  binary_offset = binary_n[1::]
  unary_length = bitarray(True for i in range(len(binary_offset))) + bitarray([False])
  return bitarray(unary_length), bitarray(binary_offset)

lambdaofgod

from toolz import compose
from google.colab import auth
from googleapiclient.discovery import build
import io
from googleapiclient.http import MediaIoBaseDownload
import pickle


"""
  Load (pickled) data files from Google Drive

lambdaofgod

lambdaofgod

pip install gensim seaborn wordcloud rank_bm25
pip install git+https://github.com/lambdaofgod/mlutil

lambdaofgod

import fire
import tqdm
from concurrent.futures import ProcessPoolExecutor
import pandas as pd
import skimage.io
import os
from PIL import Image
import requests
from io import BytesIO
import mlutil.parallel

lambdaofgod

import requests
from PIL import Image
from io import BytesIO
import numpy as np


headers = {
    'user-agent':
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
}

lambdaofgod

df = read.table('CH01PR20.txt', header=FALSE)

x = df[,2]
y = df[,1]

n = dim(df)[1]

plot(x, y)

lambdaofgod

from elasticsearch_dsl import Search


def get_everything_from_index(es, index):
    search_results = Search(index=index).using(es).scan()
    for hit in search_results:
        yield hit.to_dict()