samarthbhargav

<?xml version="1.0"?>
<configuration>
	<property>
	    <name>yarn.nodemanager.resource.memory-mb</name>
	    <value>8192</value>
	</property>
	<property>
	    <name>yarn.scheduler.maximum-allocation-mb</name>
	    <value>4096</value>
	</property>

samarthbhargav

# Hadoop Properties
export HADOOP_PREFIX=/usr/local/hadoop # installation location
export HADOOP_HOME=$HADOOP_PREFIX
export HADOOP_COMMON_HOME=$HADOOP_PREFIX
export HADOOP_CONF_DIR=$HADOOP_PREFIX/etc/hadoop
export HADOOP_HDFS_HOME=$HADOOP_PREFIX
export HADOOP_MAPRED_HOME=$HADOOP_PREFIX
export HADOOP_YARN_HOME=$HADOOP_PREFIX

export PATH=$PATH:$HADOOP_PREFIX/bin

samarthbhargav

# as seen in 6.00.2x, Week 6, Lecture 1, Video 4

def powerset(iterable):
    if len(iterable) == 0:
        return [[]]
    
    smaller = powerset(iterable[1:])
    withElem = []
    for s in smaller:
        withElem.append( s + [iterable[0]])

samarthbhargav

# source: http://www.huyng.com/posts/python-performance-analysis/
import time

class Timer(object):
    def __init__(self, verbose=False):
        self.verbose = verbose

    def __enter__(self):
        self.start = time.time()
        return self

samarthbhargav

from bs4 import BeautifulSoup
import requests
import shutil
import glob
from os import listdir
from os.path import isfile, join


def save_image(url, filename):
  response = requests.get(url, stream=True)

samarthbhargav

# source: https://www.safaribooksonline.com/library/view/python-cookbook-2nd/0596007973/ch09s03.html

import threading
class TestThread(threading.Thread):
    def _ _init_ _(self, name='TestThread'):
        """ constructor, setting initial variables """
        self._stopevent = threading.Event( )
        self._sleepperiod = 1.0
        threading.Thread._ _init_ _(self, name=name)
    def run(self):

samarthbhargav

python -c "import torch; print(torch.cuda.is_available())"

samarthbhargav

index_sets = {1, 2}

list_of_metrics = [
    ("ERR", err),
    ("MAP", average_precision),
    ("Recall@1",recall_at_1),
    ("Recall@5", recall_at_5),
    ("Recall@10", recall_at_10),
    ("Precision@1", precision_at_1),
    ("Precision@5", precision_at_5),

samarthbhargav

def pairwise_kl_divergence(mean1, var1, mean2, var2):
    # Ensure that variances are non-negative
    var1 = torch.clamp(var1, min=1e-10)
    var2 = torch.clamp(var2, min=1e-10)
    k = mean1.size(1)

    # shape = BZ_1xD
    logvar1 = torch.log(var1)
    # log determinant