email2liyang

import unittest
from datetime import datetime
import docker


class DocerBasedTest(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        container_name = "test-mysql-" + datetime.now().strftime('%y%m%d%H%M%s')
        client = docker.from_env()

email2liyang

from starbase import Connection
c = Connection("127.0.0.1","8000")
ratings = c.table("ratings")
if( ratings.exists()):
    print("Dropping existing table \n")
    ratings.drop()

ratings.create("rating")
print("parsing files")
ratingFile = open("/Users/ivan/Desktop/u.data","r")

email2liyang

ratings = LOAD '/user/maria_dev/ml-100k/u.data' AS (userId:int,movieId:int,rating:int,ratingTime:int);

metadata = LOAD '/user/maria_dev/ml-100k/u.item' USING PigStorage('|') 
AS (movieId:int,movieTitle:chararray,releaseDate:chararray,videoRelease:chararray,imdbLink:chararray);

nameLookup = FOREACH metadata GENERATE movieId,movieTitle,ToUnixTime(ToDate(releaseDate,'dd-MMM-yyyy')) AS releaseTime;

ratingByMovie = Group ratings By movieId;

avgRatings = FOREACH ratingByMovie GENERATE group as movieId,AVG(ratings.rating) AS avgRating;

email2liyang

from mrjob.job import MRJob
from mrjob.step import MRStep

class PopularityBreakdown(MRJob):
    def steps(self):
        return [
            MRStep(mapper=self.mapper_ratings,
                   reducer=self.reducer_ratings),
            MRStep(reducer=self.reducer_sortings)
        ]

email2liyang

#!/bin/sh

 yum update -y
 yum install -y wget git
 wget "http://www.haproxy.org/download/1.5/src/haproxy-1.5.3.tar.gz"
 yum groupinstall -y 'Development Tools'
 yum install -y openssl-devel
 yum install -y rpmdevtools pcre-devel
 rpmdev-setuptree 
 mv haproxy-1.5.3.tar.gz ~/rpmbuild/SOURCES/