aouyang1

#!/bin/python

import sys
import os
import requests
import json
from pprint import pprint
from urllib.parse import urlparse, parse_qs

import boto3

aouyang1

# Launch instances on AWS
peg up namenode.yml  
peg up datanodes.yml

# Store cluster information
peg fetch hadoop-cluster

# Enable passwordless SSH and install Flink, Zookeeper and Kafka
peg install hadoop-cluster ssh  
peg install hadoop-cluster flink

aouyang1

# Launch instances on AWS
peg up namenode.yml  
peg up datanodes.yml

# Store cluster information
peg fetch hadoop-cluster

# Enable passwordless SSH and install Hadoop, Spark and Alluxio
peg install hadoop-cluster ssh  
peg install hadoop-cluster hadoop

aouyang1

# Launch instances on AWS
peg up namenode.yml  
peg up datanodes.yml

# Store cluster information
peg fetch hadoop-cluster

# Enable passwordless SSH and install Hadoop, Zookeeper, Kafka, Spark and Cassandra
peg install hadoop-cluster ssh  
peg install hadoop-cluster hadoop

aouyang1

purchase_type: spot
subnet_id: subnet-3a78835f
price: 0.13
num_instances: 3
key_name: insight-cluster
security_group_ids: sg-9206aaf7
instance_type: m4.large
tag_name: hadoop-cluster
vol_size: 100
role: worker

aouyang1

purchase_type: on_demand          
subnet_id: subnet-3a78835f        
num_instances: 1                  
key_name: insight-cluster         
security_group_ids: sg-9206aaf7   
instance_type: m4.large           
tag_name: hadoop-cluster          
vol_size: 100                     
role: master 
use_eips: true 

aouyang1

#!/bin/bash

# Launch instances on AWS
peg up namenode.yml
peg up datanodes.yml

# Store cluster information
peg fetch hadoop-cluster

# Enable passwordless SSH and install Hadoop

aouyang1

#!/bin/bash

# create local dummy file to place on HDFS
echo "Hello this will be my first distributed and fault-tolerant data set\!"" | cat >> my_file.txt

# list directories from top level of HDFS. This should display nothing but a temp directory
hdfs dfs -ls /

# create /user directory on HDFS
hdfs dfs -mkdir /user

aouyang1

from flask import Flask, render_template

app = Flask(__name__)

@app.route('/')
@app.route('/index')
def index(chartID = 'chart_ID', chart_type = 'bar', chart_height = 350):
	chart = {"renderTo": chartID, "type": chart_type, "height": chart_height,}
	series = [{"name": 'Label1', "data": [1,2,3]}, {"name": 'Label2', "data": [4, 5, 6]}]
	title = {"text": 'My Title'}