jeongho

--- raid 1
[root@localhost ~]# yum install -y mdadm

[root@localhost ~]# lsblk
NAME                        MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
sr0                          11:0    1 1024M  0 rom  
sda                           8:0    0   20G  0 disk 
├─sda1                        8:1    0  500M  0 part /boot
└─sda2                        8:2    0 19.5G  0 part 
  ├─VolGroup-lv_root (dm-0) 253:0    0 17.6G  0 lvm  /

jeongho

https://www.evernote.com/shard/s31/sh/039cec4d-694c-4a66-b2cb-f5ba6976974b/d31566b087605f32361007b3c6555fb4

jeongho

#!/usr/bin/env bash

# https://cloudera.github.io/cm_api/docs/quick-start/
# https://www.cloudera.com/documentation/enterprise/latest/topics/cm_intro_api.html

set -o pipefail
set +o errexit
set -x

username=${username:-admin}

jeongho

#https://www.erlang-solutions.com/resources/download.html
#READ Installation using repository
#http://elixir-lang.org/install.html#raspberry-pi

# verify installation
which erl
/usr/local/bin/erl

# check version
erl -eval 'erlang:display(erlang:system_info(otp_release)), halt().'  -noshell

jeongho

export SPARK_CLASSPATH=$(hadoop classpath)
export SPARK_HOME=/opt/cloudera/parcels/CDH/lib/spark

spark-submit --class org.apache.spark.examples.SparkPi \
--master yarn \
--deploy-mode client \
$SPARK_HOME/lib/spark-examples.jar 10

spark-submit --class org.apache.spark.examples.SparkPi \
--master yarn \

jeongho

http://www.artificialworlds.net/blog/2015/12/07/java-https-to-a-server-with-a-self-signed-certificate/

cat << EOF > Get.java
import java.net.URL;
public class Get
{
   public static void main( String[] args ) throws Exception
   {
       try
       {

jeongho

https://www.cloudera.com/documentation/director/latest/topics/director_security_socks.html

Step 1: Set Up a SOCKS Proxy Server with SSH
nohup ssh -i "your-key-file.pem" -CND 8157 ec2-user@instance_running_director_server &

Step 2: Configure Your Browser to Use the Proxy
Setting Up SwitchyOmega on the Google Chrome Browser
proxy autoconfig:

function regExpMatch(url, pattern) {    

jeongho

ssh -i your_key_file -o UserKnownHostsFile=/dev/null \ 
 -o CheckHostIP=no -o StrictHostKeyChecking=no user@ip_address

https://cloud.google.com/solutions/connecting-securely#socks-proxy-over-ssh
gcloud compute ssh example-instance \
    --project my-project \
    --zone us-central1-a \
    --ssh-flag="-D" \
    --ssh-flag="1080" \
    --ssh-flag="-N"

jeongho

# https://github.com/ehiggs/spark-terasort
# https://samanaghazadeh.wordpress.com/2015/04/18/terasort-equivalent-for-apache-spark/

10gb
application_1492200106363_0096	TeraGen (10GB)	2017/04/14 20:33:33	2017/04/14 20:34:41	1.1 min	hdfs	2017/04/14 20:34:41
application_1492200106363_0098	TeraSort	2017/04/14 20:46:55	2017/04/14 20:48:07	1.2 min	hdfs	2017/04/14 20:48:07
application_1492200106363_0099	TeraValidate	2017/04/14 20:48:51	2017/04/14 20:49:20	29 s	hdfs	2017/04/14 20:49:20

100gb
application_1492200106363_0097	TeraGen (100GB)	2017/04/14 20:35:37	2017/04/14 20:48:15	13 min	hdfs	2017/04/14 20:48:15

jeongho

Latency Comparison Numbers
--------------------------
L1 cache reference                           0.5 ns
Branch mispredict                            5   ns
L2 cache reference                           7   ns                      14x L1 cache
Mutex lock/unlock                           25   ns
Main memory reference                      100   ns                      20x L2 cache, 200x L1 cache
Compress 1K bytes with Zippy             3,000   ns        3 us
Send 1K bytes over 1 Gbps network       10,000   ns       10 us
Read 4K randomly from SSD*             150,000   ns      150 us          ~1GB/sec SSD