jeongho

#!/usr/bin/env bash

# install with anacona
# https://www.tensorflow.org/install/install_mac#installing_with_anaconda
conda create -n tensorflow
source activate tensorflow

#Python 2.7
TF_PYTHON_URL=https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.3.0-py2-none-any.whl
#Python 3.4, 3.5, or 3.6

jeongho

echo "<h1>body</h1>" | /bin/mailx \
-S smtp=smtp://aspmx.l.google.com \
-S [email protected] \
-s 'Subject [$(date -u %Y-%m-%d)] Hello World \nContent-Type: text/html\n' [email protected] 

jeongho

Latency Comparison Numbers
--------------------------
L1 cache reference                           0.5 ns
Branch mispredict                            5   ns
L2 cache reference                           7   ns                      14x L1 cache
Mutex lock/unlock                           25   ns
Main memory reference                      100   ns                      20x L2 cache, 200x L1 cache
Compress 1K bytes with Zippy             3,000   ns        3 us
Send 1K bytes over 1 Gbps network       10,000   ns       10 us
Read 4K randomly from SSD*             150,000   ns      150 us          ~1GB/sec SSD

jeongho

# https://github.com/ehiggs/spark-terasort
# https://samanaghazadeh.wordpress.com/2015/04/18/terasort-equivalent-for-apache-spark/

10gb
application_1492200106363_0096	TeraGen (10GB)	2017/04/14 20:33:33	2017/04/14 20:34:41	1.1 min	hdfs	2017/04/14 20:34:41
application_1492200106363_0098	TeraSort	2017/04/14 20:46:55	2017/04/14 20:48:07	1.2 min	hdfs	2017/04/14 20:48:07
application_1492200106363_0099	TeraValidate	2017/04/14 20:48:51	2017/04/14 20:49:20	29 s	hdfs	2017/04/14 20:49:20

100gb
application_1492200106363_0097	TeraGen (100GB)	2017/04/14 20:35:37	2017/04/14 20:48:15	13 min	hdfs	2017/04/14 20:48:15

jeongho

ssh -i your_key_file -o UserKnownHostsFile=/dev/null \ 
 -o CheckHostIP=no -o StrictHostKeyChecking=no user@ip_address

https://cloud.google.com/solutions/connecting-securely#socks-proxy-over-ssh
gcloud compute ssh example-instance \
    --project my-project \
    --zone us-central1-a \
    --ssh-flag="-D" \
    --ssh-flag="1080" \
    --ssh-flag="-N"

jeongho

https://www.cloudera.com/documentation/director/latest/topics/director_security_socks.html

Step 1: Set Up a SOCKS Proxy Server with SSH
nohup ssh -i "your-key-file.pem" -CND 8157 ec2-user@instance_running_director_server &

Step 2: Configure Your Browser to Use the Proxy
Setting Up SwitchyOmega on the Google Chrome Browser
proxy autoconfig:

function regExpMatch(url, pattern) {    

jeongho

http://www.artificialworlds.net/blog/2015/12/07/java-https-to-a-server-with-a-self-signed-certificate/

cat << EOF > Get.java
import java.net.URL;
public class Get
{
   public static void main( String[] args ) throws Exception
   {
       try
       {

jeongho

export SPARK_CLASSPATH=$(hadoop classpath)
export SPARK_HOME=/opt/cloudera/parcels/CDH/lib/spark

spark-submit --class org.apache.spark.examples.SparkPi \
--master yarn \
--deploy-mode client \
$SPARK_HOME/lib/spark-examples.jar 10

spark-submit --class org.apache.spark.examples.SparkPi \
--master yarn \

jeongho

#https://www.erlang-solutions.com/resources/download.html
#READ Installation using repository
#http://elixir-lang.org/install.html#raspberry-pi

# verify installation
which erl
/usr/local/bin/erl

# check version
erl -eval 'erlang:display(erlang:system_info(otp_release)), halt().'  -noshell

jeongho

#!/usr/bin/env bash

# https://cloudera.github.io/cm_api/docs/quick-start/
# https://www.cloudera.com/documentation/enterprise/latest/topics/cm_intro_api.html

set -o pipefail
set +o errexit
set -x

username=${username:-admin}