DrSnowbird

import org.apache.spark.ml.feature.{CountVectorizer, RegexTokenizer, StopWordsRemover}
import org.apache.spark.mllib.clustering.{LDA, OnlineLDAOptimizer}
import org.apache.spark.mllib.linalg.Vector

import sqlContext.implicits._

val numTopics: Int = 100
val maxIterations: Int = 100
val vocabSize: Int = 10000

DrSnowbird

//
//  ========================================================================
//  Copyright (c) 1995-2015 Mort Bay Consulting Pty. Ltd.
//  ------------------------------------------------------------------------
//  All rights reserved. This program and the accompanying materials
//  are made available under the terms of the Eclipse Public License v1.0
//  and Apache License v2.0 which accompanies this distribution.
//
//      The Eclipse Public License is available at
//      http://www.eclipse.org/legal/epl-v10.html

DrSnowbird

Producer

Setup
bin/kafka-topics.sh --zookeeper esv4-hcl197.grid.linkedin.com:2181 --create --topic test-rep-one --partitions 6 --replication-factor 1
bin/kafka-topics.sh --zookeeper esv4-hcl197.grid.linkedin.com:2181 --create --topic test --partitions 6 --replication-factor 3

Single thread, no replication

bin/kafka-run-class.sh org.apache.kafka.clients.tools.ProducerPerformance test7 50000000 100 -1 acks=1 bootstrap.servers=esv4-hcl198.grid.linkedin.com:9092 buffer.memory=67108864 batch.size=8196

DrSnowbird

#!/bin/bash -x
## ref: https://github.com/NVIDIA/nvidia-docker
## To use:
## wget -O - -q 'https://gist.githubusercontent.com/DrSnowbird/b0932c4fbd4c06339754bb8b68bffbb6/raw/69d86a81adab803b76221e42a8e59f6656399630/nvidia-docker-install-ubuntu-xenial.sh' | sudo bash

#### ---- Only if you have older v1.0, then you need to remove it using code below ----
old = 0
if [ $old -eq 1 ]; then
  ## If you have nvidia-docker 1.0 installed: we need to remove it and all existing GPU containers
  #docker volume ls -q -f driver=nvidia-docker | xargs -r -I{} -n1 docker ps -q -a -f volume={} | xargs -r docker rm -f

DrSnowbird

#!/bin/bash -x
## Launch Jupyter and Tensorboard
sudo nvidia-docker run --rm --name tf1 -p 8888:8888 -p 6006:6006 gcr.io/tensorflow/tensorflow:latest-gpu jupyter notebook --allow-root

## If the above command shows a line like:
# http://localhost:8888/?token=c8caba947dfd4c97414447c074325faf399cf8a157d0ce2f
# …you’re in business. Find the external IP address of your GCE instance and connect to it on port 8888, 
# e.g. http://EXTERNAL_IP:8888/, type in the (similar) token from your console, and 
# you have a GPU enabled Jupyter notebook running Tensorflow.

DrSnowbird

#!/bin/bash -x

# ref: https://github.com/dcos/dcos-vagrant

if [ ! -d dcos-vagrant ]; then
    git clone https://github.com/dcos/dcos-vagrant
fi
cd dcos-vagrant

if [ ! -s VagrantConfig.yaml ]; then

DrSnowbird

#!/bin/bash 

# ------------------------------------ 
# maintainer: [email protected]
# license: Apache License Version 2.0
# ------------------------------------ 

ERROR_NO_ARGS_PROVIDED=99
ERROR_ERROR_ARGS_PROVIDED=98

DrSnowbird

#!/bin/bash -x

if [ $# -lt 3 ]; then
    echo "ERROR: --- Usage: $0 <config_file> <key> <value> [<delimiter>] [<prefix-pattern>]"
    echo "e.g."
    echo './replaceKeyValue.sh \"elasticsearch.yml\" \"^network.host\" \"172.20.1.92\" \":\" \"# network\" '
    exit 1
fi

CONFIG_FILE=${1}

DrSnowbird

#!/bin/bash

# ------------------------------------ 
# maintainer: [email protected]
# license: Apache License Version 2.0
# ------------------------------------ 

# This feature is not yet available for native Docker Windows containers.
# On Linux and when using Docker for Mac or Docker for Windows or 
# Docker Toolbox, 

DrSnowbird

## find ports being used
sudo netstat -antup |awk '{print $4;}'|cut -d':' -f2 |sort -u