LiShuMing

PySpark on YARN in self-contained environments

Author: https://github.com/seanorama

Reference:

• https://community.cloudera.com/t5/Community-Articles/Using-VirtualEnv-with-PySpark/ta-p/245905
• https://community.cloudera.com/t5/Community-Articles/Running-PySpark-with-Conda-Env/ta-p/247551
• https://stackoverflow.com/questions/48770263/bundling-python3-packages-for-pyspark-results-in-missing-imports
• https://stackoverflow.com/questions/53524696/livy-no-yarn-application-is-found-with-tag-livy-batch-10-hg3po7kp-in-120-seconds

LiShuMing

1) install boost
 
brew install boost
 
it will ask to create link. just follow the instruction
 
2) install libevent
 
brew install libevent
 

LiShuMing

#! /bin/bash
wget https://github.com/google/protobuf/releases/download/v2.6.1/protobuf-2.6.1.tar.gz
tar xzf protobuf-2.6.1.tar.gz
cd protobuf-2.6.1
sudo apt-get update
sudo apt-get install build-essential
sudo ./configure
sudo make
sudo make check
sudo make install 

LiShuMing

source_agent.sources = syslog_tail
source_agent.channels = memoryChannel
source_agent.sinks = kafka_sink

# http://flume.apache.org/FlumeUserGuide.html#exec-source
source_agent.sources.syslog_tail.type = exec
source_agent.sources.syslog_tail.command = tail -F /var/log/messages
source_agent.sources.syslog_tail.batchSize = 1
source_agent.sources.syslog_tail.channels = memoryChannel

LiShuMing

input {
  lumberjack {
    # The port to listen on
    port => 5043

    # The paths to your ssl cert and key
    ssl_certificate => "/etc/pki/tls/certs/logstash-forwarder/logstash-forwarder.crt"
    ssl_key => "/etc/pki/tls/private/logstash-forwarder/logstash-forwarder.key"
    
    # default type, but this will already be set by logstash-forwarder anyways

LiShuMing

#!/usr/bin/env python

"""Simple HTTP Server With Upload.

This module builds on BaseHTTPServer by implementing the standard GET
and HEAD requests in a fairly straightforward manner.

"""

LiShuMing

package main

import (
	"bufio"
	"encoding/csv"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path/filepath"

LiShuMing

// Use Gists to store code you would like to remember later on
console.log(window); // log the "window" object to the console