kumar-de

# to list all ports that are used
sudo netstat -ntlp | grep LISTEN

# you can obtain a specific port using the following command
sudo netstat -ntlp | grep :8080

# when you execute the command above you will see some thing like that
tcp        0      0 0.0.0.0:27370           0.0.0.0:*               LISTEN      4394/skype      
tcp        0      0 127.0.1.1:53            0.0.0.0:*               LISTEN      2216/dnsmasq    
tcp        0      0 127.0.0.1:631           0.0.0.0:*               LISTEN      4912/cupsd      

kumar-de

Remove existing version, if any

sudo apt-get remove scala-library scala

Change version, if necessary: e.g. 2.11.12

sudo wget www.scala-lang.org/files/archive/scala-2.11.8.deb

kumar-de

Install openvpn, its network-manager and network-manager-gnome

sudo apt-get install openvpn network-manager-openvpn network-manager-openvpn-gnome

kumar-de

apk add python wget py-pip python-dev gcc musl-dev libxml2-dev libxslt-dev

echo "Installing opcua via pip may take longer than expected..."

pip install setuptools opcua

kumar-de

Parquet

val sqlContext = new org.apache.spark.sql.SQLContext(sc)

val df = sqlContext.read.parquet("/path/to/file/without/hdfs://")

df.printSchema

df.count

kumar-de

val conf = sc.hadoopConfiguration
val fs = org.apache.hadoop.fs.FileSystem.get(conf)
val exists = fs.exists(new org.apache.hadoop.fs.Path("/path/on/hdfs")) // File or directory

val sequenceFiles1 = htu.getDFSCluster.getFileSystem.listStatus(new Path(outputFileDirforCrashes)).filter(_.isDirectory).map(_.getPath.toString)
        sequenceFiles1.foreach(dirPath=>{
 val files = htu.getDFSCluster.getFileSystem.listStatus(new Path(dirPath)).filter(_.isFile).map(_.getPath.toString)

kumar-de

CREATE EXTERNAL TABLE table_name (
    vehicle_id String,
    start_time String,
    av0 Double
)
STORED AS parquet
LOCATION '/path/in/hdfs/without//hdfs://'; 

kumar-de

Create a fat jar using Maven

<?xml version="1.0" encoding="UTF-8"?>
<build>
	<plugins>
        <!-- Set a compiler level -->
        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-compiler-plugin</artifactId>
            <version>2.3.2</version>

kumar-de

Kafka console consumer with Kerberos

1. Create a jaas.conf file with the following contents:

KafkaClient {
   com.sun.security.auth.module.Krb5LoginModule required
   useKeyTab=true
   keyTab="keytabFile.keytab"
   storeKey=true
   useTicketCache=false
   serviceName="kafka"

kumar-de

# The following example is for a job that produces daily, parquets partitioned by 'day' column
DB="database"
BASE_DIR="/output/parquets"
TBL="${DB}.table"       # name of the impala table
PQT="${BASE_DIR}/day"   # name of the parent directory containing output-subdirectories (in the format 'day=ddMMyyy')

# create/mount impala table
impala-shell -q "drop table if exists $TBL;
create external table $TBL (
account_id string,