kumar-de

CREATE EXTERNAL TABLE table_name (
    vehicle_id String,
    start_time String,
    av0 Double
)
STORED AS parquet
LOCATION '/path/in/hdfs/without//hdfs://'; 

kumar-de

val conf = sc.hadoopConfiguration
val fs = org.apache.hadoop.fs.FileSystem.get(conf)
val exists = fs.exists(new org.apache.hadoop.fs.Path("/path/on/hdfs")) // File or directory

val sequenceFiles1 = htu.getDFSCluster.getFileSystem.listStatus(new Path(outputFileDirforCrashes)).filter(_.isDirectory).map(_.getPath.toString)
        sequenceFiles1.foreach(dirPath=>{
 val files = htu.getDFSCluster.getFileSystem.listStatus(new Path(dirPath)).filter(_.isFile).map(_.getPath.toString)

kumar-de

Parquet

val sqlContext = new org.apache.spark.sql.SQLContext(sc)

val df = sqlContext.read.parquet("/path/to/file/without/hdfs://")

df.printSchema

df.count

kumar-de

apk add python wget py-pip python-dev gcc musl-dev libxml2-dev libxslt-dev

echo "Installing opcua via pip may take longer than expected..."

pip install setuptools opcua

kumar-de

Install openvpn, its network-manager and network-manager-gnome

sudo apt-get install openvpn network-manager-openvpn network-manager-openvpn-gnome

kumar-de

Remove existing version, if any

sudo apt-get remove scala-library scala

Change version, if necessary: e.g. 2.11.12

sudo wget www.scala-lang.org/files/archive/scala-2.11.8.deb

kumar-de

# to list all ports that are used
sudo netstat -ntlp | grep LISTEN

# you can obtain a specific port using the following command
sudo netstat -ntlp | grep :8080

# when you execute the command above you will see some thing like that
tcp        0      0 0.0.0.0:27370           0.0.0.0:*               LISTEN      4394/skype      
tcp        0      0 127.0.1.1:53            0.0.0.0:*               LISTEN      2216/dnsmasq    
tcp        0      0 127.0.0.1:631           0.0.0.0:*               LISTEN      4912/cupsd