dennyglee

//
// Spark Connector for Cosmos DB to Mongo container
//  This gist provides an example of how to connect to Spark Connector for Cosmos DB to a Mongo container
//

// How to start spark-shell
// spark-shell --master yarn --jars /home/sshuser/jars/0.0.3c_1.12/azure-cosmosdb-spark-0.0.3-SNAPSHOT.jar,/home/sshuser/jars/0.0.3c_1.12/azure-documentdb-1.12.0-SNAPSHOT.jar
//

// Import Necessary Libraries

dennyglee

package org.apache.spark.sql.execution.datasources.parquet

    |- class ParquetFileFormat

        |- val vectorizedReader = new VectorizedParquetRecordReader()

            |- [VectorizedParquetRecordReader.java#48](https://github.com/apache/spark/blob/39e2bad6a866d27c3ca594d15e574a1da3ee84cc/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedParquetRecordReader.java#L4

dennyglee

from pyspark.mllib.linalg import Vectors

# Sample dataset 
data = sc.parallelize([
  (0.0, [0.0, 1.0, 2.0]),
  (1.0, [1.0, 2.0, 3.0]),
  (3.0, [2.0, 3.0, 4.0]),
  (2.0, [3.0, 4.0, 5.0])
])

dennyglee

from pyspark.sql.window import Window

# Defines partitioning specification and ordering specification.
windowSpec = \
  Window \
    .partitionBy(...) \
    .orderBy(...)
# Defines a Window Specification with a ROW frame.
windowSpec.rowsBetween(start, end)
# Defines a Window Specification with a RANGE frame.

dennyglee

/* Spark Shell Executed */
./bin/spark-shell --master spark://servername:7077 --driver-class-path $CLASSPATH


/* Output */
Welcome to
      ____              __
     / __/__  ___ _____/ /__
    _\ \/ _ \/ _ `/ __/  '_/
   /___/ .__/\_,_/_/ /_/\_\   version 1.4.0