mannharleen

public class createAnyObjectRecord {

    
    public Map<String, Schema.SObjectType> gd_map_full;
    //public List<Schema.SObjecttype> gd_lst_full; **delete
    public Map<String, Schema.SObjectType> gd;
    /*public List<Schema.SObjectField> field_names_lst{get;set;}
    Schema.sObjectField field_name;
    public schema.SObjectType object_name;*/
        

mannharleen

//import necessary
import org.apache.spark.{SparkContext,SparkConf}
import org.apache.spark.sql.hive.HiveContext

//initializations
val conf = new SparkConf().setAppName("xx").setMaster("local[2]")
val sc = new SparkContext(conf)
val hiveContext = new HiveContext(sc)

//create and load temp table

mannharleen

//Data can be found at https://github.com/mannharleen/data/blob/master/retail_db.zip

hiveContext.sql("create database retail_db")
hiveContext.sql("use retail_db")

hiveContext.sql("create table categories (category_id Int,category_department_id Int, category_name String) row format delimited fields terminated by ',' ")
hiveContext.sql("create table departments (department_id Int, department_name String) row format delimited fields terminated by ',' ")
hiveContext.sql("create table products (product_id Int, product_category_id Int, product_name String, product_description String, product_price Double, product_image String) row format delimited fields terminated by ',' ")
hiveContext.sql("create table order_items (order_item_id Int, order_item_order_id Int, order_item_product_id Int, order_item_quantity Int, order_item_subtotal Double, order_item_product_price Double) row format delimited fields terminated by ',' ")
hiveContext.sql("create table orders (order_id Int, order_date String, order_customer_id Int, o

mannharleen

/*
Spark and hive on windows environment
Error: java.lang.RuntimeException: java.lang.RuntimeException: The root scratch dir: /tmp/hive on HDFS should be writable. Current permissions are: rw-rw-rw-

Pre-requisites: Have winutils.exe placed in c:\winutils\bin\
Resolve as follows:
*/

C:\user>c:\Winutils\bin\winutils.exe ls
FindFileOwnerAndPermission error (1789): The trust relationship between this workstation and the primary domain failed.

mannharleen

import org.apache.spark.{SparkContext,SparkConf}
import org.apache.spark.sql.hive.HiveContext

//Do add the following artifact in build.sbt
  //libraryDependencies += "mysql" % "mysql-connector-java" % "5.1.43"

//initializations
val conf = new SparkConf().setAppName("xx").setMaster("local[2]")
val sc = new SparkContext(conf)

mannharleen

//Programmatically Specifying the Schema


import org.apache.spark.{SparkContext,SparkConf}
import org.apache.spark.sql.hive.HiveContext

//initializations
val conf = new SparkConf().setAppName("xx").setMaster("local[2]")
val sc = new SparkContext(conf)
val hiveContext = new HiveContext(sc)

mannharleen

import org.apache.spark.{SparkContext,SparkConf}
import org.apache.spark.sql.hive.HiveContext

//initializations
val conf = new SparkConf().setAppName("xx").setMaster("local[2]")
val sc = new SparkContext(conf)
val hiveContext = new HiveContext(sc)
//

mannharleen

hiveContext.udf.register("myudf",(a:Int, b:Int) => a+b)
//pyspark
  //sqlContext.udf.register("myudf",lambda x,y: x+y)
hiveContext.sql("select myudf(1,2)").show
  //same can be used with sqlContext

/*
Output:
+---+
|_c0|

mannharleen

val rdd1 = sc.parallelize(List((1,"one"),(2,"two")))
val df1 = rdd1.toDF("col1","col2")
  //user reflection to convert to DF

//create cube with dimentions as col1 and col2 & fact as average of col1
df1.cube("col1","col2").agg( Map( "col1" -> "avg" )).show

/*
Outputs:
+----+----+---------+

mannharleen

/*
Add the numbers coming into each DStream of data
*/

import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.Seconds
import org.apache.spark.{SparkConf, SparkContext}

val conf = new SparkConf().setAppName("sbtapp").setMaster("local[3]")
val ssc = new StreamingContext(conf,Seconds(5))