dgadiraju’s gists

dgadiraju / spark-core-subtract-get-customer-details.scala

Created September 10, 2019 13:18

	val orderCustomers1 = sc.textFile("/public/retail_db/orders").
	filter(o => o.split(",")(1).contains("2013-08")).
	map(o => o.split(",")(2).toInt)

	val orderCustomers2 = sc.textFile("/public/retail_db/orders").
	filter(o => o.split(",")(1).contains("2013-09")).
	map(o => o.split(",")(2).toInt)

	val customers = sc.textFile("/public/retail_db/customers").
	map(c => (c.split(",")(0).toInt, c))

dgadiraju / spark-core-intersection-01-customer-ids.scala

Last active September 10, 2019 13:17

	val orderCustomers1 = sc.textFile("/public/retail_db/orders").
	filter(o => o.split(",")(1).contains("2013-08")).
	map(o => o.split(",")(2).toInt)

	val orderCustomers2 = sc.textFile("/public/retail_db/orders").
	filter(o => o.split(",")(1).contains("2013-09")).
	map(o => o.split(",")(2).toInt)

	orderCustomers1.count
	orderCustomers2.count

dgadiraju / spark-core-union-01-all.scala

Last active September 10, 2019 12:26

	val orderCustomers1 = sc.textFile("/public/retail_db/orders").
	filter(o => o.split(",")(1).contains("2013-08")).
	map(o => o.split(",")(2))

	val orderCustomers2 = sc.textFile("/public/retail_db/orders").
	filter(o => o.split(",")(1).contains("2013-09")).
	map(o => o.split(",")(2))

	orderCustomers1.count
	orderCustomers2.count

dgadiraju / spark-core-outer-join-orders-count-by-status.scala

Created September 9, 2019 13:30

	val orders = sc.textFile("/public/retail_db/orders")
	val ordersMap = orders.
	map(o => (o.split(",")(0).toInt, o.split(",")(3)))

	val orderItems = sc.textFile("/public/retail_db/order_items")
	val orderItemsMap = orderItems.
	map(oi => (oi.split(",")(1).toInt, oi))

	val ordersLeftOuterJoin = ordersMap.
	leftOuterJoin(orderItemsMap)

dgadiraju / spark-core-outer-join-orders-and-order-items-filter.scala

Created September 9, 2019 13:25

	val orders = sc.textFile("/public/retail_db/orders")
	val ordersMap = orders.
	map(o => (o.split(",")(0).toInt, o))

	val orderItems = sc.textFile("/public/retail_db/order_items")
	val orderItemsMap = orderItems.
	map(oi => (oi.split(",")(1).toInt, oi))

	val ordersLeftOuterJoin = ordersMap.
	leftOuterJoin(orderItemsMap)

dgadiraju / spark-core-outer-join-orders-and-order-items.scala

Created September 9, 2019 13:23

	val orders = sc.textFile("/public/retail_db/orders")
	val ordersMap = orders.
	map(o => (o.split(",")(0).toInt, o))

	val orderItems = sc.textFile("/public/retail_db/order_items")
	val orderItemsMap = orderItems.
	map(oi => (oi.split(",")(1).toInt, oi))

	val ordersJoin = ordersMap.
	join(orderItemsMap)

dgadiraju / spark-core-01-read-orders.scala

Created September 9, 2019 11:00

	val orders = sc.textFile("/public/retail_db/orders")
	val ordersMap = orders.
	map(o => (o.split(",")(0).toInt, o))

dgadiraju / spark-core-groupByKey-ranking.scala

Created September 5, 2019 14:48

	val products = sc.textFile("/public/retail_db/products")

	val productsMap = products.
	filter(p => p.split(",")(4) != "").
	map(p => (p.split(",")(1).toInt, p))

	productsMap.
	groupByKey.
	flatMap(p => {
	p._2.toList.sortBy(k => k.split(",")(4).toFloat)(Ordering.Float.reverse).take(3)

dgadiraju / spark-core-groupByKey-sorting.scala

Created September 5, 2019 14:46

	val products = sc.textFile("/public/retail_db/products")

	val productsMap = products.
	filter(p => p.split(",")(4) != "").
	map(p => (p.split(",")(1).toInt, p))

	productsMap.
	groupByKey.
	flatMap(p => {
	p._2.toList.sortBy(k => k.split(",")(4).toFloat)(Ordering.Float.reverse)

dgadiraju / spark-core-aggregateByKey-get-revenue-and-count-per-order-id.scala

Created September 5, 2019 14:25

	val orderItems = sc.textFile("/public/retail_db/order_items")

	val orderItemsMap = orderItems.
	map(oi => (oi.split(",")(1).toInt, oi.split(",")(4).toFloat))

	val revenueAndCountPerOrder = orderItemsMap.
	aggregateByKey((0.0, 0))(
	(agg, ele) => (agg._1 + ele, agg._2 + 1),
	(total, inter) => (total._1 + inter._1, total._2 + inter._2)
	)

Durga Gadiraju dgadiraju