dgadiraju’s gists

dgadiraju / spark-scala-wordcount-mapPartitions.scala

Last active August 2, 2017 07:42

	val path = "/Users/itversity/Research/data/wordcount.txt" or val path = "/public/randomtextwriter/part-m-00000"

	sc.textFile(path).
	mapPartitions(lines => {
	// Using Scala APIs to process each partition
	lines.flatMap(_.split(" ")).map((_, 1))
	}).
	reduceByKey((total, agg) => total + agg).
	take(100).
	foreach(println)

dgadiraju / spark-submit-wordcount-local-standalone.sh

Created June 1, 2017 13:48

	spark-submit --class wordcount.WordCount \
	target/scala-2.10/sparkdemo_2.10-1.0.jar \
	uat \
	/Users/itversity/Research/data/wordcount.txt \
	/Users/itversity/Research/data/wordcount

dgadiraju / spark-submit-wordcount-yarn.sh

Last active December 16, 2017 09:55

	spark-submit --class wordcount.WordCount \
	--master yarn \
	--conf spark.ui.port=54312 \
	sparkdemo_2.10-1.0.jar \
	prod \
	/public/randomtextwriter/part-m-00000 \
	/user/dgadiraju/wordcount

dgadiraju / spark-submit-wordcount-yarn-custom-capacity.sh

Created June 2, 2017 09:50

	spark-submit --class wordcount.WordCount \
	--num-executors 10 \
	--executor-memory 3584M \
	--executor-cores 4 \
	--master yarn \
	--conf spark.ui.port=54123 \
	sparkdemo_2.10-1.0.jar \
	prod /public/randomtextwriter /user/dgadiraju/wordcount

dgadiraju / spark-scala-cardcountbysuit.scala

Last active June 4, 2017 13:41

	// Make sure you do not have directory used for output path
	// hadoop fs -rm -R /user/dgadiraju/cardcountbysuit
	val inputPath = "/public/cards/largedeck.txt"
	val outputPath = "/user/dgadiraju/cardcountbysuit"

	sc.textFile(inputPath).
	map(card => (card.split("\\\|")(1), 1)).
	reduceByKey((total, card) => total + card).
	saveAsTextFile(outputPath)

dgadiraju / spark-scala-wordcount-coalesce.scala

Last active June 4, 2017 13:37

	// Make sure you do not have directory used for output path
	val path = "/Users/itversity/Research/data/wordcount.txt" or val path = "/public/randomtextwriter/part-m-00000"

	sc.textFile(path).
	coalesce(5). // with out coalesce it will try to use 9 tasks in first stage
	flatMap(_.split(" ")).
	map((_, 1)).
	reduceByKey((total, agg) => total + agg).
	coalesce(2). // second stage will use only 2 tasks
	take(100).

dgadiraju / spark-scala-cardcountbysuit-repartition.scala

Created June 4, 2017 13:40

	// Make sure you do not have directory used for output path
	// hadoop fs -rm -R /user/dgadiraju/cardcountbysuit
	val inputPath = "/public/cards/largedeck.txt"
	val outputPath = "/user/dgadiraju/cardcountbysuit"

	sc.textFile(inputPath).
	repartition(12).
	map(card => (card.split("\\\|")(1), 1)).
	reduceByKey((total, card) => total + card, 2).
	saveAsTextFile(outputPath)

dgadiraju / spark-scala-cardcountbysuit-numtasks.scala

Last active June 4, 2017 13:56


	// Make sure you do not have directory used for output path
	// hadoop fs -rm -R /user/dgadiraju/cardcountbysuit
	val inputPath = "/public/cards/largedeck.txt"
	val outputPath = "/user/dgadiraju/cardcountbysuit"

	sc.textFile(inputPath).
	map(card => (card.split("\\\|")(1), 1)).
	reduceByKey((total, card) => total + card, 1). //Only 1 file will be created and 1 task will be used in second stage.
	saveAsTextFile(outputPath)

dgadiraju / spark-scala-wordcount-numtasks.scala

Last active June 4, 2017 13:57

	val inputPath = "/public/randomtextwriter/part-m-0000*"
	val outputPath = "/user/dgadiraju/wordcount"

	sc.textFile(inputPath).
	flatMap(_.split(" ")).
	map((_, 1)).
	reduceByKey((total, agg) => total + agg, 10). //Ideal number of tasks could be 4
	saveAsTextFile(outputPath)

dgadiraju / youthspark-bootstrap-index.html

Created June 4, 2017 14:48

	<!DOCTYPE html>
	<html lang="en">
	<head>
	<meta charset="UTF-8">
	<meta name="viewport" content="width=device-width, initial-scale=1.0">
	<meta http-equiv="X-UA-Compatible" content="ie=edge">
	<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css">
	<script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js"></script>

	<title>Document</title>

Durga Gadiraju dgadiraju