Spark Script

calculate_dnn.sh

#!/bin/sh
spark-submit \
	--class org.alitouka.spark.dbscan.exploratoryAnalysis.DistanceToNearestNeighborDriver \
	--master yarn \
	--deploy-mode cluster \
	--driver-cores 4 \
	--num-executors 10 \
	--executor-memory 8g \
	--executor-cores 4 \
	--conf spark.yarn.executor.memoryOverhead=1024 \
	--conf spark.scheduler.mode=FAIR \
	spark_dbscan-assembly-0.0.5-SNAPSHOT.jar \
	--ds-master "yarn-cluster" \
	--ds-jar hdfs:///user/isaac/spark_dbscan-assembly-0.0.5-SNAPSHOT.jar \
	--ds-input hdfs:///data/isaac/dbscan-parameter-tuning \
	--ds-output hdfs:///data/isaac/gdelt-dbscan-dnn