start spark shell on cip cluster #cip #ss #spark

start_spark.sh

1.大集群
spark2.4.3_2.11
1.1 yarn模式
/opt/ubd/core/spark-2.4.3/bin/spark-shell --master yarn --queue ss_deploy --driver-memory 35g --num-executors 50 --executor-cores 4 --executor-memory 20g
1.2 local模式
/opt/ubd/core/spark-2.4.3/bin/spark-shell --queue ss_deploy --driver-memory 35g --num-executors 50 --executor-cores 4 --executor-memory 20g

spark2.4.2_2.12
/opt/ubd/core/spark-2.4.2/bin/spark-shell --queue ss_deploy --driver-memory 35g --num-executors 50 --executor-cores 4 --executor-memory 20g

2.花园集群
spark-3.1.2_2.12
/opt/ubd/core/spark-3.1.2-bin-hadoop3.2/bin/spark-shell --master yarn --queue default ---conf spark.executor.memory=21g   --conf spark.executor.cores=3   --num-executors 80

/opt/ubd/core/spark-3.1.2-bin-hadoop3.2/bin/pyspark --queue default   --conf spark.executor.memory=21g   --conf spark.executor.cores=3   --num-executors 80
################################# old #####################################3

# 大集群
spark2-shell --master yarn --queue ss_deploy --executor-memory 18g --total-executor-cores 40 --executor-cores 6 --conf spark.default.parallelism=1024 --conf=spark.serializer=org.apache.spark.serializer.JavaSerializer 

 spark2-shell --master yarn-client --queue ss_deploy --conf spark.driver.memory=18G  --executor-memory 18G   --num-executors 40 --executor-cores 6  --conf spark.default.parallelism=2048 
# daas
spark-shell --master yarn --queue ss_deploy --executor-memory 20g --num-executors 30 --executor-cores 2 --conf spark.default.parallelism=1024 --conf=spark.serializer=org.apache.spark.serializer.JavaSerializer 

#garden spark3
/opt/ubd/core/spark-3.1.2-bin-hadoop3.2/bin/spark-shell   --name ss_taozt   --master yarn   --conf spark.executor.memory=21g   --conf spark.executor.cores=3   --num-executors 200   --conf spark.yarn.spark.executor.memoryOverhead=2g   --conf spark.default.parallelism=1024     --conf spark.kryoserializer.buffer.max=512m   --conf spark.driver.maxResultSize=5g   --conf spark.driver.memory=20g   --conf spark.network.timeout=1200s   --conf spark.sql.shuffle.partitions=2048