Created
October 4, 2017 18:46
-
-
Save prehensilecode/234fb4c3b4fa392888c9c8854b809d1d to your computer and use it in GitHub Desktop.
Diff to integrate Apache Spark standalone with Univa Grid Engine (other environment stuff needed)
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- ./slaves.sh 2017-10-04 14:40:05.607320037 -0400 | |
| +++ /mnt/HA/opt/apache/spark/2.2.0/sbin/slaves.sh 2017-10-04 14:24:55.986020350 -0400 | |
| @@ -80,20 +80,29 @@ | |
| fi | |
| fi | |
| - | |
| - | |
| # By default disable strict host key checking | |
| if [ "$SPARK_SSH_OPTS" = "" ]; then | |
| SPARK_SSH_OPTS="-o StrictHostKeyChecking=no" | |
| fi | |
| +usershell=$( getent passwd ${USER} | cut -d: -f7 ) | |
| for slave in `echo "$HOSTLIST"|sed "s/#.*$//;/^$/d"`; do | |
| if [ -n "${SPARK_SSH_FOREGROUND}" ]; then | |
| - ssh $SPARK_SSH_OPTS "$slave" $"${@// /\\ }" \ | |
| - 2>&1 | sed "s/^/$slave: /" | |
| + if [ "$usershell" = "/bin/tcsh" -o "$usershell" = "/bin/csh" ]; then | |
| + ssh $SPARK_SSH_OPTS "$slave" "source ${SPARK_CONF_DIR}/spark-env.csh ;" $"${@// /\\ }" \ | |
| + 2>&1 | sed "s/^/$slave: /" | |
| + else | |
| + ssh $SPARK_SSH_OPTS "$slave" "source ${SPARK_COFI_DIR}/spark-env.sh ;" $"${@// /\\ }" \ | |
| + 2>&1 | sed "s/^/$slave: /" | |
| + fi | |
| else | |
| - ssh $SPARK_SSH_OPTS "$slave" $"${@// /\\ }" \ | |
| - 2>&1 | sed "s/^/$slave: /" & | |
| + if [ "$usershell" = "/bin/tcsh" -o "$usershell" = "/bin/csh" ]; then | |
| + ssh $SPARK_SSH_OPTS "$slave" "source ${SPARK_CONF_DIR}/spark-env.csh ;" $"${@// /\\ }" \ | |
| + 2>&1 | sed "s/^/$slave: /" & | |
| + else | |
| + ssh $SPARK_SSH_OPTS "$slave" "source ${SPARK_CONF_DIR}/spark-env.sh ;" $"${@// /\\ }" \ | |
| + 2>&1 | sed "s/^/$slave: /" & | |
| + fi | |
| fi | |
| if [ "$SPARK_SLAVE_SLEEP" != "" ]; then | |
| sleep $SPARK_SLAVE_SLEEP |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment