Skip to content

Instantly share code, notes, and snippets.

@ctaggart
Created May 20, 2016 01:50
Show Gist options
  • Save ctaggart/c40fcce0ed76342d20e78c79673abd28 to your computer and use it in GitHub Desktop.
Save ctaggart/c40fcce0ed76342d20e78c79673abd28 to your computer and use it in GitHub Desktop.
#!/bin/sh
# script for execution of Hadoop deployed component
#
if [ -n "$MATLAB_HADOOP_INSTALL" ];
then
hadoop_name=$MATLAB_HADOOP_INSTALL/bin/hadoop
echo Using MATLAB_HADOOP_INSTALL: ${hadoop_name}
elif [ -n "$HADOOP_PREFIX" ];
then
hadoop_name=$HADOOP_PREFIX/bin/hadoop
echo Using HADOOP_PREFIX: ${hadoop_name}
elif [ -n "$HADOOP_HOME" ];
then
hadoop_name=$HADOOP_HOME/bin/hadoop
echo Using HADOOP_HOME: ${hadoop_name}
else
echo "Cannot find Hadoop. Set the HADOOP_PREFIX, HADOOP_HOME, or MATLAB_HADOOP_INSTALL environment variable to a valid Hadoop installation folder and try again."
exit 1
fi
echo "Find out Hadoop version: $hadoop_name version"
hadoop_version=`$hadoop_name version | sed -n 1p | sed 's/[a-zA-Z]* \([0-9]*\).*/\1/'`
if [ $hadoop_version -eq 1 ];
then
version_path="a1.2.1"
echo "Use Hadoop V1 JAR files"
elif [ $hadoop_version -eq 2 ];
then
version_path="a2.2.0"
echo "Use Hadoop V2 JAR files"
else
echo "Hadoop version $hadoop_version is not supported."
exit 1
fi
echo "------------------------------------------"
if [ "x$1" = "x" ]; then
echo Usage:
echo $0 \<deployedMCRroot\> [-D mw.mcrroot=\<MCRRoot installed on hadoop cluster\>] [\<hadoop arguments\>] \<inputFiles\> \<outputFolder\>
else
echo Launch Hadoop job airlinesmall.ctf
if [ $# -gt 0 ]; then
MCRROOT="$1"
specifiedClusterMCRRoot=false
shift 1
fi
args=
while [ $# -gt 0 ]; do
if [ $# -eq 2 ]; then
args="${args} \"airlinesmall.ctf\""
fi
token=$1
if [ `expr match "$1" mw.mcrroot=` -gt 0 ]; then
specifiedClusterMCRRoot=true
fi
args="${args} \"${token}\""
shift
done
if [ -f "${MCRROOT}/toolbox/mlhadoop/jar/${version_path}/mwmapreduce.jar" ];
then
strJARPath="${MCRROOT}/toolbox/mlhadoop/jar/${version_path}/mwmapreduce.jar"
else
strJARPath="${MCRROOT}/toolbox/mlhadoop/jar/${version_path}/glnxa64/mwmapreduce.jar"
fi
if [ "$specifiedClusterMCRRoot" = true ];
then
strMWMCRRoot=""
else
strMWMCRRoot="-D mw.mcrroot=\"${MCRROOT}\""
fi
echo "Executing: ${hadoop_name} jar \"$strJARPath\" com.mathworks.hadoop.MWMapReduceDriver $strMWMCRRoot $args"
eval "\"${hadoop_name}\"" jar \"$strJARPath\" com.mathworks.hadoop.MWMapReduceDriver $strMWMCRRoot $args
fi
exit
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment