Last active
October 10, 2015 04:25
-
-
Save thvasilo/7696d21cb3205f5cb11d to your computer and use it in GitHub Desktop.
Script to install components needed to read LZO-compressed files in Apache Spark
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
# Step 1: Install LZO dependencies | |
yum -y install lzo lzo-devel lzop | |
# Step 2: Install maven | |
MVN_VERSION=3.2.5 | |
MVN_MD5=b2d88f02bd3a08a9df1f0b0126ebd8dc | |
# wget http://www.apache.org/dyn/closer.cgi/maven/maven-3/$MVN_VERSION/binaries/apache-maven-$MVN_VERSION-bin.tar.gz | |
wget http://apache.claz.org/maven/maven-3/$MVN_VERSION/binaries/apache-maven-$MVN_VERSION-bin.tar.gz | |
md5=$(md5sum apache-maven-$MVN_VERSION-bin.tar.gz | awk '{ print $1 }') | |
if [ $md5 != $MVN_MD5 ] | |
then | |
echo "MD5 sums didn't match; exiting" | |
exit -1 | |
fi | |
mkdir /usr/local/apache-maven | |
tar -xzf apache-maven-$MVN_VERSION-bin.tar.gz | |
cp -r apache-maven-$MVN_VERSION /usr/local/apache-maven | |
echo "export M2_HOME=/usr/local/apache-maven/apache-maven-$MVN_VERSION/" >> ~/.bash_profile | |
echo 'export M2=$M2_HOME/bin' >> ~/.bash_profile | |
echo 'export PATH=$M2:$PATH' >> ~/.bash_profile | |
echo "Finished installing maven. Re-source ~/.bash_profile for changes to take effect" | |
export SCALA_HOME=/root/scala | |
export JAVA_HOME=/usr/lib/jvm/java-1.7.0 | |
export PATH=$PATH:$SCALA_HOME/bin | |
export PS1="\u@\h \W]\$ " | |
export M2_HOME=/usr/local/apache-maven/apache-maven-3.2.5/ | |
export M2=$M2_HOME/bin | |
export PATH=$M2:$PATH | |
#Step 3: Install hadoop-lzo | |
git clone https://github.com/twitter/hadoop-lzo.git /root/hadoop-lzo | |
mvn -f /root/hadoop-lzo/pom.xml package |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment