Last active
August 2, 2022 12:33
-
-
Save achalddave/7f7323a36f85b6c6dd64 to your computer and use it in GitHub Desktop.
This is a script to compile caffe without root permission. It hasn't been extensively tested, but it will hopefully still make things easier than going about it on your own. Please let me know if there are any bugs!
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# This script installs (or at least, attempts to install) Caffe in CPU mode | |
# with Python bindings without root permission. In doing so, it installs the | |
# following dependencies: | |
# | |
# * protobuf | |
# * cmake | |
# * gflags | |
# * glog | |
# * opencv | |
# * hdf5 | |
# * python2 | |
# * pip | |
# * libpng (recent version required for freetype) | |
# * automake (recent version required for freetype) | |
# * freetype (required for scikit-image) | |
# * various python modules | |
# - numpy | |
# - scipy | |
# - scikit-image | |
# - protobuf | |
# - yaml | |
# * boost | |
# * lmdb | |
# * libtool | |
# * snappy | |
# * leveldb | |
# * openblas | |
LOCAL_INSTALL_DIR="${HOME}/local" | |
SCRATCH_DIR="${HOME}/scratch_for_setup" | |
CAFFE_INSTALL_DIR="${HOME}/caffe" | |
# Note that changing these versions may not work - it's a little fragile and | |
# depends on the URLs for these utilities to have the same template across | |
# versions (e.g. http://example.com/tool/tool_v${version_number}). The URLs | |
# work with the versions listed as of the last committed version, but should | |
# be tested before updating and committing. | |
PROTOBUF_VERSION="2.6.1" | |
GFLAGS_VERSION="2.1.2" | |
GLOG_VERSION="0.3.4" | |
OPENCV_VERSION="2.4.11" | |
LMDB_VERSION="0.87" | |
PYTHON_VERSION="2.7.10" # Must be python2.* | |
LIBPNG_VERSION="1.6.18" | |
FREETYPE_VERSION="2.6.1" | |
LIBTOOL_VERSION="2.4.6" | |
AUTOMAKE_VERSION="1.15" | |
LEVELDB_VERSION="1.18" | |
OPENBLAS_VERSION="0.2.14" | |
# Hardcoded versions - these cannot be changed here. | |
# TODO: Allow changing the below values. | |
# BOOST_VERSION="1.59.0" | |
# HDF5_VERSION="1.8.15-patch1" | |
# CMAKE_VERSION="3.2.3" | |
# E.g. 2.7.10 -> python2.7 | |
PYTHON_SHORT_VERSION="$(echo $PYTHON_VERSION | sed -e 's/\([0-9]*\.[0-9]*\)\(\..*\)\?/\1/g')" | |
PYTHON_BINARY="python${PYTHON_SHORT_VERSION}" | |
# Exit on error. | |
set -e | |
# Taken from | |
# https://github.com/achalddave/dotfiles/blob/master/misc/sudo-less-servers/install_utilities.sh | |
# Usage: untar_to_dir <tar_file> <output_directory> | |
# Untars to a specified directory, instead of using the "root" directory | |
# specified in the tar file. Useful for cd'ing. | |
untar_to_dir() { | |
if [[ "$#" -ne 2 ]] ; then | |
echo "Improper number of arguments to untar_to_dir" | |
exit | |
fi | |
TAR_FILE="${1}" | |
OUTPUT="${2}" | |
mkdir -p "${OUTPUT}" | |
tar xzvf "${TAR_FILE}" -C "${OUTPUT}" --strip-components=1 | |
} | |
# Usage: scratch_init <utility_name> | |
scratch_init() { | |
if [[ "$#" -ne 1 ]] ; then | |
echo "Improper number of arguments to scratch_init" | |
exit | |
fi | |
cd "${SCRATCH_DIR}" | |
mkdir -p "$1" | |
cd "$1" | |
} | |
install_protobufs() { | |
scratch_init protobuf | |
wget "https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz" | |
untar_to_dir "protobuf-${PROTOBUF_VERSION}.tar.gz" protobuf-${PROTOBUF_VERSION} | |
cd protobuf-${PROTOBUF_VERSION} | |
./configure --prefix="${LOCAL_INSTALL_DIR}/protobuf-${PROTOBUF_VERSION}" | |
make -j4 | |
make install | |
} | |
install_cmake() { | |
scratch_init cmake | |
# Cmake has pre-built binaries ready, so we will just untar it in | |
# ${LOCAL_INSTALL_DIR}. | |
wget "https://cmake.org/files/v3.2/cmake-3.2.3-Linux-x86_64.tar.gz" | |
cd ${LOCAL_INSTALL_DIR} | |
untar_to_dir "cmake-3.2.3-Linux-x86_64.tar.gz" . | |
echo "You will need to add the following line to your .zshrc/.bashrc:" | |
echo 'export CMAKE_ROOT="'${LOCAL_INSTALL_DIR}'/share/cmake-3.2"' | |
} | |
install_gflags() { | |
scratch_init gflags | |
wget "https://github.com/gflags/gflags/archive/v${GFLAGS_VERSION}.tar.gz" | |
untar_to_dir "v${GFLAGS_VERSION}.tar.gz" "v${GFLAGS_VERSION}" | |
cd "v${GFLAGS_VERSION}" | |
mkdir build | |
cd build | |
cmake -D CMAKE_INSTALL_PREFIX="${LOCAL_INSTALL_DIR}" -D CMAKE_POSITION_INDEPENDENT_CODE=ON .. | |
make | |
make install | |
} | |
install_glog() { | |
scratch_init glog | |
wget "https://github.com/google/glog/archive/v${GLOG_VERSION}.tar.gz" | |
untar_to_dir "v${GLOG_VERSION}.tar.gz" "v${GLOG_VERSION}" | |
cd "v${GLOG_VERSION}" | |
./configure --prefix="${LOCAL_INSTALL_DIR}/glog-${GLOG_VERSION}" | |
make | |
make install | |
} | |
install_opencv() { | |
scratch_init opencv | |
wget "https://github.com/Itseez/opencv/archive/${OPENCV_VERSION}.tar.gz" -O "opencv-${OPENCV_VERSION}.tar.gz" | |
untar_to_dir "opencv-${OPENCV_VERSION}.tar.gz" "opencv-${OPENCV_VERSION}" | |
cd "opencv-${OPENCV_VERSION}" | |
mkdir release | |
cd release | |
cmake -D BUILD_ZLIB=ON -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX="${LOCAL_INSTALL_DIR}" -D BUILD_PYTHON_SUPPORT=ON -D WITH_GTK=OFF .. | |
# Update ${SCRATCH_DIR}/opencv-2.4.11/release/modules/features2d/CMakeFiles/opencv_features2d.dir/build.make | |
# cd ${SCRATCH_DIR}/opencv/opencv-2.4.11/release/modules/features2d && /usr/bin/c++ $(CXX_DEFINES) $(CXX_FLAGS) -o CMakeFiles/opencv_features2d.dir/src/freak.cpp.o -c ${SCRATCH_DIR}/opencv/opencv-2.4.11/modules/features2d/src/freak.cpp | |
# to | |
# cd ${SCRATCH_DIR}/opencv/opencv-2.4.11/release/modules/features2d && /usr/bin/c++ $(CXX_DEFINES) $(CXX_FLAGS) -O0 -o CMakeFiles/opencv_features2d.dir/src/freak.cpp.o -c ${SCRATCH_DIR}/opencv/opencv-2.4.11/modules/features2d/src/freak.cpp | |
# (Add -O0 after CXX_FLAGS) | |
# This is necessary due to http://stackoverflow.com/a/14619427/1291812 | |
sed -i'' -e \ | |
's:'${SCRATCH_DIR}'/opencv/opencv-2.4.11/release/modules/features2d && /usr/bin/c++ $(CXX_DEFINES) $(CXX_FLAGS) -o CMakeFiles/opencv_features2d.dir/src/freak.cpp.o -c '${SCRATCH_DIR}'/opencv/opencv-2.4.11/modules/features2d/src/freak.cpp:'${SCRATCH_DIR}'/opencv/opencv-2.4.11/release/modules/features2d && /usr/bin/c++ $(CXX_DEFINES) $(CXX_FLAGS) -O0 -o CMakeFiles/opencv_features2d.dir/src/freak.cpp.o -c '${SCRATCH_DIR}'/opencv/opencv-2.4.11/modules/features2d/src/freak.cpp"' \ | |
"${SCRATCH_DIR}/opencv-2.4.11/release/modules/features2d/CMakeFiles/opencv_features2d.dir/build.make" | |
make -j16 | |
make install | |
} | |
install_hdf5() { | |
scratch_init hdf5 | |
wget "http://www.hdfgroup.org/ftp/HDF5/current/src/hdf5-1.8.15-patch1.tar.gz" | |
untar_to_dir "hdf5-1.8.15-patch1.tar.gz" "hdf5-1.8.15" | |
cd hdf5-1.8.15 | |
./configure --prefix="${LOCAL_INSTALL_DIR}/hdf5-1.8.15" | |
make -j4 | |
make install | |
} | |
install_libpng() { | |
scratch_init libpng | |
wget "http://downloads.sourceforge.net/libpng/libpng-${LIBPNG_VERSION}.tar.gz" | |
untar_to_dir "libpng-${LIBPNG_VERSION}.tar.gz" "libpng-${LIBPNG_VERSION}" | |
cd "libpng-${LIBPNG_VERSION}" | |
LDFLAGS="-L ${LOCAL_INSTALL_DIR}/lib" ./configure --prefix="${LOCAL_INSTALL_DIR}/libpng-${LIBPNG_VERSION}" | |
make -j4 | |
make install | |
} | |
install_automake() { | |
scratch_init automake | |
wget "http://ftp.gnu.org/gnu/automake/automake-${AUTOMAKE_VERSION}.tar.gz" | |
untar_to_dir "automake-${AUTOMAKE_VERSION}.tar.gz" "automake-${AUTOMAKE_VERSION}" | |
cd "automake-${AUTOMAKE_VERSION}" | |
./configure --prefix="${LOCAL_INSTALL_DIR}/automake-${AUTOMAKE_VERSION}" | |
make -j4 | |
make install | |
} | |
install_freetype() { | |
scratch_init freetype | |
wget "http://download.savannah.gnu.org/releases/freetype/freetype-${FREETYPE_VERSION}.tar.gz" | |
untar_to_dir "freetype-${FREETYPE_VERSION}.tar.gz" "freetype-${FREETYPE_VERSION}" | |
cd "freetype-${FREETYPE_VERSION}" | |
./autogen.sh | |
./configure --prefix="${LOCAL_INSTALL_DIR}/freetype-${FREETYPE_VERSION}" | |
make | |
make install | |
} | |
# Taken from | |
# https://github.com/achalddave/dotfiles/blob/master/misc/sudo-less-servers/install_utilities.sh | |
install_python2() { | |
scratch_init python2 | |
wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz | |
untar_to_dir "Python-${PYTHON_VERSION}.tgz" "python-${PYTHON_VERSION}" | |
cd "python-${PYTHON_VERSION}" | |
./configure --prefix="${LOCAL_INSTALL_DIR}/python-${PYTHON_VERSION}" --enable-shared | |
make -j4 | |
make altinstall | |
} | |
# Taken from | |
# https://github.com/achalddave/dotfiles/blob/master/misc/sudo-less-servers/install_utilities.sh | |
install_pip() { | |
scratch_init pip2 | |
wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py | |
${PYTHON_BINARY} get-pip.py | |
} | |
install_python_modules() { | |
${PYTHON_BINARY} -m pip install numpy | |
${PYTHON_BINARY} -m pip install scipy | |
${PYTHON_BINARY} -m pip install scikit-image | |
${PYTHON_BINARY} -m pip install protobuf | |
${PYTHON_BINARY} -m pip install pyyaml | |
} | |
# TODO: This does not install the python bindings, it seems... That is, | |
# import boost | |
# does not work in python. | |
install_boost() { | |
scratch_init boost | |
wget "http://sourceforge.net/projects/boost/files/boost/1.59.0/boost_1_59_0.tar.gz/download" -O "boost_1_59_0.tar.gz" | |
untar_to_dir "boost_1_59_0.tar.gz" "boost_1_59_0" | |
cd boost_1_59_0 | |
./bootstrap.sh --prefix="${LOCAL_INSTALL_DIR}/boost-1.59.0" --with-python="${LOCAL_INSTALL_DIR}/python-${PYTHON_VERSION}/bin/python${PYTHON_SHORT_VERSION}" | |
./b2 install | |
} | |
install_lmdb() { | |
scratch_init lmdb | |
wget "https://github.com/dw/py-lmdb/archive/py-lmdb_${LMDB_VERSION}.tar.gz" | |
untar_to_dir "py-lmdb_${LMDB_VERSION}.tar.gz" "py-lmdb_${LMDB_VERSION}" | |
cd "py-lmdb_${LMDB_VERSION}" | |
${PYTHON_BINARY} setup.py install | |
# Contains the actual LMDB sources? | |
scratch_init openldap | |
wget "ftp://ftp.openldap.org/pub/OpenLDAP/openldap-release/openldap-2.4.42.tgz" | |
untar_to_dir "openldap-2.4.42.tgz" openldap-2.4.42 | |
cd openldap-2.4.42/libraries/lmdb | |
make -j4 | |
sed -i'' -e 's:\(prefix.*=\).*:\1'${LOCAL_INSTALL_DIR}':g' Makefile | |
make install | |
} | |
install_libtool() { | |
scratch_init libtool | |
wget "http://ftpmirror.gnu.org/libtool/libtool-${LIBTOOL_VERSION}.tar.gz" | |
untar_to_dir "libtool-${LIBTOOL_VERSION}.tar.gz" "libtool-${LIBTOOL_VERSION}" | |
cd libtool-${LIBTOOL_VERSION} | |
./configure --prefix="${LOCAL_INSTALL_DIR}/libtool-${LIBTOOL_VERSION}" | |
make -j4 | |
make install | |
} | |
install_snappy() { | |
scratch_init snappy | |
wget "https://github.com/google/snappy/tarball/master" -O snappy.tar.gz | |
untar_to_dir "snappy.tar.gz" "libsnappy" | |
cd "libsnappy" | |
./autogen.sh | |
echo "AC_PROG_LIBTOOL" >>config.ac | |
./configure --prefix="${LOCAL_INSTALL_DIR}/snappy" | |
make | |
make install | |
# The default autogen.sh does not handle compatibility well... This one, | |
# from the following PR: https://github.com/google/snappy/pull/4 is | |
# preferable. | |
#wget https://raw.githubusercontent.com/juanmaneo/snappy/49262984cddf3985fba7d1ceca6b14986f6dbef0/autogen.sh -O autogen.sh | |
} | |
install_leveldb() { | |
scratch_init leveldb | |
wget "https://github.com/google/leveldb/archive/v${LEVELDB_VERSION}.tar.gz" | |
untar_to_dir "v${LEVELDB_VERSION}.tar.gz" "leveldb-${LEVELDB_VERSION}" | |
cd "leveldb-${LEVELDB_VERSION}" | |
LD_FLAGS="-L${LOCAL_INSTALL_DIR}/lib" make -j4 | |
# leveldb doesn't have a make install target... | |
# Instructions below taken from | |
# http://techoverflow.net/blog/2012/12/14/compiling-installing-leveldb-on-linux/ | |
cp --preserve=links libleveldb.* ${LOCAL_INSTALL_DIR}/lib | |
cp -r include/leveldb ${LOCAL_INSTALL_DIR}/include/ | |
} | |
install_openblas() { | |
scratch_init openblas | |
wget "https://github.com/xianyi/OpenBLAS/archive/v${OPENBLAS_VERSION}.tar.gz" -O "openblas_${OPENBLAS_VERSION}.tar.gz" | |
untar_to_dir openblas_${OPENBLAS_VERSION}.tar.gz openblas_${OPENBLAS_VERSION} | |
cd openblas_${OPENBLAS_VERSION} | |
make -j4 | |
make PREFIX="${LOCAL_INSTALL_DIR}" install | |
# OpenBLAS only installs libopenblas.a, but it provides the symbols for | |
# libcblas.a and libatlas.a (I think). Symlinking libatlas and libcblas | |
# allows Caffe to build. | |
ln -s "${LOCAL_INSTALL_DIR}/lib/libopenblas.a" "${LOCAL_INSTALL_DIR}/lib/libcblas.a" | |
ln -s "${LOCAL_INSTALL_DIR}/lib/libopenblas.a" "${LOCAL_INSTALL_DIR}/lib/libatlas.a" | |
} | |
install_caffe() { | |
mkdir -p "${CAFFE_INSTALL_DIR}" | |
cd "${CAFFE_INSTALL_DIR}" | |
wget https://github.com/BVLC/caffe/archive/rc2.tar.gz | |
untar_to_dir rc2.tar.gz . | |
cp Makefile.config.example Makefile.config | |
echo "=== Compiling caffe ===" | |
CPU_ONLY=1 \ | |
LDFLAGS="-L${LOCAL_INSTALL_DIR}/lib" \ | |
make -j4 | |
echo "=== Compiling caffe python bindings ===" | |
python_include_dir="${LOCAL_INSTALL_DIR}/include/python${PYTHON_SHORT_VERSION}" | |
numpy_include_dir="${LOCAL_INSTALL_DIR}/lib/python${PYTHON_SHORT_VERSION}/site-packages/numpy/core/include" | |
CPU_ONLY=1 \ | |
CPLUS_INCLUDE_PATH="${CPLUS_INCLUDE_PATH}:${python_include_dir}:${numpy_include_dir}" \ | |
LDFLAGS="-L${LOCAL_INSTALL_DIR}/lib" \ | |
make pycaffe | |
echo "=== If that worked, congrats! One last note: ===" | |
echo "You'll need to update your PYTHONPATH in .bashrc/.zshrc as follows:" | |
echo 'export PYTHONPATH="'${CAFFE_INSTALL_DIR}'/python:$PYTHONPATH"' | |
} | |
echo "=== Installing Protobufs ===" | |
install_protobuf | |
echo "=== Installing cmake ===" | |
install_cmake | |
echo "=== Installing gflags ===" | |
install_gflags | |
echo "=== Installing glog ===" | |
install_glog | |
echo "=== Installing opencv ===" | |
install_opencv | |
echo "=== Installing hdf5 ===" | |
install_hdf5 | |
echo "=== Installing python2 ===" | |
install_python2 | |
echo "=== Installing pip ===" | |
install_pip | |
echo "=== Installing libpng ===" | |
install_libpng | |
echo "=== Installing automake ===" | |
install_automake | |
echo "=== Installing freetype ===" | |
install_freetype | |
echo "=== Installing python modules ===" | |
install_python_modules | |
echo "=== Installing boost ===" | |
install_boost | |
echo "=== Installing lmdb ===" | |
install_lmdb | |
echo "=== Installing libtool ===" | |
install_libtool | |
echo "=== Installing snappy ===" | |
install_snappy | |
echo "=== Installing leveldb ===" | |
install_leveldb | |
echo "=== Installing openblas ===" | |
install_openblas | |
echo "=== The dependencies are installed! ===" | |
echo "You likely want to add the following lines to your .bashrc/.zshrc" | |
echo 'export LD_LIBRARY_PATH="'${LOCAL_INSTALL_DIR}'/lib64:'${LOCAL_INSTALL_DIR}'/lib:$LD_LIBRARY_PATH"' | |
echo 'export C_INCLUDE_PATH="'${LOCAL_INSTALL_DIR}'/include:$C_INCLUDE_PATH"' | |
echo 'export CPLUS_INCLUDE_PATH="'${LOCAL_INSTALL_DIR}'/include:$CPLUS_INCLUDE_PATH"' | |
echo "=== Once you've done that, you can run install_caffe by uncommenting below. ===" | |
# install_caffe |
There is a problem with cmake installation,
When installing gflags, it gives an error cmake command not found (line 129). Please can you look into the matter.
Error at line 374
Correct:
install_protobufs
Error at line 112 and 113.
Line should be swaped.
Correct:
untar_to_dir "cmake-3.2.3-Linux-x86_64.tar.gz"
cd ${LOCAL_INSTALL_DIR}
Thank you!
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Line 190 https://gist.github.com/achalddave/7f7323a36f85b6c6dd64#file-install_caffe-sh-L190
should have no space after -L before path. All the rest look correct.