Skip to content

Instantly share code, notes, and snippets.

@yzchen

yzchen/caffe-prerequirements.sh

Last active May 6, 2019 11:41
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save yzchen/16e7c13d5da0ec4142acb321d15e38f3 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save yzchen/16e7c13d5da0ec4142acb321d15e38f3 to your computer and use it in GitHub Desktop.
Download ZIP
Compile Caffe on HPC System without Root Access
Raw
caffe-prerequirements.sh
#!/bin/bash
module use -a /project/k1341/.usr/local/share/modulefiles
module load common
# clear previous build
LOCALROOT=/project/k1341/.usr/local
rm -rf $LOCALROOT/bin $LOCALROOT/lib $LOCALROOT/lib64 $LOCALROOT/include $LOCALROOT/share/OpenCV $LOCALROOT/doc $LOCALROOT/man
# protobuf
cd /tmp/
rm -rf protobuf
git clone https://github.com/google/protobuf.git
cd protobuf/
./autogen.sh
./configure CXX=CC CC=cc CFLAGS="-fPIC -xAVX -axCORE-AVX2" CXXFLAGS="-fPIC -xAVX -axCORE-AVX2" --prefix=$LOCALROOT
make -j && make install
# snappy
cd /tmp/
rm -rf snappy
git clone https://github.com/google/snappy.git
cd snappy
mkdir build && cd build
CFLAGS="-fPIC" CXXFLAGS="-fPIC" cmake -DCMAKE_CXX_COMPILER=CC -DCMAKE_C_COMPILER=cc -DCMAKE_INSTALL_PREFIX=$LOCALROOT ..
make -j && make install
# leveldb
cd /tmp/
rm -rf leveldb
git clone https://github.com/google/leveldb.git
cd leveldb
mkdir build && cd build
CFLAGS="-fPIC" CXXFLAGS="-fPIC" cmake -DCMAKE_CXX_COMPILER=CC -DCMAKE_C_COMPILER=cc -DCMAKE_INSTALL_PREFIX=$LOCALROOT ..
make -j && make install
# opencv
cd /tmp/
rm -rf opencv-2.4.8
wget 'https://github.com/Itseez/opencv/archive/2.4.8.tar.gz'
tar xzf 2.4.8.tar.gz
cd opencv-2.4.8/
mkdir build && cd build
CFLAGS="-fPIC" CXXFLAGS="-fPIC" cmake -DCMAKE_CXX_COMPILER=CC -DCMAKE_C_COMPILER=cc -DCMAKE_BUILD_TYPE=RELEASE -DBUILD_opencv_python=OFF -DCMAKE_INSTALL_PREFIX=$LOCALROOT -DBUILD_opencv_gpu=OFF -DENABLE_PRECOMPILED_HEADERS=OFF ..
make -j && make install
# boost
cd /tmp/
rm -rf boost_1_55_0
wget "http://sourceforge.net/projects/boost/files/boost/1.55.0/boost_1_55_0.tar.gz"
tar xzf boost_1_55_0.tar.gz
cd boost_1_55_0
./bootstrap.sh --prefix=$LOCALROOT
./b2 -j 16 && ./b2 install
# lmdb
cd /tmp
rm -rf lmdb
git clone https://gitlab.kaust.edu.sa/cheny0l/lmdb.git
cd lmdb/libraries/liblmdb
make CXX=CC CC=cc CXXFLAGS='-fPIC' CFLAGS='-fPIC' -j && make prefix=$LOCALROOT install
# gflags
cd /tmp/
rm -rf gflags
git clone https://github.com/gflags/gflags
cd gflags
mkdir build && cd build
CFLAGS="-fPIC" CXXFLAGS="-fPIC" cmake -DCMAKE_CXX_COMPILER=CC -DCMAKE_C_COMPILER=cc -DCMAKE_INSTALL_PREFIX=$LOCALROOT ..
make -j && make install
# glog
cd /tmp/
rm -rf glog
git clone https://github.com/google/glog
cd glog
./autogen.sh
./configure CXX=CC CC=cc CFLAGS="-fPIC -xAVX -axCORE-AVX2" CXXFLAGS="-fPIC -xAVX -axCORE-AVX2" --prefix=$LOCALROOT
make -j && make install
Raw
caffe-without-root.md

Prepare modules

mkdir -p /project/k1341/.usr/local/share/modulefiles
  1. create a file common
#%Module

module swap PrgEnv-cray/6.0.4 PrgEnv-intel
module load cmake/3.10.2
module load hdf5/1.8.21

module load depends_caffe/1.0
  1. create a file depends_caffe/1.0
#%Module

set		root			/project/k1341/.usr/local
prepend-path	PATH			$root/bin
prepend-path	CPLUS_INCLUDE_PATH	$root/include
prepend-path	C_INCLUDE_PATH		$root/include
prepend-path	LD_LIBRARY_PATH		$root/lib:$root/lib64
prepend-path	LIBRARY_PATH		$root/lib:$root/lib64
prepend-path	MANPATH			$root/share

prepend-path	C_INCLUDE_PATH		/sw/xc40cle6/hdf5/1.8.21/cle6.0.5_intel18.0.1.163/include
prepend-path	CPLUS_INCLUDE_PATH	/sw/xc40cle6/hdf5/1.8.21/cle6.0.5_intel18.0.1.163/include
prepend-path	LIBRARY_PATH		/sw/xc40cle6/hdf5/1.8.21/cle6.0.5_intel18.0.1.163/lib
  1. load common module every time when you login to the system, add following to $HOME/.profile
if [ -d "/project/k1341" ]; then
  module use -a /project/k1341/.usr/local/share/modulefiles
  module load common
fi

Prepare other libraries (check here)

  1. Install protobuf:
cd /tmp/
rm -rf protobuf
git clone https://github.com/google/protobuf.git
cd protobuf/
./autogen.sh
./configure --prefix=/project/k1341/.usr/local
make -j && make install
  1. Install snappy:
cd /tmp/
rm -rf snappy
git clone https://github.com/google/snappy.git
cd snappy
mkdir build && cd build
cmake -D CMAKE_INSTALL_PREFIX=/project/k1341/.usr/local ..
make -j && make install
  1. Install leveldb:
cd /tmp/
rm -rf leveldb
git clone https://github.com/google/leveldb.git
cd leveldb
mkdir build && cd build
CXXFLAGS="-fPIC" cmake -D CMAKE_INSTALL_PREFIX=/project/k1341/.usr/local ..
make -j && make install
  1. Install OpenCV:
cd /tmp/
wget 'https://github.com/Itseez/opencv/archive/2.4.8.tar.gz'
tar xzf 2.4.8.tar.gz
cd opencv-2.4.8/
mkdir build && cd build
cmake -D CMAKE_BUILD_TYPE=RELEASE -D BUILD_opencv_python=OFF -D CMAKE_INSTALL_PREFIX=/project/k1341/.usr/local -D BUILD_opencv_gpu=OFF -DENABLE_PRECOMPILED_HEADERS=OFF ..
make -j && make install
  1. Install Boost:
cd /tmp/
rm -rf boost_1_55_0
wget "http://sourceforge.net/projects/boost/files/boost/1.55.0/boost_1_55_0.tar.gz"
tar xzf boost_1_55_0.tar.gz
cd boost_1_55_0
./bootstrap.sh --prefix=/project/k1341/.usr/local
./b2 -j 16 && ./b2 install
  1. Install lmdb:
cd /tmp
rm -rf lmdb
git clone https://github.com/LMDB/lmdb
cd lmdb/libraries/liblmdb
make -j && make prefix=/project/k1341/.usr/local install
  1. Install gflags:
cd /tmp/
rm -rf gflags
git clone https://github.com/gflags/gflags
cd gflags
mkdir build && cd build
CXXFLAGS="-fPIC" cmake -D CMAKE_INSTALL_PREFIX=/project/k1341/.usr/local ..
make -j && make install
  1. Install glog:
cd /tmp/
rm -rf glog
git clone https://github.com/google/glog
cd glog
./autogen.sh
./configure --prefix=/project/k1341/.usr/local
make -j && make install

Install Caffe

git clone https://github.com/intel/caffe
cd caffe

  • change Makefile.mkldnn

    8. MKLDNN_CXX := CC
9. MKLDNN_CC := cc

  • change Makefile.config (run cp Makefile.config.example Makefile.config first)

    # Whatever else you find you need goes here.
INCLUDE_DIRS := $(PYTHON_INCLUDE) ${SERIAL_HDF5_INCLUDE} /project/k1341/.usr/local/include /usr/local/include
LIBRARY_DIRS := $(PYTHON_LIB) /project/k1341/.usr/local/lib /project/k1341/.usr/local/lib64 /usr/local/lib /usr/lib

# Allow distributed training
ALLOW_LMDB_NOLOCK := 1

  • final make

    export CRAYPE_LINK_TYPE=dynamic
make all -j 12
./examples/mnist/train_lenet.sh

  • make on Shaheen

    module use -a /project/k1341/.usr/local/share/modulefiles
module load common
export CRAYPE_LINK_TYPE=dynamic
cd /project/k1341/softwares/caffe/
make

  • run benchmark

    git checkout origin/DAWN_Bench
git checkout -b DAWN_Bench

less models/intel_optimized_models/multinode/resnet_50_64_nodes_2k_batch/solver.prototxt
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment