TensorFlow Build Script

1.12.0.mk

#!/usr/bin/make -f
# Makefile for installing Tensorflow
#
# -Andreas Herten, 1 Feb 2019

SHELL = /bin/bash

NAME = tensorflow

VERSION = 1.12.0

DEP_COMPILER = gcc/5.4.0
DEP_PYTHON_PACKAGES = numpy/1.16.0 keras_applications/1.0.7 keras_preprocessing/1.0.8 h5py/2.9.0
DEP_CUDA = cuda/10.0.130
DEP_CUDNN = cuDNN/7.4.2
DEP_NCCL = nccl/2.3.7
DEP_ETC = bazel/0.19.2 protobuf-python/3.6.1

PYTHONVERSION = 3.6

######

CUDNN_LIBPATH = $(shell module show ${DEP_CUDNN} 2>&1 | grep LD_LIBRARY_PATH | cut -d "\"" -f4)
CUDNN_VERSION = $(shell ls ${CUDNN_LIBPATH} | grep -o "[0-9].[0-9].[0-9]")
CUDNN_BASEPATH = $(CUDNN_LIBPATH:%/lib=%)

LONGCUDA = $(shell echo ${DEP_CUDA} | grep -Po "\d{1,3}.\d{1,3}.\d{1,3}")
SHORTCUDA = $(shell echo ${DEP_CUDA} | grep -Po "(\d{1,3}\.\d{1,3})")

NCCL_VERSION = $(shell echo ${DEP_NCCL} | grep -Po "\d{1,3}.\d{1,3}.\d{1,3}")
NCCL_PATH = $(shell dirname `module show  ${DEP_NCCL} |& grep LD_LIBRARY_PATH | cut -d '"' -f4`)

VERSION_IDENTIFIER = $(shell echo "${VERSION}-${DEP_COMPILER}-${DEP_CUDA}" | sed 's./._.g')

PREFIX = /gpfs/software/opt/python/modules/${PYTHONVERSION}/${NAME}/${VERSION_IDENTIFIER}

SOURCEBASE = /gpfs/software/src/python/${NAME}
SOURCEDIR = ${SOURCEBASE}/source/${VERSION}

BUILDDIR = ${SOURCEBASE}/build/${VERSION_IDENTIFIER}

ifeq (${PYTHONVERSION},2.7)
	PYTHONVERSIONLONG = 2.7.14
	PIP = pip2
	PYTHON_EXE = python2
else ifeq (${PYTHONVERSION},3.6)
	PYTHONVERSIONLONG = 3.6.1
	PIP = pip3
	PYTHON_EXE = python3
else
$(error Please choose a valid PYTHONVERSION in header of Buildfile)
endif

DEP_PYTHON = python/${PYTHONVERSIONLONG}
MODULE_DEPS = ${DEP_COMPILER} ${DEP_PYTHON} ${DEP_PYTHON_PACKAGES} ${DEP_CUDA} ${DEP_CUDNN} ${DEP_NCCL} ${DEP_ETC}

NTHREADS = 20

.PHONY: hello all

hello:
	@echo "Building ${NAME}, version ${VERSION}, installing to ${PREFIX}"

all: hello all

.PHONY: download configure build install all
download: ${SOURCEDIR}/.downloaded
configure: ${SOURCEDIR}/.configured
patch: ${SOURCEDIR}/.patched
build: ${SOURCEDIR}/.built
wheel: ${BUILDDIR}/.wheeled
install: ${PREFIX}/.installed
all: download build install

${SOURCEDIR}/.downloaded:
	cd ${SOURCEBASE} && \
		mkdir -p ${SOURCEDIR} && \
		git clone -b v${VERSION} https://github.com/tensorflow/tensorflow.git ${SOURCEDIR}
	echo "$$(date)" > $@

${SOURCEDIR}/.configured: ${SOURCEDIR}/.downloaded
	cd ${dir $@}/ && \
	module load ${MODULE_DEPS} && \
		LD_LIBRARY_PATH=$$CUDA_ROOT/extras/CUPTI/lib64:$$LD_LIBRARY_PATH \
		PYTHON_BIN_PATH=`which ${PYTHON_EXE}` \
		PYTHON_LIB_PATH=`${PYTHON_EXE} -c "import sys; print(sys.path[-1])"` \
		TF_NEED_JEMALLOC=1 \
		TF_NEED_GCP=0 \
		TF_NEED_HDFS=0 \
		TF_NEED_S3=0 \
		TF_ENABLE_XLA=0 \
		TF_NEED_GDR=0 \
		TF_NEED_VERBS=0 \
		TF_NEED_OPENCL=0 \
		TF_NEED_CUDA=1 \
		TF_CUDA_VERSION=${SHORTCUDA} \
		CUDA_TOOLKIT_PATH=$$CUDA_ROOT \
		TF_CUDA_COMPUTE_CAPABILITIES=6.0 \
		TF_CUDNN_VERSION=${CUDNN_VERSION} \
		TF_CUDA_CLANG=0 \
		CUDNN_INSTALL_PATH=${CUDNN_BASEPATH} \
		GCC_HOST_COMPILER_PATH=`which gcc` \
		TF_NEED_MPI=0 \
		TF_NEED_KAFKA=0 \
		TF_NEED_OPENCL_SYCL=0 \
		TF_NEED_TENSORRT=0 \
		TF_SET_ANDROID_WORKSPACE=0 \
		CC_OPT_FLAGS="-mcpu=native" \
		TF_NEED_IGNITE=0 \
		TF_NEED_ROCM=0 \
		TF_NCCL_VERSION=${NCCL_VERSION} \
		NCCL_INSTALL_PATH=${NCCL_PATH} \
		TMP=/tmp/ \
			${SOURCEDIR}/configure
	echo "$$(date)" > $@

${SOURCEDIR}/.patched: ${SOURCEDIR}/.configured
	# For new bazel versions, the old tools/bazel.rc file is not read anymore
	cd ${SOURCEDIR} && \
		cat tools/bazel.rc >> .tf_configure.bazelrc
	
	# Patch for wrong Swig handling for CentOS 
	# See https://github.com/bazelbuild/bazel/issues/4053
	cd ${SOURCEDIR} && \
		patch -p0 < ../../swig.patch
	echo "$$(date)" > $@

${SOURCEDIR}/.built: ${SOURCEDIR}/.patched
	cd ${dir $@}/ && \
		module load ${MODULE_DEPS} && \
		export TMP=/tmp/ && \
		bazel \
			build \
			-s \
			--verbose_failures \
			--config=opt \
			--config=cuda \
			--action_env PYTHONPATH=$$PYTHONPATH \
			--distinct_host_configuration=false \
			//tensorflow/tools/pip_package:build_pip_package 
	echo "$$(date)" > $@

${BUILDDIR}/.wheeled: ${SOURCEDIR}/.built
	mkdir -p ${dir $@}
	cd ${SOURCEDIR}/ && \
		module load ${MODULE_DEPS} && \
		bazel-bin/tensorflow/tools/pip_package/build_pip_package ${dir $@}/
	echo "$$(date)" > $@

${PREFIX}/.installed: WHL = ${shell ls ${BUILDDIR}/*.whl}
${PREFIX}/.installed: ${BUILDDIR}/.wheeled
	mkdir -p ${dir $@}
	cd ${BUILDDIR}/ && \
		module load ${MODULE_DEPS} && \
			${PIP} install --prefix ${PREFIX} ${WHL}
	echo "$$(date)" > $@


### RESET & CLEAN

.PHONY: reset-download reset-patch reset-configure reset-build reset-wheel reset-install reset clean-build clean-source clean-configure clean-install clean-all clean-wheel clean

reset-download:
	rm ${SOURCEDIR}/.downloaded
reset-patch:
	rm ${SOURCEDIR}/.patched
reset-configure:
	rm ${SOURCEDIR}/.configured
reset-build:
	rm ${SOURCEDIR}/.built
reset-wheel:
	rm ${BUILDDIR}/.wheeled
reset-install:
	rm ${PREFIX}/.installed
reset: reset-download reset-patch reset-configure reset-build reset-wheel reset-install

clean-build:
	@echo "TF builds into the source directory. Consider `clean-source` or `clean-wheel`."
clean-source:
	rm -rf ${SOURCEDIR}/
clean-configure: reset-configure
	rm ${SOURCEDIR}/.tf_configure.bazelrc ${SOURCEDIR}/.bazelrc
clean-wheel:
	rm -rf ${BUILDDIR}/
clean-install:
	rm -rf ${PREFIX}/
clean-all: clean-wheel clean-source
clean:
	@echo "Please use make clean-all and be sure about it!"

# check:
# ifneq ("${CURRENTGCCVERSION}","${SUPPOSEGCCVERSION}")
# 	@$(error "This is suppose to be built with GCC ${SUPPOSEGCCVERSION}")
# else
# 	@echo "Passed all checks"
# endif

swig.patch

--- tensorflow/tensorflow.bzl	2017-11-01 21:21:13.000000000 +0100
+++ tensorflow/tensorflow.bzl	2017-11-09 11:49:41.536361562 +0100
@@ -1297,7 +1297,8 @@ def _py_wrap_cc_impl(ctx):
     args += [src.path]
     outputs = [ctx.outputs.cc_out, ctx.outputs.py_out]
     ctx.action(
+        use_default_shell_env = True,
         executable = ctx.executable._swig,
         arguments = args,
         inputs = list(inputs),
         outputs = outputs,