m1-mac-source-builds-tensorflow-tf_io-tf_addons-tf_text.sh

#!/usr/bin/env bash

########################################################################################################################
# This is a build script for misc. TensorFlow dependencies which do NOT have pre-compiled packages available for
# M-series (Apple Silicon) macs.
#
# Who: team members who use M1 macs and would like to run the project
#
# Caution: This script will use all available processors (rendering the machine relatively unusable) and will take a
# significant amount of time (about 2h 0m 15s on an M1 mac).
#
# Please set aside sufficient time to run this build script.
########################################################################################################################

# WARNING: MUST run prior to enabling bash strict mode or else it will fail
source deactivate || true # deactivate any current venvs

# Set Bash "strict" mode
# see: http://redsymbol.net/articles/unofficial-bash-strict-mode/
set -euo pipefail
set -x # Print command traces for debugging (feel free to comment this out if it is too noisy for you)
IFS=$'\n\t'

# Variables used throughout script
PROJECT_ROOT="$(git rev-parse --show-toplevel || pwd)"
SRC_BUILD_FOLDER_NAME="source_builds"
SRC_BUILDS_ROOT="${PROJECT_ROOT}/${SRC_BUILD_FOLDER_NAME}"

# In Monterey, there is an issue with OS versions not being properly inferred for built wheels, resulting in wheels
# using older platform specifiers. When attempting to install these wheels, they may erroneously be reported as,
# "... *.whl is not a supported wheel on this platform." However, they *are* supported, and the error is due solely to
# metadata mismatch (i.e., if you manually changed the metadata, it would install and work just fine).
# Manually set this C-compiler flag so wheels are correctly built for the host platform.
MACOSX_MAJOR_VERSION="$(sw_vers -productVersion | awk -F. '{print $1}')" # e.g., 12
export MACOSX_DEPLOYMENT_TARGET="${MACOSX_MAJOR_VERSION}.0"              # e.g., 12.0
ARCH="$(arch)"                                                           #i.e., arm64

# Install Bazel build tool for tensorflow ecosystem projects source builds
brew install bazelisk
export USE_BAZEL_VERSION="4.2.2" # the max version known to work with the below TF versions combination as of 2021/11/13

# TF ecosystem versions as of 2021/11/13
TF_VERSION="2.7.0"
TF_TEXT_VERSION="2.7.3"
TF_IO_VERSION="0.22.0"     # built against TF version 2.7.0
TF_ADDONS_VERSION="0.15.0" # built against TF version 2.7.0

## Main build logic

main() {
  pushd "$(pwd)" # save the current directory to restore after the build process completes
  sudo -v        # login at start of script so that remainder can run without interruption

  build_dependencies_from_source

  sudo -k # logout of sudo
  popd
  printf "\n************************************************************************************"
  printf "\nSource packages built:\n\n%s" "$(ls "$(get_source_build_wheel_output_root)")"
  printf "\n************************************************************************************"
}

build_dependencies_from_source() {
  setup_src_build_dir_venv_and_enter

  build_tensorflow
  build_tensorflow_io # depends on tensorflow

  build_tensorflow_addons # depends on tensorflow, tensorflow-io
  build_tensorflow_text   # depends on tensorflow, tensorflow-io
}

## Build commands
setup_src_build_dir_venv_and_enter() {
  mkdir -p "${SRC_BUILDS_ROOT}"
  cd "${SRC_BUILDS_ROOT}"
  python -m venv .venv
  source .venv/bin/activate
  pip install pip wheel numpy
  pip install keras_preprocessing --no-deps
}

build_tensorflow() {
  clone_setup_and_enter_tf_proj_dir "tensorflow" "v${TF_VERSION}"
  yes '' | ./configure || true #fallback to true to prevent non-zero exit code of configuration script from making the script exit early

  local WHEEL_OUTPUT_ROOT="./tmp/tensorflow_pkg"
  # sudo or else won't pick up the darwin_arm64 cc toolchain...
  sudo bazel build --config="macos_${ARCH}" //tensorflow/tools/pip_package:build_pip_package
  ./bazel-bin/tensorflow/tools/pip_package/build_pip_package "${WHEEL_OUTPUT_ROOT}"

  copy_wheel_to_source_builds_packages_dir "${WHEEL_OUTPUT_ROOT}/tensorflow-${TF_VERSION}-"*.whl
}

build_tensorflow_io() {
  # https://github.com/tensorflow/io/issues/1298#issuecomment-958610412
  clone_setup_and_enter_tf_proj_dir "io" "v${TF_IO_VERSION}"
  python setup.py -q bdist_wheel --project tensorflow_io_gcs_filesystem
  copy_wheel_to_source_builds_packages_dir ./dist/tensorflow_io_gcs_filesystem-*.whl
}

build_tensorflow_addons() {
  # https://github.com/tensorflow/addons
  clone_setup_and_enter_tf_proj_dir "addons" "v${TF_ADDONS_VERSION}"

  local WHEEL_OUTPUT_ROOT="./tmp/tensorflow_addon_pkg"
  # Install Tensorflow IO & Tensorflow needed for text (built in the previous steps)
  # Note: TF depends on TF IO, so must be installed in this order
  pip install "$(get_source_build_wheel_output_root)/tensorflow_io_gcs_filesystem-${TF_IO_VERSION}-"*"${ARCH}.whl"
  pip install "$(get_source_build_wheel_output_root)/tensorflow-${TF_VERSION}-"*"${ARCH}.whl"
  python ./configure.py

  # Update to compatible platform extension [OPTIONAL; here for conformity ]
  sed -i '' "s/11_0_arm64/${MACOSX_DEPLOYMENT_TARGET}-$(arch)/g" "./build_deps/build_pip_pkg.sh"
  bazel build build_pip_pkg
  bazel-bin/build_pip_pkg "${WHEEL_OUTPUT_ROOT}"

  copy_wheel_to_source_builds_packages_dir "${WHEEL_OUTPUT_ROOT}/tensorflow_addons-"*.whl
}

build_tensorflow_text() {

  # Note: built bazel bin exported to path, so must restore old path upon function exit
  local OLD_PATH="${PATH}"
  local ORIG_BAZEL="$(which bazel)"
  local ORIG_BAZEL_VER="$(get_bazel_version)"

  _build_bazel3_7_2

  ## https://github.com/tensorflow/text
  clone_setup_and_enter_tf_proj_dir "text" "v${TF_TEXT_VERSION}"
  # Install deps needed for text (built in the previous steps)
  pip install "$(get_source_build_wheel_output_root)/tensorflow"*"${ARCH}.whl"

  # Update to compatible platform extension [MANDATORY]
  sed -i '' "s/10.9-x86_64/${MACOSX_DEPLOYMENT_TARGET}-$(arch)/g" "./oss_scripts/pip_package/build_pip_package.sh"
  ./oss_scripts/run_build.sh
  copy_wheel_to_source_builds_packages_dir ./tensorflow_text-*.whl

  rm .bazelversion
  export PATH="${OLD_PATH}"
  if [ "${ORIG_BAZEL_VER}" != "$(get_bazel_version)" ]; then
    echo "Original bazel on path (${ORIG_BAZEL}) not the same bazel that is on path now! ($(which bazel)"
    exit 2
  fi
}

## Bazel 3.7.2 needed for tensorflow-text
_build_bazel3_7_2() {
  cd "${SRC_BUILDS_ROOT}"

  local BAZEL_VER="3.7.2"
  local BAZEL_RELEASES_URL="https://github.com/bazelbuild/bazel/releases/download"
  local BAZEL_DIST_ARCHIVE="bazel-${BAZEL_VER}-dist.zip"
  local BAZEL_DIST_ARCHIVE_CHECKSUM="${BAZEL_DIST_ARCHIVE}.sha256"

  local BAZEL_DIST_PATH="${SRC_BUILDS_ROOT}/bazel"
  mkdir -p "${BAZEL_DIST_PATH}"

  # WARNING: do NOT try to be clever and pipe `curl` with `-L` directly to `tar`; the redirect will grab the GitHub source
  # tree which is NOT what we want since only the distribution archive is configured to boostrap Bazel.
  # https://docs.bazel.build/versions/main/install-compile-source.html#download-distfile
  wget "${BAZEL_RELEASES_URL}/${BAZEL_VER}/${BAZEL_DIST_ARCHIVE}"
  wget "${BAZEL_RELEASES_URL}/${BAZEL_VER}/${BAZEL_DIST_ARCHIVE_CHECKSUM}"
  sha256sum -c "${BAZEL_DIST_ARCHIVE_CHECKSUM}"
  unzip "${BAZEL_DIST_ARCHIVE}" -d "${BAZEL_DIST_PATH}"
  rm "${BAZEL_DIST_ARCHIVE}" "${BAZEL_DIST_ARCHIVE_CHECKSUM}"
  cd "${BAZEL_DIST_PATH}"

  # https://github.com/bazelbuild/bazel/issues/11399#issuecomment-628945756
  brew install openjdk@11
  export JAVA_HOME="$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home"

  # Specify EMBED_LABEL to override auto-generated (potentially-broken) bazel version imputed on dist builds
  env \
    EXTRA_BAZEL_ARGS="--host_javabase=@local_jdk//:jdk" \
    EMBED_LABEL="${BAZEL_VER}" \
    bash ./compile.sh

  # Note: would ideally like to just return `BAZEL_BIN_PATH`
  # and perform the below in the calling function, but for some reason
  # this doesn't make it out of the function to be captured; leaving
  # it here is the only way to make it work
  local BAZEL_BIN_PATH="$(pwd)/output"
  export PATH="${BAZEL_BIN_PATH}:${PATH}"
  if [ "${BAZEL_VER}" != "$(get_bazel_version)" ]; then
    echo "bazel on path ($(which bazel)) not the same bazel that was just built! (${BAZEL_BIN_PATH}/bazel)"
    exit 1
  fi
  bazel --version # Log bazel version as a sanity check
}

## Helper commands
clone_setup_and_enter_tf_proj_dir() {
  local TF_PROJ="${1}"
  local GIT_TAG="${2}"
  clone_and_enter_proj_dir "tensorflow/${TF_PROJ}"
  git checkout "${GIT_TAG}"
  get_bazel_version >.bazelversion # Use the newest bazel version; versions < 4.0 not supported on M1 and tensorflow sets this to version 3.7.1 as of 2021/11/13
}

clone_and_enter_proj_dir() {
  local GIT_REPO="${1}"
  cd "${SRC_BUILDS_ROOT}"
  local PROJ_DIR="$(basename ${GIT_REPO})"
  rm -rf "${PROJ_DIR}"
  git clone "https://github.com/${GIT_REPO}.git" && cd "${PROJ_DIR}"
}

get_bazel_version() {
  bazel --version | awk '{print $2}'
}

copy_wheel_to_source_builds_packages_dir() {
  local WHEEL_PATH="${1}"
  cp -v "${WHEEL_PATH}" "$(get_source_build_wheel_output_root)"
}

get_source_build_wheel_output_root() {
  local WHEEL_OUTPUT_ROOT="${PROJECT_ROOT}/packages/${SRC_BUILD_FOLDER_NAME}"
  mkdir -p "${WHEEL_OUTPUT_ROOT}"
  echo "${WHEEL_OUTPUT_ROOT}"
}

## Execute build process

main

@TeoZosa thanks for the reply Teo. No , no other error messages. But for now I could use the prebuilt wheel from here https://github.com/sun1638650145/Libraries-and-Extensions-for-TensorFlow-for-Apple-Silicon/releases/tag/v2.8 . It worked to my surprise. The io and tf wheels were built but i guess add-on build failed because of "*" in the filename. Fo some reason script could not expand it.