-
-
Save jeesmon/00f479a896548aadc0b32ae184890b4e to your computer and use it in GitHub Desktop.
| #!/bin/bash -ex | |
| # yum install -y docker git patch jq | |
| # systemctl start docker | |
| # docker info | |
| ISTIO_VERSION=${ISTIO_VERSION:-1.19.3} | |
| MAJOR_ISTIO_VERSION=$(cut -f1-2 -d. <<< ${ISTIO_VERSION}) | |
| # Need a custom build-tools-proxy image for 1.18.3+ | |
| # https://istio.slack.com/archives/C6FCV6WN4/p1696463622534729 | |
| # https://github.com/istio/tools/pull/2566 | |
| git clone https://github.com/istio/tools.git --depth 1 | |
| cd tools | |
| git fetch --tags | |
| git checkout "${ISTIO_VERSION}" | |
| # Patch tools | |
| sed -i'' \ | |
| -e 's/FROM ubuntu:xenial AS clang_context_amd64/FROM ubuntu:jammy AS clang_context_amd64/' \ | |
| -e 's/FROM ubuntu:xenial AS build_env_proxy_amd64/FROM ubuntu:jammy AS build_env_proxy_amd64/' \ | |
| -e 's/ENV UBUNTU_RELEASE_CODE_NAME=xenial/ENV UBUNTU_RELEASE_CODE_NAME=jammy/' \ | |
| -e 's/ENV DOCKER_VERSION=5:20.10.7~3-0~ubuntu/ENV DOCKER_VERSION=5:20.10.14~3-0~ubuntu/' \ | |
| -e 's/ENV CONTAINERD_VERSION=1.4.6-1/ENV CONTAINERD_VERSION=1.6.12-1/' \ | |
| -e 's/python \\/#python \\/' \ | |
| docker/build-tools/Dockerfile | |
| # Build tools | |
| cd docker/build-tools/ | |
| DRY_RUN=true ./build-and-push.sh | |
| cd ../../.. | |
| git clone https://github.com/istio/proxy.git --depth 1 | |
| pushd proxy | |
| git fetch --tags | |
| git checkout "${ISTIO_VERSION}" | |
| export GOOS=linux | |
| # Patch Makefile for BAZEL_BIN_PATH in 1.19.3 | |
| # https://github.com/istio/proxy/pull/5087 | |
| sed -i '/exportcache:/i \ | |
| exportcache: BAZEL_BIN_PATH ?= $(shell bazel info $(BAZEL_BUILD_ARGS) $(BAZEL_CONFIG_CURRENT) bazel-bin)' \ | |
| Makefile.core.mk | |
| # Compile envoy with FIPS: https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/security/ssl#fips-140-2 | |
| echo "build --define boringssl=fips" >> .bazelrc | |
| IMG=gcr.io/istio-testing/build-tools-proxy:release-${MAJOR_ISTIO_VERSION}-latest-amd64 BUILD_WITH_CONTAINER=1 BAZEL_BUILD_ARGS=--config=release TARGET_OS=linux make build_wasm build build_envoy exportcache | |
| popd | |
| git clone https://github.com/istio/istio.git --depth 1 | |
| pushd istio | |
| git fetch --tags | |
| git checkout "${ISTIO_VERSION}" | |
| # Pre-built binaries need to copied with SHA in name, otherwise build process will download it from gc bucket | |
| # https://github.com/istio/istio/blob/1.18.1/bin/init.sh#L106 | |
| # Populate the git version for istio/proxy (i.e. Envoy) | |
| # PROXY_REPO_SHA="${PROXY_REPO_SHA:-$(grep PROXY_REPO_SHA istio.deps -A 4 | grep lastStableSHA | cut -f 4 -d '"')}" | |
| PROXY_REPO_SHA=$(jq -r '.[] | select(.name == "PROXY_REPO_SHA").lastStableSHA' istio.deps) | |
| # Copy locally built binaries | |
| mkdir -p out/linux_amd64/release | |
| cp -f ../proxy/out/linux_amd64/envoy out/linux_amd64/release/envoy-${PROXY_REPO_SHA} | |
| cp -f out/linux_amd64/release/envoy-${PROXY_REPO_SHA} out/linux_amd64/release/envoy | |
| # Patch Makefile to use BoringCrypto: https://github.com/tetratelabs/istio/blob/tetrate-workflow/tetrateci/docs/fips.md | |
| sed -i'' -e 's%GOOS=linux%CGO_ENABLED=1 GOEXPERIMENT=boringcrypto GOOS=linux%' Makefile.core.mk | |
| # Envoy built with BoringSSL requires libc++ installed in the docker image | |
| # Patch pilot/docker/Dockerfile.proxyv2 to install libc++ | |
| cat > Dockerfile.proxyv2.patch << EOF | |
| RUN apt-get update \ | |
| && apt-get upgrade -y \ | |
| && apt-get install -y libc++1 \ | |
| && apt-get autoremove -y \ | |
| && apt-get clean \ | |
| && rm -rf /tmp/* /var/tmp/* \ | |
| && rm -rf /var/lib/apt/lists/* | |
| EOF | |
| sed -i'' '/FROM ${BASE_DISTRIBUTION/r Dockerfile.proxyv2.patch' pilot/docker/Dockerfile.proxyv2 | |
| rm Dockerfile.proxyv2.patch | |
| # Build pilot and proxy | |
| TARGET_OS=linux make docker.pilot docker.proxyv2 | |
| # Confirm version | |
| docker run --rm --entrypoint="" localhost:5000/proxyv2 envoy --version | |
| docker run --rm --entrypoint="" localhost:5000/proxyv2 pilot-agent version | |
| docker run --rm --entrypoint="" localhost:5000/pilot pilot-discovery version | |
| # docker tag localhost:5000/proxyv2 quay.io/jeesmon/proxyv2:${ISTIO_VERSION} | |
| # docker tag localhost:5000/pilot:latest quay.io/jeesmon/pilot:${ISTIO_VERSION} | |
| # docker login quay.io | |
| # docker push quay.io/jeesmon/proxyv2:${ISTIO_VERSION} | |
| # docker push quay.io/jeesmon/pilot:${ISTIO_VERSION} | |
Also @jeesmon: I was thinking an update is in order for the script to still allow the version override to work now that we are creating a custom build image. It's currently hardcoded to use the 1.18 image, but if it's not found locally (like if we're building 1.19 so the local image is tagged differently) it will just pull the upstream 1.18 which might not work.
If all patch releases of a given Istio version are ok with their respective release-x.xx-latest(?) instead of needing a specific revision, we could just use something like this in the script:
ISTIO_VERSION=1.18.3
MAJOR_ISTIO_VERSION=$(cut -f1-2 -d. <<< $ISTIO_VERSION)
echo $MAJOR_ISTIO_VERSION
# 1.18and update the image URI to use the major version dynamically:
IMG=gcr.io/istio-testing/build-tools-proxy:release-$MAJOR_ISTIO_VERSION-latest-amd64 BUILD_WITH_CONTAINER=1 BAZEL_BUILD_ARGS=--config=release TARGET_OS=linux make build_wasm build build_envoy exportcacheso that we'd use the correct tools image that we generated earlier in the script.
Looking at build-and-push.sh though it looks like we could override the tag we build to instead, but it would require overriding PULL_BASE_SHA which I don't know all the implications of since I don't see it used anywhere other than to populate SUFFIX if present. Not aware if it would affect anything further down the line since we'd effectively be exporting it for anything called by the script to access. I'd say the simpler solution above is less intrusive too since it keeps the tags the same.
Any thoughts on this?
Hey folks. I am wondering if anyone has tried converting this script to work for arm64?
I know the Envoy docs say:
Currently, this option is only available on Linux-x86_64.
But if you look at this issue and this PR, it seems arm64 support has been added.
Updated for 1.18.5
@sspaeth-r7 Thanks for your suggestion. Updated gist accordingly.
@jacob-kuder @teddy-wahle I just built again today with 1.18.5 and updated the patch for build-tools. Sorry for not responding earlier. For some reason I'm not getting notified on gist comments. I need to check my junk folder.
I was using Amazon Linux 2. Can you make the changes in the patch manually and see? It's only 6 lines in tools/docker/build-tools/Dockerfile.
I think may be better to use a sed command instead of patch. I will look into it.
Changes in tools/docker/build-tools/Dockerfile are:
L655:
FROM ubuntu:jammy AS clang_context_amd64
L747-748:
FROM ubuntu:jammy AS build_env_proxy_amd64
ENV UBUNTU_RELEASE_CODE_NAME=jammy
L764-765:
ENV DOCKER_VERSION=5:20.10.14~3-0~ubuntu-${UBUNTU_RELEASE_CODE_NAME}
ENV CONTAINERD_VERSION=1.6.12-1
L868:
#python \
@jacob-kuder Updated gist to use sed instead of patch
For building 1.19.3, use the following patch in proxy repo before make command (L42) until this PR is merged.
diff --git a/Makefile.core.mk b/Makefile.core.mk
index b1c0e61a..bdbcb2ce 100644
--- a/Makefile.core.mk
+++ b/Makefile.core.mk
@@ -64,6 +64,7 @@ BAZEL_CONFIG_TSAN = # no working config
endif
BAZEL_CONFIG_CURRENT ?= $(BAZEL_CONFIG_DEV)
+BAZEL_BIN_PATH ?= $(shell bazel info $(BAZEL_BUILD_ARGS) $(BAZEL_CONFIG_CURRENT) bazel-bin)
TEST_ENVOY_TARGET ?= //:envoy
TEST_ENVOY_DEBUG ?= trace
@jacob-kuder Updated gist to use
sedinstead ofpatch
Thanks so much! I'll give the new script a whirl on my environment and I'll let you know how it goes.
@jacob-kuder Which linux distro do you use for your host? I tried only on Amazon Linux 2. I think someone else successfully built on Ubuntu. Can we please collaborate on this thread in istio slack? https://istio.slack.com/archives/C6FCV6WN4/p1696463622534729
Updated for 1.19.3
Experimental distroless build script: https://gist.github.com/jeesmon/e5672fdc6362156ad756295e1b5b1521
@jacob-kuder Which linux distro do you use for your host? I tried only on Amazon Linux 2. I think someone else successfully built on Ubuntu. Can we please collaborate on this thread in istio slack? https://istio.slack.com/archives/C6FCV6WN4/p1696463622534729
Can confirm have successfully built on Ubuntu 20.
We can remove the BAZEL_BIN_PATH fix for exportcache now since they've merged the PR to master and cherry picked the changes into release-1.18 as well as release-1.19. 🎉
Also the version variable at the top no longer allows for override. Not sure if that was intentional. We had been setting ISTIO_VERSION when calling the script like:
ISTIO_VERSION=1.19.3 ./istio-fips-build.shwhich worked when the line:
ISTIO_VERSION=${ISTIO_VERSION:-1.18.5}was at the top but not anymore.
@sspaeth-r7 ISTIO_VERSION variable is fixed. As we are checking out proxy version by tag, exportcache fix is still not there yet: https://github.com/istio/proxy/blob/1.19.3/Makefile.core.mk#L177. May be wait until 1.19.4 is released?
Ah good point. I mistakenly thought we were using the release branches for a minute. Yeah I guess we have to wait since they wouldn't change their existing tags.
I am trying to build this for istio 1.20.0 version on amazon linux2 box. I am getting the following errors:
- export GOOS=linux
- GOOS=linux
- echo 'build --define boringssl=fips'
- IMG=gcr.io/istio-testing/build-tools-proxy:release-1.20-latest-amd64
- BUILD_WITH_CONTAINER=1
- BAZEL_BUILD_ARGS=--config=release
- TARGET_OS=linux
- make build_wasm build build_envoy exportcache
make[1]: *** No rule to make target 'build_wasm'. Stop.
make: *** [build_wasm] Error 2
I removed build_wasm target and reran from that step
ERROR: /home/.cache/bazel/_bazel_user/1e0bb3bee2d09d2e4ad3523530d3b40c/external/boringssl_fips/BUILD.bazel:25:8: Executing genrule @boringssl_fips//:build failed: (Exit 1): bash failed: error executing command (from target @boringssl_fips//:build) /bin/bash -c ... (remaining 1 argument skipped)
Use --sandbox_debug to see verbose messages from the sandbox and retain the sandbox build root for debugging
clang: error while loading shared libraries: libtinfo.so.6: cannot open shared object file: No such file or directory
/home/.cache/bazel/_bazel_user/1e0bb3bee2d09d2e4ad3523530d3b40c/sandbox/processwrapper-sandbox/2/execroot/io_istio_proxy/external/boringssl_fips /home/.cache/bazel/_bazel_user/1e0bb3bee2d09d2e4ad3523530d3b40c/sandbox/processwrapper-sandbox/2/execroot/io_istio_proxy
ERROR: Clang version doesn't match.
ERROR: /work/extensions/common/BUILD:100:24 Linking extensions/common/proto_util_test failed: (Exit 1): bash failed: error executing command (from target @boringssl_fips//:build) /bin/bash -c ... (remaining 1 argument skipped)
Use --sandbox_debug to see verbose messages from the sandbox and retain the sandbox build root for debugging
INFO: Elapsed time: 95.662s, Critical Path: 73.61s
INFO: 512 processes: 66 internal, 446 processwrapper-sandbox.
FAILED: Build did NOT complete successfully
Makefile.core.mk:56: recipe for target 'build' failed
make[1]: *** [build] Error 1
make: *** [build] Error 2
@vinodshoreline I did not have any issues building 1.20.x after successfully removing build_wasm. Please make sure the build tools version is matching for 1.20.x as well.
@stanpalatnik are you able to build 1.20.3? We're getting a checksum mismatch error on that version using the same build script that works fine for 1.20.2.
@sspaeth-r7 & whoever is facing the checksum issue : istio/istio#49561
Edit: Looks like @sspaeth-r7 was the one to report the issue in the istio slack :)
@vinodshoreline not sure if you managed to figure out the issue, but I found for the 1.20.2 tag the tools dockerfile had changed and the sed lines in the script were doing nothing so it was still using bionic which does not have libncurses6 available and it, at some point, became required by the envoy fips build.
Make the changes from this PR and it should build.
Hey folks. I am wondering if anyone has tried converting this script to work for
arm64?I know the Envoy docs say:
Currently, this option is only available on Linux-x86_64.But if you look at this issue and this PR, it seems
arm64support has been added.
@teddy-wahle / anyone else: did you ever have any luck with building Istio for ARM? I see you have a fork for it which is basically the same thing I tried and unfortunately I received a strange error:
gcc: error: unrecognized command-line option '-m64'
I'm currently trying to build with a couple architecture variables set but not sure it'll make a difference.
For anyone else who may be trying to solve the above issue, it looks like setting DOCKER_ARCHITECTURES=linux/arm64 before make build docker.pilot docker.proxyv2 did the trick:
DOCKER_ARCHITECTURES=linux/arm64 <other_vars> make build docker.pilot docker.proxyv2
The build_wasm step can't find emscripten, or a particular file within emscripten. I lifted the effective command out to re-run it independently and try to debug but the same error occurs.
Edit: Running this build on MacOS, probably need to run this on Linux
(⎈)➜ proxy git:(1.19.3) ✗ vim ../istio-fips-build-old.sh
(⎈)➜ proxy git:(1.19.3) ✗ MAJOR_ISTIO_VERSION=1.19 IMG=gcr.io/istio-testing/build-tools-proxy:release-${MAJOR_ISTIO_VERSION}-latest-amd64 BUILD_WITH_CONTAINER=1 BAZEL_BUILD_ARGS='--config=release --sandbox_debug' TARGET_OS=linux make build_wasm build build_envoy exportcache
export PATH=/usr/lib/llvm/bin:/usr/local/go/bin:/gobin:/usr/local/google-cloud-sdk/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin CC=clang CXX=clang++ && bazel build --config=release --sandbox_debug --config=libc++ --config=release //extensions:metadata_exchange.wasm
Starting local Bazel server and connecting to it...
WARNING: The following configs were expanded more than once: [release]. For repeatable flags, repeats are counted twice and may lead to unexpected behavior.
WARNING: The following configs were expanded more than once: [release]. For repeatable flags, repeats are counted twice and may lead to unexpected behavior.
INFO: Analyzed target //extensions:metadata_exchange.wasm (184 packages loaded, 12725 targets configured).
INFO: Found 1 target...
INFO: Deleting stale sandbox base /home/.cache/bazel/_bazel_user/1e0bb3bee2d09d2e4ad3523530d3b40c/sandbox
ERROR: /home/.cache/bazel/_bazel_user/1e0bb3bee2d09d2e4ad3523530d3b40c/external/com_google_protobuf/src/google/protobuf/BUILD.bazel:348:11: Compiling src/google/protobuf/any_lite.cc failed: (Exit 2): process-wrapper failed: error executing command
(cd /home/.cache/bazel/_bazel_user/1e0bb3bee2d09d2e4ad3523530d3b40c/sandbox/processwrapper-sandbox/16/execroot/io_istio_proxy && \
exec env - \
BAZEL_COMPILER=clang \
BAZEL_CXXOPTS='-stdlib=libc++' \
BAZEL_LINKLIBS=-l%:libc++.a:-l%:libc++abi.a \
BAZEL_LINKOPTS=-lm:-pthread \
CC=clang \
CXX=clang++ \
CXXFLAGS='-stdlib=libc++' \
EMCC_WASM_BACKEND=1 \
EM_BIN_PATH=external/emsdk \
EM_CONFIG_PATH=external/emsdk/emscripten_toolchain/emscripten_config \
LDFLAGS='-stdlib=libc++' \
PATH=/home/.cache/bazelisk/downloads/bazelbuild/bazel-6.3.2-linux-arm64/bin:/usr/lib/llvm/bin:/usr/local/go/bin:/gobin:/usr/local/google-cloud-sdk/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
PWD=/proc/self/cwd \
TMPDIR=/tmp \
/home/.cache/bazel/_bazel_user/install/690906d82aa78975b597bd4f0948f7cd/process-wrapper '--timeout=0' '--kill_delay=15' '--stats=/home/.cache/bazel/_bazel_user/1e0bb3bee2d09d2e4ad3523530d3b40c/sandbox/processwrapper-sandbox/16/stats.out' external/emsdk/emscripten_toolchain/emcc.sh '--sysroot=external/emsdk/emscripten/cache/sysroot' -fdiagnostics-color -fno-exceptions -fno-strict-aliasing -funsigned-char -no-canonical-prefixes '-std=gnu++17' -nostdinc -nostdinc++ -DNDEBUG -fomit-frame-pointer -O2 -Wall '-DBAZEL_CURRENT_REPOSITORY="com_google_protobuf"' -iquote external/com_google_protobuf -iquote bazel-out/wasm-opt-ST-2c0208fb70bf/bin/external/com_google_protobuf -iquote external/com_google_absl -iquote bazel-out/wasm-opt-ST-2c0208fb70bf/bin/external/com_google_absl -iquote external/utf8_range -iquote bazel-out/wasm-opt-ST-2c0208fb70bf/bin/external/utf8_range -Ibazel-out/wasm-opt-ST-2c0208fb70bf/bin/external/com_google_protobuf/src/google/protobuf/_virtual_includes/protobuf_lite -Ibazel-out/wasm-opt-ST-2c0208fb70bf/bin/external/com_google_protobuf/src/google/protobuf/_virtual_includes/arena -Ibazel-out/wasm-opt-ST-2c0208fb70bf/bin/external/com_google_protobuf/src/google/protobuf/_virtual_includes/arena_align -Ibazel-out/wasm-opt-ST-2c0208fb70bf/bin/external/com_google_protobuf/src/google/protobuf/stubs/_virtual_includes/lite -Ibazel-out/wasm-opt-ST-2c0208fb70bf/bin/external/com_google_protobuf/src/google/protobuf/_virtual_includes/port_def -Ibazel-out/wasm-opt-ST-2c0208fb70bf/bin/external/com_google_protobuf/src/google/protobuf/_virtual_includes/arena_allocation_policy -Ibazel-out/wasm-opt-ST-2c0208fb70bf/bin/external/com_google_protobuf/src/google/protobuf/_virtual_includes/arena_config -Ibazel-out/wasm-opt-ST-2c0208fb70bf/bin/external/com_google_protobuf/src/google/protobuf/_virtual_includes/arena_cleanup -Ibazel-out/wasm-opt-ST-2c0208fb70bf/bin/external/com_google_protobuf/src/google/protobuf/_virtual_includes/string_block -Ibazel-out/wasm-opt-ST-2c0208fb70bf/bin/external/com_google_protobuf/src/google/protobuf/_virtual_includes/varint_shuffle -Ibazel-out/wasm-opt-ST-2c0208fb70bf/bin/external/com_google_protobuf/src/google/protobuf/io/_virtual_includes/io -Ibazel-out/wasm-opt-ST-2c0208fb70bf/bin/external/com_google_protobuf/src/google/protobuf/io/_virtual_includes/io_win32 -O3 -DHAVE_ZLIB -Woverloaded-virtual -Wno-sign-compare -Wno-nonnull -Werror -Wno-error -iwithsysroot/include/c++/v1 -iwithsysroot/include/compat -iwithsysroot/include -isystem external/emsdk/lib/clang/15.0.0/include -MD -MF bazel-out/wasm-opt-ST-2c0208fb70bf/bin/external/com_google_protobuf/src/google/protobuf/_objs/protobuf_lite/any_lite.d -c external/com_google_protobuf/src/google/protobuf/any_lite.cc -o bazel-out/wasm-opt-ST-2c0208fb70bf/bin/external/com_google_protobuf/src/google/protobuf/_objs/protobuf_lite/any_lite.o -Wno-builtin-macro-redefined '-D__DATE__="redacted"' '-D__TIMESTAMP__="redacted"' '-D__TIME__="redacted"' -Werror)
python3: can't open file '/home/.cache/bazel/_bazel_user/1e0bb3bee2d09d2e4ad3523530d3b40c/sandbox/processwrapper-sandbox/16/execroot/io_istio_proxy/external/emsdk/emscripten/emcc.py': [Errno 2] No such file or directory
Target //extensions:metadata_exchange.wasm failed to build
Use --verbose_failures to see the command lines of failed build steps.
INFO: Elapsed time: 7.598s, Critical Path: 1.83s
INFO: 27 processes: 13 internal, 14 processwrapper-sandbox.
FAILED: Build did NOT complete successfully
Makefile.core.mk:87: recipe for target 'build_wasm' failed
make[1]: *** [build_wasm] Error 1
make: *** [build_wasm] Error 2
Yeah, I think I am wrong here after reading this thread. I don't see where the
build_wasmtarget is but that seems irrelevant.