SDC 3.16.1 Docker Image build file to update AWS Java SDK libs (workaround for https://issues.streamsets.com/browse/SDC-10577)

build.sh

#!/usr/bin/env bash

SDC_VERSION=3.16.1

# Your SDC Docker Image Name
IMAGE_NAME=<your repo>/<your image>:$SDC_VERSION

# Cleanup old runs
rm -rf streamsets-datacollector-aws-lib

# Download and extract the SDC AWS Stage Lib
wget https://archives.streamsets.com/datacollector/$SDC_VERSION/tarball/streamsets-datacollector-aws-lib-$SDC_VERSION.tgz
tar -xvf streamsets-datacollector-aws-lib-$SDC_VERSION.tgz
rm streamsets-datacollector-aws-lib-$SDC_VERSION.tgz

# Remove the old AWS SDK libs
cd streamsets-datacollector-$SDC_VERSION/streamsets-libs/streamsets-datacollector-aws-lib/lib
rm -rf aws-java-sdk-*-1.11.385.jar

# Download the updated AWS SDK libs
BASE_AWS_SDK_URL=https://repo1.maven.org/maven2/com/amazonaws
wget $BASE_AWS_SDK_URL/aws-java-sdk-cloudwatch/1.11.813/aws-java-sdk-cloudwatch-1.11.813.jar
wget $BASE_AWS_SDK_URL/aws-java-sdk-core/1.11.813/aws-java-sdk-core-1.11.813.jar
wget $BASE_AWS_SDK_URL/aws-java-sdk-dynamodb/1.11.813/aws-java-sdk-dynamodb-1.11.813.jar
wget $BASE_AWS_SDK_URL/aws-java-sdk-iam/1.11.813/aws-java-sdk-iam-1.11.813.jar
wget $BASE_AWS_SDK_URL/aws-java-sdk-kinesis/1.11.813/aws-java-sdk-kinesis-1.11.813.jar
wget $BASE_AWS_SDK_URL//aws-java-sdk-kms/1.11.813/aws-java-sdk-kms-1.11.813.jar
wget $BASE_AWS_SDK_URL/aws-java-sdk-s3/1.11.813/aws-java-sdk-s3-1.11.813.jar
wget $BASE_AWS_SDK_URL/aws-java-sdk-sqs/1.11.813/aws-java-sdk-sqs-1.11.813.jar
wget $BASE_AWS_SDK_URL/aws-java-sdk-sts/1.11.813/aws-java-sdk-sts-1.11.813.jar

cd ../../../..

# Save only the new streamsets-datacollector-aws-lib dir
mv streamsets-datacollector-$SDC_VERSION/streamsets-libs/streamsets-datacollector-aws-lib .
rm -rf streamsets-datacollector-$SDC_VERSION

# Build the SDC Docker Image with your selection of 
# stage libs. Do not include the aws-lib in this list
# as the Dockerfile will copy this in from the local dir.  
# For this example I'll include the Kafka and Jython stage libs
docker build -t $IMAGE_NAME \
--build-arg SDC_VERSION=$SDC_VERSION \
--build-arg SDC_LIBS=\
streamsets-datacollector-apache-kafka_2_0-lib,\
streamsets-datacollector-jython_2_7-lib \
.

docker push $IMAGE_NAME