build_pyarrow_layer.sh

#!/bin/bash

set -euo pipefail

LAYER_NAME="pyarrow-layer"
PYTHON_VERSION="3.11"
TARGET_DIR="$(pwd)/layer_build"
OUTPUT_ZIP="pyarrow-layer.zip"
IMAGE_TAG="${LAYER_NAME}-builder"

echo "🧹 Cleaning previous build artifacts..."
rm -rf "$TARGET_DIR" "$OUTPUT_ZIP" Dockerfile
mkdir -p "$TARGET_DIR"

# Safety: Remove any cached ECR image reference
echo "🧼 Removing any broken ECR-based images..."
podman image rm public.ecr.aws/lambda/python:3.11 2>/dev/null || true

# Warn about Podman DNS issues
if [[ "$(uname)" == "Darwin" ]] && command -v podman &>/dev/null; then
  echo "⚠️  macOS + Podman detected — if you hit DNS issues, run:"
  echo "     podman machine stop && podman machine start"
  echo "  and ensure /etc/resolv.conf in the Podman VM has nameserver 8.8.8.8"
fi

echo "📄 Writing Dockerfile..."
cat > Dockerfile <<EOF
FROM amazonlinux:2023

RUN dnf install -y gcc-c++ cmake make python3-devel python3-pip unzip zip ca-certificates && \
    update-ca-trust && \
    python3 -m pip install --upgrade pip setuptools wheel

ENV PYTHON_VERSION=${PYTHON_VERSION}
WORKDIR /opt

# Install pyarrow into Lambda layer structure
RUN python3 -m pip install pyarrow -t python/lib/python\${PYTHON_VERSION}/site-packages

# Strip shared libs
RUN find python/lib/python\${PYTHON_VERSION}/site-packages/pyarrow -name "*.so" -exec strip --strip-unneeded {} \; || true

# Cleanup
RUN rm -rf python/lib/python\${PYTHON_VERSION}/site-packages/pyarrow/tests \\
           python/lib/python\${PYTHON_VERSION}/site-packages/pyarrow/include \\
           python/lib/python\${PYTHON_VERSION}/site-packages/pyarrow/*.pyd \\
           python/lib/python\${PYTHON_VERSION}/site-packages/pyarrow/*.cmake \\
           python/lib/python\${PYTHON_VERSION}/site-packages/pyarrow/__pycache__

# Create layer ZIP
RUN cd python && zip -r9 /${OUTPUT_ZIP} .
EOF

echo "🐳 Building image..."
podman build -t "$IMAGE_TAG" .

echo "📦 Extracting layer ZIP from image..."
podman run --rm -v "$TARGET_DIR":/out:z "$IMAGE_TAG" cp "/${OUTPUT_ZIP}" /out/

echo "📁 Moving output..."
mv "$TARGET_DIR/${OUTPUT_ZIP}" .

echo "✅ Done! Lambda layer created: ${OUTPUT_ZIP}"
echo ""
echo "👉 To publish the layer, run:"
echo "aws lambda publish-layer-version --layer-name pyarrow --zip-file fileb://${OUTPUT_ZIP} --compatible-runtimes python3.11"