Skip to content

Instantly share code, notes, and snippets.

@quiiver
Created October 18, 2022 20:52
Show Gist options
  • Select an option

  • Save quiiver/281007aba37efbcbaff879ba6d8cb798 to your computer and use it in GitHub Desktop.

Select an option

Save quiiver/281007aba37efbcbaff879ba6d8cb798 to your computer and use it in GitHub Desktop.
diff --git c/Dockerfile i/Dockerfile
index 36cff7bc7..39378fd12 100644
--- c/Dockerfile
+++ i/Dockerfile
@@ -34,5 +34,6 @@ COPY --from=java-deps /app/target/*.jar /app/target/
COPY --from=python-deps /usr/local /usr/local
COPY .bigqueryrc /root/
COPY . .
+RUN pip-sync --pip-args=--no-deps requirements.txt
RUN pip install .
ENTRYPOINT ["/app/script/entrypoint"]
diff --git c/bqetl-docker i/bqetl-docker
new file mode 100755
index 000000000..238aa9e1f
--- /dev/null
+++ i/bqetl-docker
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -e
+
+# Absolute path this script is in
+SCRIPT_PATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+CMD=$1
+
+if [ "$PWD" != "$SCRIPT_PATH" ]; then
+ echo "Please run this command from the root of your bigquery-etl checkout ($SCRIPT_PATH)"
+ exit 1
+fi
+
+if ! command -v docker &> /dev/null; then
+ echo "Please install docker"
+ exit 1
+fi
+
+if [ "$CMD" == "bootstrap" ]; then
+ docker build . -t bqetl
+ exit
+fi
+
+ADC_PATH="$HOME/.config/gcloud/application_default_credentials.json"
+
+docker run\
+ --mount type=bind,source=$ADC_PATH,target=/gcp/creds.json\
+ --mount type=bind,source=$(pwd)/dags,target=/app/dags\
+ --mount type=bind,source=$(pwd)/sql,target=/app/sql\
+ --mount type=bind,source=$(pwd)/tests,target=/app/tests\
+ --mount type=bind,source=$(pwd)/dags.yaml,target=/app/dags.yaml\
+ --mount type=bind,source=$(pwd)/script/entrypoint,target=/app/script/entrypoint\
+ --env GOOGLE_APPLICATION_CREDENTIALS=/gcp/creds.json\
+ --env GOOGLE_CLOUD_PROJECT=$(gcloud config get project)\
+ bqetl script/bqetl "$@"
+exit $?
diff --git c/script/entrypoint i/script/entrypoint
index e37617b2e..c3e040cdc 100755
--- c/script/entrypoint
+++ i/script/entrypoint
@@ -11,10 +11,13 @@ if [ -n "$GCLOUD_SERVICE_KEY" ]; then
echo "$GCLOUD_SERVICE_KEY" > "$GOOGLE_APPLICATION_CREDENTIALS"
fi
+# Activate service account if the application credentials are for a service account.
if [ -n "$GOOGLE_APPLICATION_CREDENTIALS" ] && which gcloud > /dev/null; then
+ if grep -Fq "service_account" $GOOGLE_APPLICATION_CREDENTIALS; then
gcloud --quiet auth activate-service-account --key-file "$GOOGLE_APPLICATION_CREDENTIALS"
python -W ignore -c 'import google.auth; print("project = ", google.auth.default()[1])' >> ~/.config/gcloud/configurations/config_default
sed "1i--project_id=$(gcloud config get-value project)" -i ~/.bigqueryrc
+ fi
fi
if [ "$#" = 0 ]; then
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment