Last active
November 22, 2024 13:04
-
-
Save astromechza/b5cedbce68dd0baa6252728ff1de9505 to your computer and use it in GitHub Desktop.
A score file that works with score-compose and score-k8s to demonstrate a duckdb app working with an s3 bucket endpoint.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
apiVersion: score.dev/v1b1 | |
metadata: | |
name: duckdb-cli-demo | |
containers: | |
main: | |
image: ubuntu:latest | |
command: | |
- /bin/bash | |
- -c | |
- /mnt/setup.sh && sleep 3600 | |
files: | |
- target: /mnt/setup.sh | |
mode: "0755" | |
content: | | |
#!/bin/bash | |
set -eu | |
rm -rfv duckdb_cli-linux-aarch64.zip duckdb /root/.duckdb | |
apt update && apt install -y wget unzip | |
wget https://github.com/duckdb/duckdb/releases/download/v1.1.3/duckdb_cli-linux-aarch64.zip | |
unzip duckdb_cli-linux-aarch64.zip | |
export S3_ENDPOINT_HOST=$( echo "$${S3_ENDPOINT}" | sed -E -e 's,https?://,,' ) | |
export S3_ENDPOINT_SSL=$( if [[ "$${S3_ENDPOINT}" == "https://" ]]; then echo true; else echo false; fi ) | |
./duckdb -c "CREATE PERSISTENT SECRET secret (TYPE S3, KEY_ID '$${S3_ACCESS_KEY_ID}', SECRET '$${S3_SECRET_KEY}', REGION '$${S3_REGION}', ENDPOINT '$${S3_ENDPOINT_HOST}', USE_SSL $${S3_ENDPOINT_SSL}, URL_STYLE 'path', SCOPE 's3://$${S3_BUCKET}');" | |
./duckdb -c "SELECT * FROM glob('s3://$${S3_BUCKET}/*.parquet');" || true | |
echo "CREATE TABLE foo (name TEXT PRIMARY KEY);" >> /root/.duckdb_history | |
echo "INSERT INTO foo (name) VALUES ('ben');" >> /root/.duckdb_history | |
echo "COPY foo TO 's3://$${S3_BUCKET}/foo.parquet';" >> /root/.duckdb_history | |
echo "SELECT * FROM glob('s3://$${S3_BUCKET}/*.parquet');" >> /root/.duckdb_history | |
echo "SELECT * FROM read_parquet('s3://$${S3_BUCKET}/foo.parquet');" >> /root/.duckdb_history | |
variables: | |
S3_ENDPOINT: ${resources.bucket.endpoint} | |
S3_ACCESS_KEY_ID: ${resources.bucket.access_key_id} | |
S3_SECRET_KEY: ${resources.bucket.secret_key} | |
S3_REGION: ${resources.bucket.region} | |
S3_BUCKET: ${resources.bucket.bucket} | |
resources: | |
bucket: | |
type: s3 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment