fetch_and_split.sh

#!/usr/bin/env bash

set -eu

QUERY_FILE=
COUNTRY=
BUCKET=
OUTPUT=cars_${COUNTRY}
RAW_OUTPUT=${OUTPUT}.csv
SPLIT_DIR=${OUTPUT}_split
SPLIT_PREFIX=${SPLIT_DIR}/${OUTPUT}-split-

# Query Snowflake with QUERY_FILE
# Assumes SNOWSQL_ACCOUNT, SNOWSQL_PWD, and SNOWSQL_USER are set
snowsql -d ${COUNTRY} -f ${QUERY_FILE} -o friendly=false -o header=false -o output_format=csv -o quiet=true -o output_file=${RAW_OUTPUT}

mkdir ${SPLIT_DIR}

# split by 10_000 lines, with numerical suffixes
split -l 10000 -a 5 -d --additional-suffix=.csv ${RAW_OUTPUT} ${SPLIT_PREFIX}

# Upload to S3
aws s3 cp ${SPLIT_DIR} s3://${BUCKET}/${COUNTRY} --recursive