load and transform csv to bigquery

entrypoint.sh

#!/bin/bash

echo "Download BigQuery Credentials"

aws s3 cp s3://$GCP_AUTH_BUCKET/auth.json .

echo "Upload CSV to GCS"

mkdir -p csv
rm tables

for raw in $(aws s3 ls s3://$S3_BUCKET/ | awk -F " " '{print $2}');
do
    table=${raw%/}
    if [[ $table != "" && $table != df* ]]
    then
        echo "Table: $table"
        csv=$(aws s3 ls s3://$S3_BUCKET/$table/ | awk -F " " '{print $4}' | grep ^ | sort -r | head -n1)

        echo $table >> tables

        echo "CSV: $csv"

        echo "Copy csv from S3"
        aws s3 cp s3://$S3_BUCKET/$table/$csv csv/$table.csv

        echo "Upload csv to GCP"
        gsutil cp csv/$table.csv gs://$GS_BUCKET/$table.csv
    fi
done

echo "Import CSV to BigQuery"

python app.py