Big Query Table Merge

big query merge.sql

# Google BigQuery - Merge data
bigquery_merge() {
    echo "Loading ${GC_BQ_WORK_TABLE} table into ${GC_BQ_MERGE}"
    bq --location=US load --source_format=NEWLINE_DELIMITED_JSON --encoding=UTF-8 \
        "${GC_BQ_MERGE}.${GC_BQ_WORK_TABLE}" "${TRANSFORM_STORE}/${filename}" \
        schemas/${GC_BQ_TABLE}.json.schema

    echo "Merging into ${GC_BQ_DATASET}.${GC_BQ_TABLE}"
    cat <<bigtable-merge  | bq --location=US query --use_legacy_sql=false --max_rows=0
MERGE \`${GC_BQ_DATASET}.${GC_BQ_TABLE}\` T
USING \`${GC_BQ_MERGE}.${GC_BQ_WORK_TABLE}\` S
ON T.user_id = S.user_id
WHEN MATCHED THEN DELETE
bigtable-merge

    bq --location=US query --use_legacy_sql=false --append_table=true --max_rows=0 \
        --destination_table="${GC_BQ_DATASET}.${GC_BQ_TABLE}" \
        "select * from \`${GC_BQ_MERGE}.${GC_BQ_WORK_TABLE}\`"

    echo "Cleaning..."
    bq rm -f -t "${GC_BQ_MERGE}.${GC_BQ_WORK_TABLE}"

    echo "******** DONE ${GC_BQ_DATASET}.${GC_BQ_TABLE} ********"
}