push.sh

#!/bin/bash

set -e
set -o pipefail

BASE_URL=$1

TABLES_URL="${BASE_URL}/api/tables"

DESTINATIONS_URL="${BASE_URL}/api/destinations"

DESTINATION_BASE_URL="${BASE_URL}/api/destination"

TABLE_BASE_URL="${BASE_URL}/api/table"

echo "Getting tables"

# Unforunately tables, despite giving the frontend definition, does not give columns so we'll need to get the tables individually.
TABLE_IDS=$(for i in {1..15}; do curl --insecure --fail "${TABLES_URL}" && break || sleep 5; done | jq -r 'keys | .[]') 
echo "$TABLE_IDS"

TABLE_INFO=$(while IFS= read -r ID; do curl --insecure --fail "${TABLE_BASE_URL}/${ID}" | jq -rc "{\"$ID\": .}"; done <<< "$TABLE_IDS" | jq -nc 'reduce inputs as $in (null; . + $in)')
echo "$TABLE_INFO"

echo "Getting destinations"

DESTINATION_IDS=$(curl --insecure --fail "${DESTINATIONS_URL}" | jq -r '.destinations | keys | .[]')

echo "Pushing"

while IFS= read -r ID; do
  DESTINATION_TYPES=$(curl --insecure --fail "${DESTINATION_BASE_URL}/${ID}" | jq -rc '.coercions | to_entries | map(.value = (.value[0] | del(.label))) | from_entries')
  echo $DESTINATION_TYPES
  PUSH_TABLE_INFO=$(printf "${TABLE_INFO}" | jq -rc --argjson types "${DESTINATION_TYPES}" "to_entries | map(select(select(.value.name | (contains(\"[Archived]\") or contains(\"${ID}\")) | not)) | (.value = { path: (\"/\" + (.value.name | gsub(\"[^\\\w]+\";\"_\"))), columns: .value.columns | map({ name: .column, type: \$types[.type] })})) | from_entries")
  PUSH_BODY="{ \"tables\": ${PUSH_TABLE_INFO} }"
  echo ""
  echo "Starting push to destination ${ID}"
  echo "${PUSH_BODY}"
  curl --insecure -v --fail -H 'Content-Type: application/vnd.slamdata.start-pushes; version="1"' -X POST "${DESTINATION_BASE_URL}/${ID}/pushes" --data-binary "${PUSH_BODY}"
done <<< "$DESTINATION_IDS"

echo ""
echo "Success"