Enable basic normalization on all connections

enable_basic_normalization.sh

#!/bin/bash
# This script creates a basic normalization operation and attaches it
# to all connections in the local Airbyte instance
#
#
# Background / Context
#
# - Currently, a new Airbyte instance does not have a normalization
#   operation registered
# - The first connection enabling normalization from the UI triggers the
#   creation of the related operation on the backend
# - The Octavia CLI team took a shortcut of disabling operations for the
#   alpha release, but opened an issue to make sure they implement this:
#   see https://github.com/airbytehq/airbyte/issues/10973
# - This script can be safely removed from the deployment pipeline once
#   there is official support for configuring operations in Octavia and
#   we implement the official method
#
#
# Usage
#
# - Ensure that the Airbyte instance is running on localhost:8000 via
#   `docker-compose up -d`
# - Apply the octavia configuration to the instance `docker-compose run
#   octavia apply --force`
# - Observe in the UI that all of the connections default to an
#   "inactive" state per the configuration YAMLs (`status: inactive``) see
#   http://localhost:8000/workspaces/<workspaceId>/connections
# - Observe in the UI that all of the connections default to Raw data
#   (JSON) with no normalization
#   http://localhost:8000/workspaces/<workspaceId>/connections/<connectionId>/transformation
#  - Run this script and observe the logging/outputs
#  - Observe in the UI that basic normalization is now enabled on all
#    connections
#  - Observe in the UI that all connections are now in an "active" state
​
workspaceId=$(curl --silent -X POST http://localhost:8000/api/v1/workspaces/list | jq -r .workspaces[0].workspaceId)
​
connectionIds=$(curl --silent -X POST http://localhost:8000/api/v1/connections/list \
  -H "Content-Type: application/json" \
  -d $(jq -nc --arg workspaceId $workspaceId '$ARGS.named') \
| jq -r .connections[].connectionId)
​
# Creates a basic normalization operation.
operationId=$(curl --silent -X POST http://localhost:8000/api/v1/operations/create \
  -H "Content-Type: application/json" \
  -d $(jq -nc \
    --arg workspaceId $workspaceId \
    --arg name Normalization \
    --arg operatorType normalization \
    --arg option basic \
    '{$workspaceId, $name, operatorConfiguration: {$operatorType, normalization: {$option}}}'
) | jq -r .operationId)
​
​
for connectionId in $connectionIds
do
  # Check to see whether there are operations already configured on the connection
  if curl --silent -X POST http://localhost:8000/api/v1/operations/list \
  -H "Content-Type: application/json" \
  -d $(jq -nc --arg connectionId $connectionId '$ARGS.named') \
  | jq -e '.operations | length == 1'
  then
    printf "🤗 - Basic normalization operation is already configured for connectionId $connectionId \n"
  else
    printf "\n😳 - Basic normalization operation is not configured for connectionId $connectionId"
    connectionData=$(curl --silent -X POST http://localhost:8000/api/v1/connections/get \
      -H "Content-Type: application/json" \
    -d $(jq -ncr --arg connectionId $connectionId '$ARGS.named'))
    printf "\n✍️  - Adding basic normalization . . . "
​
    curl --silent -X POST http://localhost:8000/api/v1/connections/update \
    -H "Content-Type: application/json" \
    -d "$(echo $connectionData \
      | jq -c \
      --arg operationId $operationId \
      --arg status active \
      '.operationIds = [ $operationId ] | .status = $status' \
      | jq -crM \
      --argjson keys '["connectionId", "namespaceDefinition", "namespaceFormat", "operationIds", "status", "syncCatalog", "prefix", "schedule"]' \
    'with_entries( select( .key as $k | $keys | index($k) ) )')"
​
    printf "\n😻 - . . . basic normalization added to connectionId $connectionId"
  fi
done