import_cloudwatch_logs_into_opensearch.sh

#!/usr/bin/env bash

export AWS_REGION=<aws-region>
export AWS_PROFILE=<aws-profile>
LOG_GROUP=<log-group-name>

PATTERN='$${instance-id} $${interface-id} $${flow-direction} $${srcaddr} $${pkt-srcaddr} $${dstaddr} $${pkt-dstaddr} $${srcport} $${dstport} $${protocol} $${tcp-flags} $${traffic-path} $${type} $${pkt-src-aws-service} $${pkt-dst-aws-service} $${packets} $${bytes} $${start} $${end} $${action} $${log-status} $${account-id}'
PATTERN=${PATTERN//[\\$\{\}]/}
IFS=' ' read -ra PATTERN_PARTS <<< "$PATTERN"

event_json() {
  ENTRY=$1
  MESSAGE=$(echo "$ENTRY" | jq -c -r '.message')
  IFS=' ' read -ra MESSAGE_PARTS <<< "$MESSAGE"
  TIMESTAMP=$(echo "$ENTRY" | jq -r '.timestamp')
  INGESTION_TIMESTAMP=$(echo "$ENTRY" | jq -r '.ingestionTime')

  JSON="{\"timestamp\": \"$TIMESTAMP\", \"ingestion-timestamp\": \"$INGESTION_TIMESTAMP\", \"log-group\": \"$LOG_GROUP\", \"log-stream-id\": \"$stream_id\","
  for i in $(seq ${#PATTERN_PARTS[@]}); do
    i=$((i-1))
    JSON="$JSON\" ${PATTERN_PARTS[$i]}\": \"${MESSAGE_PARTS[$i]}\","
  done
  JSON=${JSON%?} # remove last comma
  echo "$JSON}"
}

while IFS= read -r stream_id; do

  DATA=""
  while IFS= read -r event; do
    JSON=$(event_json "$event")
    DATA=$(printf "%s\n{ \"index\" : { \"_index\" : \"test01\" } }\n%s\n" "$DATA" "$JSON")

  done <<< "$(aws logs --region me-south-1 get-log-events --log-group-name ${LOG_GROUP} --log-stream-name ${stream_id} --limit 10 | jq -c '.events[]')" # TODO: use next token

  echo "${DATA:1}"
  curl http://localhost:4080/api/_bulk -i -u admin:Complexpass#123  --data "${DATA:1}"
  exit 0
done <<< "$(aws logs --region me-south-1 describe-log-streams --log-group-name ${LOG_GROUP} | jq -c -r '.logStreams[].logStreamName')"