ecs_run_migration_task_and_wait.rb

gistfile1.txt

# frozen_string_literal: true

# This script is used during the deployment process. It runs 2 threads in parallel:
#
#   1) Calls run-task to execute a migration task within a cluster, and then waits for it to exit with 0 code.
#      Throws an exception in case of non-0 exit code;
#
#   2) Waits for all the service to become stable after the deployment.
#
#   Both have actually 10 minutes timeout, but if any of them fails before with an exception - the whole process dies immediatelly.

require 'aws-sdk-ecs'
require_relative '../ecs_task_execution_link_helper'

# rubocop:disable Lint/Env, Layout/LineLength
CLUSTER = ENV.fetch('CLUSTER')
SERVICES = ENV.fetch('WAIT_SERVICES').strip.split(/\s+/)
AWS_REGION = ENV.fetch('AWS_REGION')
APP_ENV = ENV.fetch('APP_ENV')

Thread.report_on_exception = true

def run_migrations
  client = Aws::ECS::Client.new

  task = client.run_task(cluster: CLUSTER, task_definition: ENV.fetch('TASK_DEF')).tasks&.first # rubocop:disable Lint/Env
  task_arn = task&.task_arn

  fail 'Failed to run migrations' if task_arn.nil?

  ecs_task_helper = EcsTaskExecutionLinkHelper.new(AWS_REGION, CLUSTER, task_arn, APP_ENV)

  puts "Amazon Console task: #{ecs_task_helper.ecs_task_url}"
  puts "OpenSearch logs (logs appear with delay): #{ecs_task_helper.opensearch_logs_url}"

  res = client.wait_until(
    :tasks_stopped,
    {tasks: [task_arn], cluster: CLUSTER},
    {max_attempts: 20, delay: 30}
  )

  # for some reason sometimes exit_code is nil
  exit_code = res.tasks.first.containers.first.exit_code.to_i

  fail "Migrations failed with exit_code: #{exit_code}" if exit_code.positive?
end

def wait_services
  Aws::ECS::Client.new.wait_until(
    :services_stable,
    {services: SERVICES, cluster: CLUSTER},
    max_attempts: 30,
    delay: 60
  )
rescue Aws::Waiters::Errors::WaiterFailed, Aws::Waiters::Errors::TooManyAttemptsError
  # Print detailed services info
  puts '== Service wait failed!'

  require 'pp'
  pp Aws::ECS::Client.new.describe_services(
    services: SERVICES, cluster: CLUSTER
  )&.to_h

  raise
end

[
  Thread.new { wait_services },
  Thread.new { run_migrations }
].map(&:join)