dmabamboo · February 12, 2025 09:37 · aimestereo · Dec 7, 2020 · dmabamboo · Jan 28, 2021
diff --git a/delete-aws-glacier-vault-archives.sh b/delete-aws-glacier-vault-archives.sh
 #!/usr/bin/env bash
 #Checking pre-requisites (aws cli v2 and jq installed)
 if ! command -v jq &> /dev/null
 then
    echo "jq could not be found - check how to download and install it here https://stedolan.github.io/jq/download/"
    exit
 fi

 if ! command -v aws --version &> /dev/null
 then
    echo "AWS CLI could not be found - check how to download and install it here https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html"
    echo "How to configure the AWS CLI to use secrets for your Glacier IAM - https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html"
    echo "You need to configure it with your appropriate secrets for an IAM that has full access over your Glacier resources (AmazonGlacierFullAccess)."
    echo "In JSON: "
    echo "{"
    echo "  \"Version\": \"2012-10-17\","
    echo "  \"Statement\": ["
    echo "    {"
    echo "      \"Action\": \"glacier:*\","
    echo "      \"Effect\": \"Allow\","
    echo "      \"Resource\": \"*\""
    echo "    }"
    echo "  ]"
    echo "}"
    exit
 fi

 account_id=$1
 region=$2
 vault_name=$3

 if [[ -z ${account_id} ]] || [[ -z ${region} ]] || [[ -z ${vault_name} ]]; then
  echo "#################################################################"
  echo "Attention!!! Parameters required are missing."
  echo "Account: ${account_id}"
  echo "Region: ${region}"
  echo "Vault: ${vault_name}"
  echo "#################################################################"
  echo "run this command like: sh ./delete-aws-glacier-vault-archives.sh AWS_ACCOUNT_ID AWS_REGION AWS_GLACIER_VAULT_NAME"
  exit 1
 fi

 echo "Initiating delete process for the vault."
 echo "    Account:${account_id}"
 echo "    Region:${region}"
 echo "    Vault:${vault_name}"

 echo "Starting Step 1/4 - Glacier Inventory Retrieval Job - it's Async and can take hours or days to complete"

 # Step 1 - inventory retrieval job for the given vault
 job_initiation_file=./glacier-inventory-retrieval-job-file-${account_id}-${region}-${vault_name}.json
 if test -f "${job_initiation_file}"; then
  echo "There is already a file for this job. Using it now. If you don't want to use it you need to delete the file ${job_initiation_file}."
 else
  echo "No previous job file found for this vault."
  echo "Starting a new Job."
  aws glacier initiate-job --job-parameters '{"Type": "inventory-retrieval"}' --account-id ${account_id} --region ${region} --vault-name ${vault_name} &> ${job_initiation_file}
  echo "Job request made."
 fi

 echo "Checking if the job initiation file is in good shape."
 job_id="Undefined"
 if cat ${job_initiation_file} | jq ".jobId" > /dev/null; then
  job_id=$(cat ${job_initiation_file} | jq -r ".jobId")
  echo "File is OK, jobId=${job_id}"
 else
  echo "Failed to obtain Job Id from file, file may be corrupted or your retrieve-inventory call failed - check parameters passed to this script or aws cli config and connectivity."
 fi

 echo "Starting Step 2/4 - Checking state of the Job to see if it's completed and can have its inventory retrieved for deletion."
 job_completed_flag=false
 job_status_file=./glacier-describe-job-file-${account_id}-${region}-${vault_name}-${job_id}.json
 while [ "${job_completed_flag}" = false ]
 do
  aws glacier describe-job --account-id ${account_id} --region ${region} --vault-name ${vault_name} --job-id ${job_id} &> ${job_status_file}

  if cat ${job_status_file} | jq ".Completed" > /dev/null; then
    job_completed_flag=$(cat ${job_status_file} | jq -r ".Completed")
    echo "File is OK. Job completed? ${job_completed_flag}"
    if ${job_completed_flag} = true; then
      break
    fi
  else
    echo "$(date) Failed to check status from describe job."
  fi
  #sleeps for 1/2 hour - 1800 seconds before trying to fetch status again - Glacier is slow...
  echo "$(date) Will try again in 1/2 hour... "
  sleep 1800
 done

 echo "Starting Step 3/4 - Obtaining output from retrieval job - finally getting archive ids to delete"
 inventory_output_file=./glacier-inventory-output-file-${account_id}-${region}-${vault_name}-${job_id}.json
 aws glacier get-job-output --account-id ${account_id} --region ${region} --vault-name ${vault_name} --job-id ${job_id} ${inventory_output_file}
 echo "Output file: ${inventory_output_file} created for vault ${vault_name} and job ${job_id}"

 inventory_id_file=./glacier-inventory-output-file-${account_id}-${region}-${vault_name}-${job_id}.txt
 echo "Creating archive list from output file at ${inventory_id_file}"
 if [[ ! -f ${inventory_id_file} ]]; then
  cat ${inventory_output_file} | jq -r --stream ". | { (.[0][2]): .[1]} | select(.ArchiveId) | .ArchiveId" > ${inventory_id_file} 2> /dev/null
 fi
 total=$(wc -l ${inventory_id_file} | awk '{print $1}')
 echo "Total archives to delete: ${total} in vault ${vault_name}"

 echo "Starting Step 4/4 - Delete process starting now $(date)"

 case "$(uname -s)" in
    Linux*)     numCPU="$(nproc)";;
    Darwin*)    numCPU="$(sysctl -n hw.logicalcpu)";;
    *)          numCPU=1
 esac

 num=0
 while read -r archive_id; do
  num=$((num+1))

  aws glacier delete-archive --account-id ${account_id} --region ${region} --vault-name ${vault_name} --archive-id=${archive_id} &
  [ $( jobs | wc -l ) -ge $numCPU ] && wait

  echo "Archive ${num}/${total} deleted at $(date) - id: ${archive_id}"
 done < "${inventory_id_file}"

 wait
 echo "Finished at $(date)"
 echo "Deleted all archives listed in ${inventory_id_file}"
diff --git a/deleting_all_archives_from_aws_glacier_vault.md b/deleting_all_archives_from_aws_glacier_vault.md