Script to fetch the latest failed CI pipeline and run all the failing steps locally (RSpec, Rubocop, etc.)

run_failed_gitlab_pipeline_specs.sh

#!/bin/bash
set -eo pipefail
CURRENT_DIR="$(realpath $(dirname "$0"))"
ROOT_DIR="$(realpath $CURRENT_DIR/..)"

# https://gitlab.com/*********/**********
PROJECT_ID="**********"

if ! [ -f "$ROOT_DIR/.gitlab-api-token" ]; then
  echo "Please create a file named .gitlab-api-token in $ROOT_DIR, containing your GitLab API token" >&2
  exit 1
fi
GITLAB_TOKEN=$(cat $ROOT_DIR/.gitlab-api-token)

PIPELINE_ID="$1"
if [ -z "$PIPELINE_ID" ]; then
  CURRENT_GIT_BRANCH="$(git rev-parse --abbrev-ref HEAD)"
  echo "Fetching latest failed pipeline ID for '$CURRENT_GIT_BRANCH'..."

  ESCAPED_BRANCH=$(ruby -r cgi -e "puts CGI.escape('$CURRENT_GIT_BRANCH')")
  LATEST_PIPELINE_RESPONSE=$(curl -s --header "PRIVATE-TOKEN: $GITLAB_TOKEN" \
    "https://gitlab.com/api/v4/projects/$PROJECT_ID/pipelines?ref=$ESCAPED_BRANCH&status=failed&per_page=1&page=1")
  PIPELINE_ID="$(echo $LATEST_PIPELINE_RESPONSE | jq -r '.[0].id')"
  if [ -z "$PIPELINE_ID" ]; then
    echo "No failed pipeline found for '$CURRENT_GIT_BRANCH'" >&2
    exit 1
  fi
  echo "Found latest failed pipeline ID: $PIPELINE_ID"
  echo
  echo $LATEST_PIPELINE_RESPONSE | jq -r '"  * " + .[0].web_url'
  echo
fi

# Check if pipeline id matches "$ROOT_DIR/tmp/gitlab_pipeline_results/current_pipeline_id" (if it exists)
# If not, update it and delete tmp/rspec_examples.txt
CURRENT_PIPELINE_ID=""
if [ -f "$ROOT_DIR/tmp/gitlab_pipeline_results/current_pipeline_id" ]; then
  CURRENT_PIPELINE_ID=$(cat $ROOT_DIR/tmp/gitlab_pipeline_results/current_pipeline_id)
fi
if [ "$CURRENT_PIPELINE_ID" != "$PIPELINE_ID" ]; then
  echo "New GitLab pipeline ID: ${PIPELINE_ID}. Deleting tmp/rspec_examples.txt and tmp/gitlab_pipeline_results/fixed.*"
  rm -f $ROOT_DIR/tmp/rspec_examples.txt
  rm -f $ROOT_DIR/tmp/gitlab_pipeline_results/fixed.*
  echo "$PIPELINE_ID" > "$ROOT_DIR/tmp/gitlab_pipeline_results/current_pipeline_id"
fi


PIPELINE_CACHE_DIR="$ROOT_DIR/tmp/gitlab_pipeline_results/$PIPELINE_ID"
mkdir -p "$PIPELINE_CACHE_DIR"

JOB_IDS_FILE="$PIPELINE_CACHE_DIR/job_ids.json"
if [ -f "$JOB_IDS_FILE" ]; then
  echo "Found jobs for GitLab CI Pipeline $PIPELINE_ID in $JOB_IDS_FILE"
  FAILED_PIPELINE_JOBS_RESPONSE="$(cat "$JOB_IDS_FILE")"
else
  echo "Listing jobs for GitLab CI Pipeline $PIPELINE_ID..."
  FAILED_PIPELINE_JOBS_RESPONSE="$(curl -s --header "PRIVATE-TOKEN: $GITLAB_TOKEN" \
    "https://gitlab.com/api/v4/projects/$PROJECT_ID/pipelines/$PIPELINE_ID/jobs?scope[]=failed")"
  echo "$FAILED_PIPELINE_JOBS_RESPONSE" > "$JOB_IDS_FILE"
fi

FAILED_RSPEC_JOB_COUNT=$(echo "$FAILED_PIPELINE_JOBS_RESPONSE" | \
  jq -r '. | map(select(.name | contains ("rspec"))) | map(.id) | length')
FAILED_RSPEC_JOB_IDS=$(echo "$FAILED_PIPELINE_JOBS_RESPONSE" | \
  jq -r '. | map(select(.name | contains ("rspec"))) | map(.id) | .[]')
FAILED_RUBYLINT_JOB_COUNT=$(echo "$FAILED_PIPELINE_JOBS_RESPONSE" | \
  jq -r '. | map(select(.name | contains ("ruby_lint"))) | map(.id) | length')
FAILED_RUBYLINT_JOB_IDS=$(echo "$FAILED_PIPELINE_JOBS_RESPONSE" | \
  jq -r '. | map(select(.name | contains ("ruby_lint"))) | map(.id) | .[]')
FAILED_RSWAG_JOB_COUNT=$(echo "$FAILED_PIPELINE_JOBS_RESPONSE" | \
  jq -r '. | map(select(.name | contains ("rswag"))) | map(.id) | length')
FAILED_RSWAG_JOB_IDS=$(echo "$FAILED_PIPELINE_JOBS_RESPONSE" | \
  jq -r '. | map(select(.name | contains ("rswag"))) | map(.id) | .[]')
FAILED_SECURITY_JOB_COUNT=$(echo "$FAILED_PIPELINE_JOBS_RESPONSE" | \
  jq -r '. | map(select(.name | contains ("security"))) | map(.id) | length')
FAILED_SECURITY_JOB_IDS=$(echo "$FAILED_PIPELINE_JOBS_RESPONSE" | \
  jq -r '. | map(select(.name | contains ("security"))) | map(.id) | .[]')

echo "Found $FAILED_RSPEC_JOB_COUNT failed RSpec job(s)"
echo
if [ "$FAILED_RSPEC_JOB_COUNT" != "0" ]; then
  echo "$FAILED_PIPELINE_JOBS_RESPONSE" | \
    jq -r '. | map(select(.name | contains ("rspec"))) | map("  * " + .web_url) | .[]'
  echo
fi

echo "Found $FAILED_RUBYLINT_JOB_COUNT failed Ruby Lint job(s)"
echo
if [ "$FAILED_RUBYLINT_JOB_COUNT" != "0" ]; then
  echo "$FAILED_PIPELINE_JOBS_RESPONSE" | \
    jq -r '. | map(select(.name | contains ("ruby_lint"))) | map("  * " + .web_url) | .[]'
  echo
fi

echo "Found $FAILED_RSWAG_JOB_COUNT failed RSwag job(s)"
echo
if [ "$FAILED_RSWAG_JOB_COUNT" != "0" ]; then
  echo "$FAILED_PIPELINE_JOBS_RESPONSE" | \
    jq -r '. | map(select(.name | contains ("rswag"))) | map("  * " + .web_url) | .[]'
  echo
fi

echo "Found $FAILED_SECURITY_JOB_COUNT failed security job(s)"
echo
if [ "$FAILED_SECURITY_JOB_COUNT" != "0" ]; then
  echo "$FAILED_PIPELINE_JOBS_RESPONSE" | \
    jq -r '. | map(select(.name | contains ("security"))) | map("  * " + .web_url) | .[]'
  echo
fi

if [ "$FAILED_RSPEC_JOB_COUNT" == "0" ] &&
  [ "$FAILED_RUBYLINT_JOB_COUNT" == "0" ] &&
  [ "$FAILED_RSWAG_JOB_IDS" == "0" ] &&
  [ "$FAILED_SECURITY_JOB_COUNT" == "0" ]; then
  exit
fi

for JOB_ID in $FAILED_RSPEC_JOB_IDS; do
  JOB_LOGS_FILE="$PIPELINE_CACHE_DIR/rspec_job_$JOB_ID.txt"
  if [ -f "$JOB_LOGS_FILE" ]; then
    echo "Found logs for failed RSpec job $JOB_ID in $JOB_LOGS_FILE"
  else
    echo "Fetching logs for failed RSpec job $JOB_ID"
    curl -s --location --header "PRIVATE-TOKEN: $GITLAB_TOKEN" \
      "https://gitlab.com/api/v4/projects/$PROJECT_ID/jobs/$JOB_ID/trace" > "$JOB_LOGS_FILE"
  fi
done

for JOB_ID in $FAILED_RUBYLINT_JOB_IDS; do
  HAS_FAILED_RUBOCOP_JOB=true
  JOB_LOGS_FILE="$PIPELINE_CACHE_DIR/ruby_lint_job_$JOB_ID.txt"
  if [ -f "$JOB_LOGS_FILE" ]; then
    echo "Found logs for failed Ruby Lint job $JOB_ID in $JOB_LOGS_FILE"
  else
    echo "Fetching logs for failed Ruby Lint job $JOB_ID"
    curl -s --location --header "PRIVATE-TOKEN: $GITLAB_TOKEN" \
      "https://gitlab.com/api/v4/projects/$PROJECT_ID/jobs/$JOB_ID/trace" > "$JOB_LOGS_FILE"
  fi
done

for JOB_ID in $FAILED_RSWAG_JOB_IDS; do
  JOB_LOGS_FILE="$PIPELINE_CACHE_DIR/rswag_job_$JOB_ID.txt"
  if [ -f "$JOB_LOGS_FILE" ]; then
    echo "Found logs for failed RSwag job $JOB_ID in $JOB_LOGS_FILE"
  else
    echo "Fetching logs for failed RSwag job $JOB_ID"
    curl -s --location --header "PRIVATE-TOKEN: $GITLAB_TOKEN" \
      "https://gitlab.com/api/v4/projects/$PROJECT_ID/jobs/$JOB_ID/trace" > "$JOB_LOGS_FILE"
  fi
done
echo

if [ "$HAS_FAILED_RUBOCOP_JOB" == true ]; then
  if [ -f "$ROOT_DIR/tmp/gitlab_pipeline_results/fixed.rubocop" ]; then
    echo "rubocop was already fixed."
  else
    FAILING_RUBOCOP_FILES="$(ruby -e "puts Dir.glob(\"$PIPELINE_CACHE_DIR/ruby_lint_job_*.txt\").map { |f|
      logs = File.read(f);
      start_regex = /Offenses:/;
      end_regex = /\d+ files? inspected, \d+ offenses? detected/;
      match_regex = /^(.+):\d+:\d+:/;
      next nil if !logs.match?(start_regex) || !logs.match?(end_regex);
      logs.
        split(start_regex)[1].
        split(end_regex)[0].
        scan(match_regex)
}.flatten.reject(&:nil?).map(&:strip).
select { |f| File.exist?(f) }.join(' ')")"

    if [ -z "$FAILING_RUBOCOP_FILES" ]; then
      echo "Could not find any Ruby files in the RuboCop output! Please check the logs manually." >&2
    else
      echo "Running 'rubocop -A' to fix Rubocop errors..."
      echo "=> bundle exec rubocop -A $FAILING_RUBOCOP_FILES"
      bundle exec rubocop -A $FAILING_RUBOCOP_FILES \
        && touch "$ROOT_DIR/tmp/gitlab_pipeline_results/fixed.rubocop"
    fi
  fi
fi

for RSWAG_LOG_FILE in $PIPELINE_CACHE_DIR/rswag_job_*.txt; do
  if [ -f $RSWAG_LOG_FILE ]; then
    if grep -q "Tasks: TOP => traceroute" $RSWAG_LOG_FILE; then
      if [ -f "$ROOT_DIR/tmp/gitlab_pipeline_results/fixed.traceroute" ]; then
        echo "rake traceroute was already fixed."
      else
        echo "rake traceroute failed! Running locally..."
        ./scripts/traceroute \
          && touch "$ROOT_DIR/tmp/gitlab_pipeline_results/fixed.traceroute" \
          && echo "Traceroute succeeded!" || echo "Traceroute failed!"
      fi
    fi

    UPDATE_SWAGGER_LINES_COUNT="$(grep "Please run ./scripts/update_swagger and commit the changes" $RSWAG_LOG_FILE | wc -l || true)"
    if [ "$UPDATE_SWAGGER_LINES_COUNT" == "2" ] && [ -z "$SKIP_SWAGGER_UPDATE" ]; then
      if [ -f "$ROOT_DIR/tmp/gitlab_pipeline_results/fixed.swagger" ]; then
        echo "swagger/v1/swagger.json was already updated."
      else
      echo "swagger/v1/swagger.json needs to be updated! Running ./scripts/update_swagger locally..."
      ./scripts/update_swagger && touch "$ROOT_DIR/tmp/gitlab_pipeline_results/fixed.swagger"
      fi
    fi
  fi
done

FAILING_SPECS=$(ruby -e "puts (Dir.glob(\"$PIPELINE_CACHE_DIR/rspec_job_*.txt\") + Dir.glob(\"$PIPELINE_CACHE_DIR/rswag_job_*.txt\")).map { |f|
  logs = File.read(f);
  start_regex = /Failed examples?:/;
  end_regex = /section_end:/;
  match_regex = /rspec '?([^'#]+)/;
  next nil if !logs.match?(start_regex) || !logs.match?(end_regex);
  logs.
    split(start_regex)[1].
    split(end_regex)[0].
    scan(match_regex)
}.flatten.reject(&:nil?).map(&:strip).join(' ')")

# echo "FAILING_SPECS: '$FAILING_SPECS'"
if [ -z "$FAILING_SPECS" ]; then
  echo "Could not find any failing specs in the RSpec output! Please check the logs manually." >&2
else
  # Persist results and use '--only-failures' so that we stop running tests once they are successful.
  echo "Running failed specs locally..."
  if [ -f $ROOT_DIR/tmp/rspec_examples.txt ]; then
    # Once we have an examples file from the initial run, then we only need the `--only-failures` flag.
    # (--only-failures is ignored when passing file names with line numbers.)
    # See: https://github.com/rspec/rspec-core/issues/2526
    COMMAND="bundle exec ./bin/rspec --only-failures"
  else
    COMMAND="bundle exec ./bin/rspec $FAILING_SPECS"
  fi
  echo "=> $COMMAND"
  $COMMAND
fi

if [ "$FAILED_SECURITY_JOB_COUNT" != "0" ]; then
  ./scripts/security
fi