Checking All possible alloc events details when termination already occurs

nomad-failed-alloc-checker.go

package main

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/hashicorp/nomad/api"
	"github.com/sirupsen/logrus"
)

type allocEventStruct struct {
	JobName string              `json:"job_name"`
	AllocID string              `json:"alloc_id"`
	Events  []map[string]string `json:"events"`
}

func main() {
	allocEvents := []allocEventStruct{}
	var allocsCount int
	nomadCfg := api.DefaultConfig()
	nomadCfg.Address = "http://localhost:4646"
	nc, err := api.NewClient(nomadCfg)
	if err != nil {
		logrus.WithField("error_message", err).Fatal("can't connect to nomad")
	}

	leaderStatus, err := nc.Status().Leader()
	if err != nil {
		logrus.WithField("error_message", err).Fatal("can't query server status")
	}
	fmt.Println("Nomad Leader : ", leaderStatus)

	listJob, _, err := nc.Jobs().List(&api.QueryOptions{
		Prefix: "<Job_Prefix>",
		Region: "global",
		Params: map[string]string{
			"status": "dead",
		},
	})

	if err != nil {
		logrus.WithField("error_message", err).Fatal("query job list error")
	}

	fmt.Printf("Processing, Please Wait.")
	for _, job := range listJob {
		jobSummary := job.JobSummary.Summary
		for _, summaryStatus := range jobSummary {
			if summaryStatus.Failed != 0 {
				jobAllocs, _, err := nc.Jobs().Allocations(job.ID, true, &api.QueryOptions{})
				if err != nil {
					logrus.WithField("error_message", err).Fatal("query allocs in %s error", job.Name)
				}
				for _, alloc := range jobAllocs {
					allocInfo, _, err := nc.Allocations().Info(alloc.ID, &api.QueryOptions{})
					if err != nil {
						logrus.WithField("error_message", err).Fatal("query allocs info %s error", alloc.ID)
					}
					for _, taskState := range allocInfo.TaskStates {
						evenLength := len(taskState.Events) - 1
						var evenIndexMarker int
						for eventIndex, event := range taskState.Events {
							if event.Type == "Terminated" {
								evenIndexMarker = eventIndex
							}
						}
						if evenLength > evenIndexMarker {
							var eventDetail []map[string]string
							for _, event := range taskState.Events {
								eventDetail = append(eventDetail, map[string]string{
									"type":    event.Type,
									"time":    time.Unix(0, event.Time).Format("2006-01-02T15:04:05Z07:00"),
									"details": event.DisplayMessage,
								})
							}
							allocEvents = append(allocEvents, allocEventStruct{
								JobName: job.Name,
								AllocID: alloc.ID,
								Events:  eventDetail,
							})
							allocsCount++
							fmt.Printf(".")
						}
					}
				}
			}
		}
	}
	prettyAllocEvents, err := json.MarshalIndent(allocEvents, "", "  ")
	if err != nil {
		logrus.WithField("error_message", err).Fatal("error prettify output")
	}
	fmt.Printf("\nList of Alloc with strange terminated events : \n%s\n", string(prettyAllocEvents))
	fmt.Printf("Captured Allocs Count: %v\n", allocsCount)
}