At incident.io, we use Sentry to manage our exceptions, and PagerDuty to handle our on-call rota and escalate to an engineer. This code allows us to set the 'urgency' on an error, and apply rules in Sentry so we don't page on specific errors.

error_with_urgency.go

package errors

import "context"

// ErrorWithUrgency represents an error with a specified urgency. We currently
// support two urgencies:
//   page - this means we will escalate to the on-call engineer. This is the default behaviour.
//   sentry - this means we will send it to Sentry, but won't page someone.
type ErrorWithUrgency struct {
	cause   error
	urgency ErrorUrgencyType
}

type ErrorUrgencyType string

const (
	ErrorUrgencyPage   ErrorUrgencyType = "page"
	ErrorUrgencySentry ErrorUrgencyType = "sentry"
)

func (e ErrorWithUrgency) Error() string {
	return e.cause.Error()
}

func (e ErrorWithUrgency) Unwrap() error             { return e.cause }
func (e ErrorWithUrgency) Cause() error              { return e.cause }
func (e ErrorWithUrgency) Urgency() ErrorUrgencyType { return e.urgency }

// WithUrgency adds a chosen urgency to an error. As these are wrapped, the last
// caller will win: i.e. if we call WithUrgency(err, ErrorUrgencySentry) and
// then WithUrgency(err, ErrorUrgencyPage), then the on-call engineer will get
// paged.
func WithUrgency(err error, urgency ErrorUrgencyType) error {
	return ErrorWithUrgency{
		cause:   err,
		urgency: urgency,
	}
}

type ctxKey string

const (
	defaultUrgencyKey ctxKey = "errors.DefaultUrgency"
)

// GetUrgency returns the urgency attached to a specific error. This will
// default to ErrorUrgencyPage if nothing has been specified.
func GetUrgency(ctx context.Context, err error) ErrorUrgencyType {
	errWithUrgency := &ErrorWithUrgency{}
	if As(err, errWithUrgency) {
		return errWithUrgency.Urgency()
	}

	if urgencyCtx, ok := ctx.Value(defaultUrgencyKey).(urgencyContext); ok {
		if urgencyCtx.defaultUrgency != nil {
			return *urgencyCtx.defaultUrgency
		}
	}

	// default to page, if we're unsure.
	return ErrorUrgencyPage
}

type urgencyContext struct {
	defaultUrgency *ErrorUrgencyType
}

// WithDefaultUrgency allows the caller to override the default urgency (which is
// page) for a particular code path.
func WithDefaultUrgency(ctx context.Context, urgency ErrorUrgencyType) context.Context {
	return context.WithValue(ctx, defaultUrgencyKey, urgencyContext{defaultUrgency: &urgency})
}

// NewDefaultUrgency sets the default urgency key in the context, so that things
// further down the stack can use SetDefaultUrgency to update it
func NewDefaultUrgency(ctx context.Context) context.Context {
	return context.WithValue(ctx, defaultUrgencyKey, urgencyContext{})
}

func SetDefaultUrgency(ctx context.Context, urgency ErrorUrgencyType) {
	urgencyCtx, ok := ctx.Value(defaultUrgencyKey).(urgencyContext)
	if !ok {
		// TODO: we are very sad here
		return
	}
	urgencyCtx.defaultUrgency = &urgency
}

// WithoutUrgency returns the underlying error, with the ErrorWithUrgency wrapper
// removed
func WithoutUrgency(err error) error {
	for {
		switch err.(type) {
		case ErrorWithUrgency, *ErrorWithUrgency:
			err = Unwrap(err)
		default:
			return err
		}
	}
}

log.go

// A snippet from log.go where we push our errors to Sentry
  
if err != nil {
  // Explicitly set the urgency param, which defaults to page, for all errors.
  // This tells Sentry whether or not to page us.
  // we do this at the last moment so we can be as sure as possible that
  // we haven't accidentally overriden it.
  hub.Scope().SetTag("urgency", string(errors.GetUrgency(ctx, err)))
}

if err != nil {
  err = errors.WithoutUrgency(err)
  hub.CaptureException(err)
} else {
  hub.CaptureMessage(entry.Message)
}

organisation.go

// Set the default urgency to `Sentry` when an organisation is a Demo org

// ApplyToContext is like SetContext, only it updates the existing context
// rather than creating a new one. For this to work, the context has to have
// already been prepared with `log.NewMetadata` and `errors.NewDefaultUrgency`
func (org *Organisation) ApplyToContext(ctx context.Context) {
	if org == nil {
		return
	}

	log.SetMetadata(ctx, log.OrganisationID, org.ID)
	log.SetMetadata(ctx, log.OrganisationName, org.Name)

	if org.IsDemo {
		errors.SetDefaultUrgency(ctx, errors.ErrorUrgencySentry)
	}
}