Created
March 8, 2019 10:43
-
-
Save d-kuro/e86a51967b34db1d91ebd5d4f10ff5ce to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// from: https://github.com/kubernetes/kubernetes/blob/87f9429087d4e31201412548517d36e83abebc8d/pkg/controller/cronjob/utils.go#L89-L149 | |
// getRecentUnmetScheduleTimes gets a slice of times (from oldest to latest) that have passed when a Job should have started but did not. | |
// | |
// If there are too many (>100) unstarted times, just give up and return an empty slice. | |
// If there were missed times prior to the last known start time, then those are not returned. | |
func getRecentUnmetScheduleTimes(sj batchv1beta1.CronJob, now time.Time) ([]time.Time, error) { | |
starts := []time.Time{} | |
sched, err := cron.ParseStandard(sj.Spec.Schedule) | |
if err != nil { | |
return starts, fmt.Errorf("Unparseable schedule: %s : %s", sj.Spec.Schedule, err) | |
} | |
var earliestTime time.Time | |
if sj.Status.LastScheduleTime != nil { | |
earliestTime = sj.Status.LastScheduleTime.Time | |
} else { | |
// If none found, then this is either a recently created scheduledJob, | |
// or the active/completed info was somehow lost (contract for status | |
// in kubernetes says it may need to be recreated), or that we have | |
// started a job, but have not noticed it yet (distributed systems can | |
// have arbitrary delays). In any case, use the creation time of the | |
// CronJob as last known start time. | |
earliestTime = sj.ObjectMeta.CreationTimestamp.Time | |
} | |
if sj.Spec.StartingDeadlineSeconds != nil { | |
// Controller is not going to schedule anything below this point | |
schedulingDeadline := now.Add(-time.Second * time.Duration(*sj.Spec.StartingDeadlineSeconds)) | |
if schedulingDeadline.After(earliestTime) { | |
earliestTime = schedulingDeadline | |
} | |
} | |
if earliestTime.After(now) { | |
return []time.Time{}, nil | |
} | |
for t := sched.Next(earliestTime); !t.After(now); t = sched.Next(t) { | |
starts = append(starts, t) | |
// An object might miss several starts. For example, if | |
// controller gets wedged on friday at 5:01pm when everyone has | |
// gone home, and someone comes in on tuesday AM and discovers | |
// the problem and restarts the controller, then all the hourly | |
// jobs, more than 80 of them for one hourly scheduledJob, should | |
// all start running with no further intervention (if the scheduledJob | |
// allows concurrency and late starts). | |
// | |
// However, if there is a bug somewhere, or incorrect clock | |
// on controller's server or apiservers (for setting creationTimestamp) | |
// then there could be so many missed start times (it could be off | |
// by decades or more), that it would eat up all the CPU and memory | |
// of this controller. In that case, we want to not try to list | |
// all the missed start times. | |
// | |
// I've somewhat arbitrarily picked 100, as more than 80, | |
// but less than "lots". | |
if len(starts) > 100 { | |
// We can't get the most recent times so just return an empty slice | |
return []time.Time{}, fmt.Errorf("Too many missed start time (> 100). Set or decrease .spec.startingDeadlineSeconds or check clock skew.") | |
} | |
} | |
return starts, nil | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment