Master node

masternode.go

// Run a thread in the background, which continually tries to become the master node.
func (m *Tracker) runMasterWatcher() {
	// Interval at which we update our status, and read the latest status
	pollInterval := 5 * time.Second

	// We will declare ourselves the master after observing our own nodeID in the table for this long
	delayToBecomeMaster := 30 * time.Second

	// Time that we will wait for another node to go dormant, before trying to become master
	gracePeriod := 30 * time.Second

	firstSelfMaster := time.Time{}

	run := true
	for iter := int64(0); run; iter++ {
		pause := pollInterval
		if iter == 0 {
			pause = 0
		}
		select {
		case <-m.systemClosed:
			run = false
			break
		case <-time.After(pause):
			isMaster := false
			nodeID := ""
			lockedAt := time.Time{}
			if err := m.db.QueryRow("select node_id, updated_at from master where domain = @p1", m.domain).Scan(&nodeID, &lockedAt); err != nil {
				if errors.Is(err, sql.ErrNoRows) {
					// No master node exists, so we try to become master
					if _, err := m.db.Exec("insert into master (domain, node_id, updated_at) values (@p1, @p2, getdate())", m.domain, m.nodeID); err != nil {
						m.log.Warnf("Error inserting master node: %v", err)
					} else {
						m.log.Infof("Inserted self as master")
					}
				} else {
					m.log.Warnf("Error querying master node: %v", err)
				}
			} else {
				makeSelfMaster := false
				if nodeID == m.nodeID {
					makeSelfMaster = true
					if firstSelfMaster.IsZero() {
						m.log.Infof("First observed self as master")
						firstSelfMaster = time.Now()
					} else if time.Since(firstSelfMaster) > delayToBecomeMaster {
						isMaster = true
					}
				} else {
					firstSelfMaster = time.Time{}
					if time.Since(lockedAt) > gracePeriod {
						// Since the current master seems to have gone away, try to become the new master
						makeSelfMaster = true
						m.log.Infof("Master node has gone away, trying to become master")
					}
				}
				if makeSelfMaster {
					if _, err := m.db.Exec("update master set node_id = @p1, updated_at = getdate() where domain = @p2", m.nodeID, m.domain); err != nil {
						// If we were unable to perform the update, then assume others can't see it either, so reset our clock.
						firstSelfMaster = time.Time{}
						m.log.Warnf("Error updating master node: %v", err)
					} else if !isMaster {
						m.log.Infof("Updated self as master")
					}
				}
			}
			if m.isMaster.Load() != isMaster {
				if isMaster {
					m.log.Infof("Became master node")
				} else {
					m.log.Infof("No longer master node")
				}
			}
			m.isMaster.Store(isMaster)
		}
	}
	m.Closed <- true
}