Break the stateless rule

main.go

// This is a Go-like pseudocode.
// It is an attempt to breifly describe how a stateful service
// for a game server could potentially work. 

// Objective: create a cluster of server instances ("nodes") where each node
//            is stateful and dynamically aware of other nodes and can redirect
//            an incoming request to the right node.

// 1. Every node embeds a cluster agent.
//    The agent starts along with the server and constantly communicates with
//    other agents in the cluster using a gossip protocol (https://github.com/hashicorp/serf).
//    This allowes each node to have an up-to-date (but eventually consistent)
//    list of all other nodes at runtime.
func (srv *Server) startClusterAgent() error {
	cfg := srv.cfg
	logger := log.New(os.Stdout, fmt.Sprintf("cluster: agent: %s", cfg.Server.NodeId), log.LstdFlags)

	agent, err := cluster.NewAgent(cfg.Cluster.AgentHost, cfg.Cluster.AgentPort, cfg.Server.NodeId, logger)
	if err != nil {
		return errors.Wrap(err, "create new agent")
	}

	if err := agent.Start(cfg.Cluster.KnownMembers...); err != nil {
		return errors.Wrap(err, "start cluster agent")
	}

	srv.clusterAgent = agent

	return nil
}

// 2. When an incoming request arrives, it is initially load-balanced to some node in the cluster
//    by some external load balancer. The node then decides wether it should process the request itself
//    or forward it to another node. The decision is based on this info:
//    - List of all currently available nodes
//    - ID of the game
//    - Constant hashing "rendezvous" algorithm which maps ID to a partucular node.
func (router *GameRouter) routeCommand(ctx actor.Context, env *mscore.Envelope) *mscore.Error {
  ...
  
  // Determine the target node for this command
	destinationNode := router.nodesHash.GetNodeName(fmt.Sprintf("%d", env.EntityId))
	if destinationNode == "" {
		return &mscore.Error{Code: "cluster.empty", Origin: "gsrv.game.router"}
	}

	// If it's local, forward to the local game actor
	if router.nodesHash.IsLocal(destinationNode) {
		router.ForwardToGame(ctx, env)
		return nil
	}

	// At this moment it's clear that the command is not for local processing.
	// So, forward the command to the topic of the destination node.
	topic := msgsrv.GameNodeCommandsTopic(destinationNode)
	if err := router.publisher.Publish(topic, env); err != nil {
		err := errors.Wrapf(err, "publish to: %s", topic)
		router.logger.Error("Failed to forward a game command", zap.String("cmd", env.Name), zap.Error(err))
	}

	return nil
}

// See the implementation of the rendezvous algorithm in the next file.

rendezvous.go

package rendezvous

import (
	"crypto/md5"
	"sync"
)

type Hash struct {
	nodes      []string
	nodesMutex sync.RWMutex
}

func New(nodes []string) *Hash {
	hash := &Hash{}
	hash.nodes = make([]string, len(nodes))
	copy(hash.nodes, nodes)
	return hash
}

// Select one of the nodes based on an arbitrary string key.
func (h *Hash) Get(key string) string {
	var maxScore, curScore uint32
	var maxNode string

	h.nodesMutex.RLock()
	for _, node := range h.nodes {
		curScore = calculateNodeScore(node, key)
		if curScore > maxScore {
			maxScore = curScore
			maxNode = node
		}
	}
	h.nodesMutex.RUnlock()

	return maxNode
}

// Try add a new node to the list
func (h *Hash) Add(nodeToAdd string) bool {
	h.nodesMutex.Lock()
	defer h.nodesMutex.Unlock()

	// Do not add the node if it's a duplicate
	for _, node := range h.nodes {
		if node == nodeToAdd {
			return false
		}
	}

	h.nodes = append(h.nodes, nodeToAdd)

	return true
}

// DeleteNode tries to delete a given node from the list.
// If the node is found and then deleted, returns true.
func (h *Hash) Delete(nodeToDelete string) bool {
	h.nodesMutex.Lock()
	defer h.nodesMutex.Unlock()

	maxIdx := len(h.nodes) - 1
	for i, node := range h.nodes {
		if node == nodeToDelete {
			if i == maxIdx {
				// Delete the last node
				h.nodes = h.nodes[:i]
			} else {
				// Delete the node in the middle
				h.nodes = append(h.nodes[:i], h.nodes[i+1:]...)
			}
			return true
		}
	}

	return false
}

func calculateNodeScore(node, key string) uint32 {
	var sum uint32

	hashSum := md5.Sum([]byte(node + key))
	for _, b := range hashSum {
		sum += uint32(b)
	}

	return sum
}