Last active
March 18, 2021 06:47
-
-
Save ismaelhamed/4341152fd27b8617c2667deba57c82c9 to your computer and use it in GitHub Desktop.
An extension to monitor whether we have been quarantined
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
public class QuarantinedNodesExtension : IExtension | |
{ | |
private readonly ExtendedActorSystem system; | |
private readonly IActorRef listener; | |
public QuarantinedNodesExtension(ExtendedActorSystem system) | |
{ | |
this.system = system; | |
listener = system.SystemActorOf(Props.Create<QuarantinedNodesListener>().WithDeploy(Deploy.Local), "QuarantinedNodesListener"); | |
} | |
public static QuarantinedNodesExtension Get(ActorSystem system) => | |
system.WithExtension<QuarantinedNodesExtension, QuarantinedNodesExtensionProvider>(); | |
} | |
public class QuarantinedNodesExtensionProvider : ExtensionIdProvider<QuarantinedNodesExtension> | |
{ | |
public override QuarantinedNodesExtension CreateExtension(ExtendedActorSystem system) => new QuarantinedNodesExtension(system); | |
} | |
/// <summary> | |
/// Listens to RemotingLifecycleEvent events to detect when this node has been quarantined by another. | |
/// </summary> | |
public class QuarantinedNodesListener : UntypedActor | |
{ | |
private const int MessageThreshold = 10; | |
private readonly HashSet<Address> addressesCollection = new HashSet<Address>(); | |
private int addresses; | |
private bool quarantined; | |
protected override void PreStart() | |
{ | |
base.PreStart(); | |
Context.System.EventStream.Subscribe(Self, typeof(RemotingLifecycleEvent)); | |
} | |
protected override void OnReceive(object message) | |
{ | |
switch (message) | |
{ | |
case ThisActorSystemQuarantinedEvent qe when !quarantined: | |
{ | |
Context.System.Log.Warning("[DEVOPS] Node got quarantined by [{RemoteAddress}]", qe.RemoteAddress); | |
quarantined = true; | |
} | |
break; | |
case AssociationErrorEvent ae when !quarantined: | |
{ | |
var errorMessage = ae.ToString(); | |
if (errorMessage.Contains("The remote system has a UID that has been quarantined")) | |
{ | |
var address = ae.RemoteAddress; | |
addressesCollection.Add(address); | |
addresses++; | |
if (addresses >= MessageThreshold && addressesCollection.Count > 1) | |
{ | |
addressesCollection.Clear(); | |
addresses = 0; | |
Context.System.Log.Warning("[DEVOPS] Node got quarantined via AssociationEvent by [{RemoteAddress}]", ae.RemoteAddress); | |
quarantined = true; | |
} | |
} | |
else if (errorMessage.Contains("The remote system explicitly disassociated")) | |
{ | |
addressesCollection.Clear(); | |
addresses = 0; | |
} | |
} | |
break; | |
} | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment