Skip to content

Instantly share code, notes, and snippets.

@ismaelhamed
Last active March 18, 2021 06:47
Show Gist options
  • Save ismaelhamed/4341152fd27b8617c2667deba57c82c9 to your computer and use it in GitHub Desktop.
Save ismaelhamed/4341152fd27b8617c2667deba57c82c9 to your computer and use it in GitHub Desktop.
An extension to monitor whether we have been quarantined
public class QuarantinedNodesExtension : IExtension
{
private readonly ExtendedActorSystem system;
private readonly IActorRef listener;
public QuarantinedNodesExtension(ExtendedActorSystem system)
{
this.system = system;
listener = system.SystemActorOf(Props.Create<QuarantinedNodesListener>().WithDeploy(Deploy.Local), "QuarantinedNodesListener");
}
public static QuarantinedNodesExtension Get(ActorSystem system) =>
system.WithExtension<QuarantinedNodesExtension, QuarantinedNodesExtensionProvider>();
}
public class QuarantinedNodesExtensionProvider : ExtensionIdProvider<QuarantinedNodesExtension>
{
public override QuarantinedNodesExtension CreateExtension(ExtendedActorSystem system) => new QuarantinedNodesExtension(system);
}
/// <summary>
/// Listens to RemotingLifecycleEvent events to detect when this node has been quarantined by another.
/// </summary>
public class QuarantinedNodesListener : UntypedActor
{
private const int MessageThreshold = 10;
private readonly HashSet<Address> addressesCollection = new HashSet<Address>();
private int addresses;
private bool quarantined;
protected override void PreStart()
{
base.PreStart();
Context.System.EventStream.Subscribe(Self, typeof(RemotingLifecycleEvent));
}
protected override void OnReceive(object message)
{
switch (message)
{
case ThisActorSystemQuarantinedEvent qe when !quarantined:
{
Context.System.Log.Warning("[DEVOPS] Node got quarantined by [{RemoteAddress}]", qe.RemoteAddress);
quarantined = true;
}
break;
case AssociationErrorEvent ae when !quarantined:
{
var errorMessage = ae.ToString();
if (errorMessage.Contains("The remote system has a UID that has been quarantined"))
{
var address = ae.RemoteAddress;
addressesCollection.Add(address);
addresses++;
if (addresses >= MessageThreshold && addressesCollection.Count > 1)
{
addressesCollection.Clear();
addresses = 0;
Context.System.Log.Warning("[DEVOPS] Node got quarantined via AssociationEvent by [{RemoteAddress}]", ae.RemoteAddress);
quarantined = true;
}
}
else if (errorMessage.Contains("The remote system explicitly disassociated"))
{
addressesCollection.Clear();
addresses = 0;
}
}
break;
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment