SkipWhenPreviousJobIsRunningAttribute.cs

SkipWhenPreviousJobIsRunningAttribute.cs

// Zero-Clause BSD (more permissive than MIT, doesn't require copyright notice)
// 
// Permission to use, copy, modify, and/or distribute this software for any purpose
// with or without fee is hereby granted.
// 
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
// OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
// TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF
// THIS SOFTWARE.

// Hangfire.Core 1.8+ is required, for previous versions please see revision from year 2022.

using System;
using System.Collections.Generic;
using Hangfire.Client;
using Hangfire.Common;
using Hangfire.States;
using Hangfire.Storage;

namespace ConsoleApp28
{
    public class SkipWhenPreviousJobIsRunningAttribute : JobFilterAttribute, IClientFilter, IApplyStateFilter
    {
        public void OnCreating(CreatingContext context)
        {
            // We can't handle old storages
            if (!(context.Connection is JobStorageConnection connection)) return;

            // We should run this filter only for background jobs based on 
            // recurring ones
            if (!context.Parameters.TryGetValue("RecurringJobId", out var parameter)) return;

            var recurringJobId = parameter as string;

            // RecurringJobId is malformed. This should not happen, but anyway.
            if (String.IsNullOrWhiteSpace(recurringJobId)) return;

            var running = connection.GetValueFromHash($"recurring-job:{recurringJobId}", "Running");
            if ("yes".Equals(running, StringComparison.OrdinalIgnoreCase))
            {
                context.Canceled = true;
            }
        }

        public void OnCreated(CreatedContext filterContext)
        {
        }

        public void OnStateApplied(ApplyStateContext context, IWriteOnlyTransaction transaction)
        {
            if (context.NewState is EnqueuedState)
            {
                ChangeRunningState(context, "yes");
            }
            else if ((context.NewState.IsFinal && !FailedState.StateName.Equals(context.OldStateName, StringComparison.OrdinalIgnoreCase)) || 
                     (context.NewState is FailedState))
            {
                ChangeRunningState(context, "no");
            }
        }

        public void OnStateUnapplied(ApplyStateContext context, IWriteOnlyTransaction transaction)
        {
        }

        private static void ChangeRunningState(ApplyStateContext context, string state)
        {
            // We can't handle old storages
            if (!(context.Connection is JobStorageConnection connection)) return;

            // Obtaining a recurring job identifier
            var recurringJobId = context.GetJobParameter<string>("RecurringJobId", allowStale: true);
            if (String.IsNullOrWhiteSpace(recurringJobId)) return;

            if (context.Storage.HasFeature(JobStorageFeatures.Transaction.AcquireDistributedLock))
            {
                // Acquire a lock in newer storages to avoid race conditions
                ((JobStorageTransaction)context.Transaction).AcquireDistributedLock(
                    $"lock:recurring-job:{recurringJobId}",
                    TimeSpan.FromSeconds(5));
            }

            // Checking whether recurring job exists
            var recurringJob = connection.GetValueFromHash($"recurring-job:{recurringJobId}", "Job");
            if (String.IsNullOrEmpty(recurringJob)) return;

            // Changing the running state
            context.Transaction.SetRangeInHash(
                $"recurring-job:{recurringJobId}",
                new[] { new KeyValuePair<string, string>("Running", state) });
        }
    }
}

hello there, I have a problem with this code. After some time (cannot really say what is some) some of my recurring jobs which are decorated with this attribute are stopping execution. I mean they are immediately canceled after new execution and when I placed logger there here is what it produces:

2023-11-28 16:05:33.7064|INFO|Namespace.Jobs.Hangfire.Attributes.SkipWhenPreviousJobIsRunningAttribute|running: yes
2023-11-28 16:05:33.7064|INFO|Namespace.Jobs.Hangfire.Attributes.SkipWhenPreviousJobIsRunningAttribute|Job will be canceled

and here is code in C#

var logger = new NLogFactory().Create(this);
string running = connection.GetValueFromHash($"recurring-job:{recurringJobId}", "Running");
logger.Info($"running: {running}");
if ("yes".Equals(running, StringComparison.OrdinalIgnoreCase))
{
    logger.Info($"Job will be canceled");
    context.Canceled = true;
}

I don't know why is it happening, but it won't trigger from this moment... I am using SQLite file as DB.

@frozzen10 The issue you're experiencing might be because the "Running" status of the job is not being reset properly when the job fails or when it's in a final state. This could cause the job to be immediately canceled on the next execution because the system thinks it's still running. To fix this, you should ensure that the "Running" status is reset in all cases when the job is in a final state, not just when it's not in a FailedState.

in OnStateApplied
`
var recurringJobId = SerializationHelper.Deserialize(
context.Connection.GetJobParameter(context.BackgroundJob.Id, "RecurringJobId"));

if (string.IsNullOrWhiteSpace(recurringJobId)) return;

if (context.NewState is EnqueuedState)
{
transaction.SetRangeInHash(
$"recurring-job:{recurringJobId}",
new[] {new KeyValuePair<string, string>("Running", "yes")});
}
else if (context.NewState.IsFinal)
{
transaction.SetRangeInHash(
$"recurring-job:{recurringJobId}",
new[] {new KeyValuePair<string, string>("Running", "no")});
}
`

If you have this issue here is the solution for it.

After investigation, I found out that our customer SkipWhenPreviousJobIsRunningAttribute will trigger
OnStateApplied method after the job has been deleted.

The reason why it is doing that is this scenario :
The job will be triggered for running, during the running time and still not completed we delete the job!
On time the job is completed method OnStateApplied will be triggered even if the job is deleted and it will create a new row in the table hash.

If we want to keep the custom attribute SkipWhenPreviousJobIsRunningAttribute
Before adding the new row to hash table check if the job is deleted or not

var job = JobStorage.Current.GetConnection().GetRecurringJobs(new[] { recurringJobId }).FirstOrDefault();
 if(job is { Removed: true})  return;

 transaction.SetRangeInHash(
     $"recurring-job:{recurringJobId}",
     new[] { new KeyValuePair<string, string>(RunningKey, "no") });

The benefit of this approach is that it will save us of creating unneeded data in the DB.
Unfortunately, it will increase the load on the DB since it is one more additional request.

Thanks for handling this. I have updated the gist with new methods available in Hangfire 1.8+ to avoid any race conditions. They work by acquiring a lock in the same transaction, and checks existence of a recurring job first. So now should be no troubles running this script even if everything is going wrong.