Demonstrates scheduling unbounded work and applying a throttle to keep the threadpool responsive.

ThrottleDemo.cs

using System;
using System.Diagnostics;
using System.Threading;
using System.Threading.Tasks;

class Program
{
    static void Main(string[] args)
    {
        Console.WriteLine("*** WITHOUT throttling ***");
        MeasureWork(false);

        Console.WriteLine("*** WITH throttling ***");
        MeasureWork(true);
    }

    private static void MeasureWork(bool throttle)
    {
        TaskScheduler scheduler = throttle
            ? new ConcurrentExclusiveSchedulerPair(TaskScheduler.Default, Environment.ProcessorCount * 2).ConcurrentScheduler
            : TaskScheduler.Default;

        Console.WriteLine("Race has begun.");
        var sw = Stopwatch.StartNew();
        var workerTasks = new Task[800];
        for (int i = 0; i < workerTasks.Length; i++)
        {
            workerTasks[i] = Task.Factory.StartNew(Work, CancellationToken.None, TaskCreationOptions.None, scheduler);
        }

        Task allFinished = Task.WhenAll(workerTasks);
        allFinished.ContinueWith(_ => sw.Stop());
        while (!allFinished.IsCompleted)
        {
            Thread.Sleep(500);
            Console.WriteLine("Threadpool responded in: " + MeasureThreadPoolResponsiveness());
        }

        Console.WriteLine($"FINISHED in {sw.Elapsed}");
    }

    static void Work()
    {
        SpinWait.SpinUntil(() => false, 40);
    }

    static TimeSpan MeasureThreadPoolResponsiveness()
    {
        var sw = Stopwatch.StartNew();
        Task.Run(() => sw.Stop()).Wait();
        return sw.Elapsed;
    }
}

ThrottleDemoAsync.cs

using System;
using System.Diagnostics;
using System.Threading;
using System.Threading.Tasks;

class Program
{
    static void Main(string[] args)
    {
        Console.WriteLine("*** WITHOUT throttling ***");
        MeasureWork(false);

        Console.WriteLine("*** WITH throttling ***");
        MeasureWork(true);
    }

    private static void MeasureWork(bool throttled)
    {
        SemaphoreSlim semaphore = throttled ? new SemaphoreSlim(Environment.ProcessorCount * 2) : null;

        Console.WriteLine("Race has begun.");
        var sw = Stopwatch.StartNew();
        var workerTasks = new Task[800];
        for (int i = 0; i < workerTasks.Length; i++)
        {
            workerTasks[i] = Task.Run(async delegate
            {
                if (semaphore != null)
                {
                    await semaphore.WaitAsync();
                }

                await WorkAsync();

                semaphore?.Release();
            });
        }

        Task allFinished = Task.WhenAll(workerTasks);
        allFinished.ContinueWith(_ => sw.Stop());
        while (!allFinished.IsCompleted)
        {
            Thread.Sleep(500);
            Console.WriteLine("Threadpool responded in: " + MeasureThreadPoolResponsiveness());
        }

        Console.WriteLine($"FINISHED in {sw.Elapsed}");
    }
    
    static async Task WorkAsync()
    {
        SpinWait.SpinUntil(() => false, 20);
        await Task.Yield();
        SpinWait.SpinUntil(() => false, 20);
    }

    static TimeSpan MeasureThreadPoolResponsiveness()
    {
        var sw = Stopwatch.StartNew();
        Task.Run(() => sw.Stop()).Wait();
        return sw.Elapsed;
    }
}

@AArnott Have you considered comparing the numbers after letting the thread pool warm up? I think that getting numbers from the throttled variant when the threadpool has not scaled itself up to the workload yet gives the throttling numbers an unfair advantage if you’re going to be comparing the FINISHED in times. These are the numbers from a run after I modified the sync version to use warmup:

Warming up…
Running…
*** WITHOUT throttling ***
Race has begun.
Threadpool responded in: 00:00:02.8652196
Threadpool responded in: 00:00:00.0000353
FINISHED in 00:00:03.4135232
*** WITH throttling ***
Race has begun.
Threadpool responded in: 00:00:00.0000336
Threadpool responded in: 00:00:00.0000357
Threadpool responded in: 00:00:00.0000275
Threadpool responded in: 00:00:00.0000238
Threadpool responded in: 00:00:00.0000246
Threadpool responded in: 00:00:00.0000299
Threadpool responded in: 00:00:00.0000336
Threadpool responded in: 00:00:00.0000299
Threadpool responded in: 00:00:00.0000336
Threadpool responded in: 00:00:00.0000751
FINISHED in 00:00:04.6864532

Good point, @binki. That would explain why the throttled variety ran mysteriously faster.
With your change and output that shows throttling made it slower, I suspect some optimization or picking a more efficient scheduler, or a different concurrency level than I did would reduce the difference between the two.