Results pattern with a Post-Processor doing the response sending.

Program.cs

var bld = WebApplication.CreateBuilder(args);
bld.Services
   .AddFastEndpoints()
   .SwaggerDocument();

var app = bld.Build();
app.UseFastEndpoints()
   .UseSwaggerGen();
app.Run();

sealed class Request
{
    public bool IsHappyPath { get; set; }
}

sealed class Response
{
    public string Message { get; set; }
}

sealed class TestEndpoint : Endpoint<Request, Result<Response>> //set response type to ardalis Result<T>
{
    public override void Configure()
    {
        Get("test/{IsHappyPath}");
        AllowAnonymous();
        DontAutoSendResponse();            //disable auto send to allow post-processor to handle sending
        PostProcessor<ResponseSender>();   //register post processor
        Description(
            x => x.Produces<Response>(200) //override swagger response type for 200 ok
                  .Produces<ErrorResponse>(400));
    }

    public override Task<Result<Response>> ExecuteAsync(Request r, CancellationToken ct)
        => Task.FromResult(HelloService.SayHello(r.IsHappyPath)); //return a Result<T>
}

sealed class ResponseSender : IPostProcessor<Request, Result<Response>>
{
    public async Task PostProcessAsync(IPostProcessorContext<Request, Result<Response>> ctx, CancellationToken ct)
    {
        if (!ctx.HttpContext.ResponseStarted())
        {
            var result = ctx.Response!;

            switch (result.Status)
            {
                case ResultStatus.Ok:
                    await ctx.HttpContext.Response.SendAsync(result.GetValue());

                    break;

                case ResultStatus.Invalid:
                    var failures = result.ValidationErrors.Select(e => new ValidationFailure(e.Identifier, e.ErrorMessage)).ToList();
                    await ctx.HttpContext.Response.SendErrorsAsync(failures);

                    break;
            }
        }
    }
}

sealed class HelloService
{
    public static Result<Response> SayHello(bool isHappyPath)
    {
        if (!isHappyPath)
        {
            return Result<Response>.Invalid(
                new List<ValidationError>
                {
                    new()
                    {
                        Identifier = nameof(Request.IsHappyPath),
                        ErrorMessage = "I am unhappy!"
                    }
                });
        }

        return Result<Response>.Success(new() { Message = "hello world..." });
    }
}

@nhwilly
since you wrote {Edit: Hold up on this, I might have this wrong...}, i didn't read your code.
if you need further assitance regarding this, don't hesitate to create a new gh issue or discord post with a "simplified" repro ;-)