Here is a solution that allows to write ObjectResult inside Middleware (write http response using MVC formatters (serializers) from Middleware).

ResponseDefaultFormatterService.cs

using Microsoft.AspNetCore.Mvc;
using Microsoft.AspNetCore.Mvc.Formatters;
using Microsoft.AspNetCore.Mvc.Infrastructure;

namespace Services;

public class ResponseDefaultFormatterService
{
    private readonly IHttpResponseStreamWriterFactory _streamWriterFactory;
    private readonly OutputFormatterSelector _formatterSelector;

    public ResponseDefaultFormatterService(
        IHttpResponseStreamWriterFactory streamWriterFactory,
        OutputFormatterSelector formatterSelector)
    {
        _streamWriterFactory = streamWriterFactory;
        _formatterSelector = formatterSelector;
    }
    
    public async Task WriteAsync(HttpContext context, ObjectResult result)
    {
        var formatterContext = new OutputFormatterWriteContext(
            context,
            _streamWriterFactory.CreateWriter,
            result.DeclaredType ?? result.Value?.GetType(),
            result.Value);

        var selectedFormatter = _formatterSelector.SelectFormatter(
            formatterContext,
            (IList<IOutputFormatter>)result.Formatters ?? Array.Empty<IOutputFormatter>(),
            result.ContentTypes);
        if (selectedFormatter == null)
        {
            context.Response.StatusCode = StatusCodes.Status406NotAcceptable;
            return;
        }

        await selectedFormatter.WriteAsync(formatterContext);
    }
}

UnauthorizedResponseModelMiddlewareExample.cs

using Microsoft.AspNetCore.Mvc;

namespace Middlewares;

public class UnauthorizedResponseModelMiddleware
{
    private readonly RequestDelegate _next;
    private readonly ResponseDefaultFormatterService _formatterService;

    public UnauthorizedResponseModelMiddleware(
        RequestDelegate next,
        ResponseDefaultFormatterService formatterService)
    {
        _next = next;
        _formatterService = formatterService;
    }

    public async Task InvokeAsync(HttpContext context)
    {
        await _next(context);

        if (context.Response.StatusCode == StatusCodes.Status401Unauthorized)
        {
            var result = new UnauthorizedObjectResult(new { Message = "Unauthorized", Code = 401 });
            await _formatterService.WriteAsync(context, result);
        }
    }
}