Simple robots.txt handler to cater for multiple deploy environments. This example stores the a "build environment" variable in the web.config AppSettings and the handler reads this setting and serves either the production or non-production (dev) physical text file from the file system.

robots_dev.txt

User-agent: *
Disallow: /

robots_prod.txt

User-agent: *
Disallow: /api
Disallow: /api/*
Disallow: /dist/*

RobotsTxtHandler.cs

using System.Web;
using System.Web.Routing;
using System.Configuration;

namespace MyWebsite.Infrastructure
{
    public class RobotsTxtHandler : IRouteHandler
    {
        public IHttpHandler GetHttpHandler(RequestContext requestContext)
        {
            // Determine which robots.txt file to use based on environment setting in AppSettings
            // How the environment is determined would depend on your individual application.
            string robotsFile;

            if (ConfigurationManager.AppSettings["BuildEnvironment"] == "Prod")
            {
                robotsFile = "/robots_prod.txt";
            }
            else
            {
                robotsFile = "/robots_dev.txt";
            }

            requestContext.HttpContext.Response.Clear();
            requestContext.HttpContext.Response.ContentType = "text/plain";
            requestContext.HttpContext.Response.ContentEncoding = System.Text.Encoding.UTF8;

            requestContext.HttpContext.Response.WriteFile(requestContext.HttpContext.Server.MapPath(robotsFile));

            requestContext.HttpContext.Response.End();

            return null;
        }
    }
}

RouteConfig.cs

using System.Web.Mvc;
using System.Web.Routing;

namespace MyWebsite
{
    public class RouteConfig
    {
        public static void RegisterRoutes(RouteCollection routes)
        {
            // Register custom handler for /robots.txt
            RouteTable.Routes.Add("RobotsTxt", new Route("robots.txt", new RobotsTxtHandler()));
          
            // Omitted rest of RouteConfig for brevity
        }
    }
}

web.config

<?xml version="1.0" encoding="utf-8"?>
<configuration>
  <appSettings>
    <!-- Could be Prod|Dev|Staging|UAT -->
    <add key="BuildEnvironment" value="Prod" />
  </appSettings>
  <!-- omitted rest of web.config for brevity -->
</configuration>