Serving robots.txt depeding on the domain requested.

.gitignore

/robots*
!/robots.php

.htaccess

Options -Indexes 
ErrorDocument 404 /404.php

<IfModule mod_rewrite.c>
  Options +FollowSymLinks
	RewriteEngine On

	# Serve robots.txt with robots.php only if the latter exists
	RewriteCond %{REQUEST_FILENAME} robots.txt
	RewriteCond %{DOCUMENT_ROOT}/robots.php -f
	RewriteRule ^(.*)$ /robots.php [L]
</IfModule>

README.md

Serving robots.txt depeding on the domain requested.

Domain-specific robots files should be named robots_<domain name with dots>.txt. If there's no such file for the requested domain, a default robots_default.txt is read. If nothing is found, it returns a 404 error code.

Be sure not to put a file named robots.txt in the root dir as some servers ignores the .htaccess UrlRewrite rules when the requested file exists.

robots.php

<?
$hostname = $_SERVER['HTTP_HOST'];

function echoTextFile($file) {
  if (! file_exists($file)) return false;
	if (! is_readable($file)) return false;

	$timestamp = filemtime($file);
	$tsstring = gmdate('D, d M Y H:i:s ', $timestamp) . 'GMT';
	$etag = md5($file . $timestamp);

	header('Content-Type: text/plain');
	header('Content-Length: '.filesize($file));
	header("Last-Modified: $tsstring");
    header("ETag: \"{$etag}\"");

	readfile($file);

	return true;
}

$robotsHost = dirname(__FILE__) . "/robots_{$hostname}.txt";
$robotsDefault = dirname(__FILE__) . "/robots_default.txt";

if(!echoTextFile($robotsHost) && !echoTextFile($robotsDefault)) 
{
	header('HTTP/1.0 404 Not Found');
}