PHP | Get Contents of Robots.txt File for a Text File of Domains

php-get-contents-of-robots-txt-file.php

<?php
//Purpose to Check Contents of the Robots.txt File for a Series of Domains in a Local Text File
//define text file location
$old_url_text_file = "list.txt";
//get contents of text file
$old_url_list = file_get_contents($old_url_text_file);
//process text file
$old_urls = explode("\n", $old_url_list);
$old_urls = array_filter(array_map('trim', $old_urls));
//loop through text files
foreach($old_urls as $urls) {
    //Define XML Data - In this Case - The Site to Be Parked
    $domain = trim($urls);
//print results as table
    print "<table>";
    print "<tr><td>" . $domain . "</td>" . "<td>" . file_get_contents($domain) . "</td></tr>" . "<br />";
    print "</table>";
}
?>