Top 1M

parse.php

<?php

require 'vendor/autoload.php';

$top = fopen('./top-1m.csv','r');
$client = new GuzzleHttp\Client();

while ( ($data = fgetcsv($top) ) !== FALSE ) {

    $sitename = $data[1]; // google.com

    try {
        $response = $client->get('http://' . $sitename . '/.git/',[
            'connect_timeout' => 3,
            'verify' => false,
        ]);

        $content = $response->getBody()->getContents();
        if(stristr($content, 'branches') === FALSE) {
            continue;
        }

        $file = 'white.txt';
        $sitename = $sitename."\n";
        file_put_contents($file, $sitename, FILE_APPEND | LOCK_EX);


    }catch (Exception $exception){
        continue;
    }

}