Last active
June 11, 2016 18:57
-
-
Save senz/55c80d29a6ac256387e8890d6f469b16 to your computer and use it in GitHub Desktop.
P2P i-Blocklist generator uses unix gzip and sort
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<?php | |
return [ | |
'list' => [ | |
['http://list.iblocklist.com/?list=ydxerpxkpcfqjaybcssw&fileformat=p2p&archiveformat=gz', 'p2p', 'gz', 'bluetack-level1'], | |
['http://list.iblocklist.com/?list=gyisgnzbhppbvsphucsw&fileformat=p2p&archiveformat=gz', 'p2p', 'gz', 'bluetack-level2'], | |
['http://list.iblocklist.com/?list=plkehquoahljmyxjixpu&fileformat=p2p&archiveformat=gz', 'p2p', 'gz', 'bluetack-suspicious'], | |
['http://list.iblocklist.com/?list=xshktygkujudfnjfioro&fileformat=p2p&archiveformat=gz', 'p2p', 'gz', 'bluetack-microsoft'], | |
['http://list.iblocklist.com/?list=gihxqmhyunbxhbmgqrla&fileformat=p2p&archiveformat=gz', 'p2p', 'gz', 'blutack-bogon'], | |
['http://list.iblocklist.com/?list=lujdnbasfaaixitgmxpp&fileformat=p2p&archiveformat=gz', 'p2p', 'gz', 'cidr-report.org-bogon'], | |
['http://list.iblocklist.com/?list=cwworuawihqvocglcoss&fileformat=p2p&archiveformat=gz', 'p2p', 'gz', 'bluetack-badpeers'], | |
], | |
'sortOut' => '/srv/www/blocklist/public/outsorted' | |
]; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<?php | |
/** | |
* @author Konstantin G Romanov | |
*/ | |
if (!defined('STDERR') || !defined('STDOUT')) { | |
echo "Must be ran in console\n"; | |
exit(1); | |
} | |
$configPath = getenv('BLOC_CONFIG'); | |
if (!$configPath || !is_readable($configPath)) { | |
echo "BLOC_CONFIG is not set or readable"; | |
exit(1); | |
} | |
$config = require $configPath; | |
function logstr($msg, $error = false, $insertnl = true) { | |
fwrite($error ? STDERR : STDOUT, $msg . ($insertnl ? "\n" : '')); | |
} | |
function sortTextFile($name) { | |
$seeker = new TextLineSeeker($name); | |
$total = $seeker->getTotalLines(); | |
$pivot = floor($total / 2); | |
} | |
$curl = curl_init(); | |
$linecount = 0; | |
$bufflength = 1024; | |
$bigoutname = tempnam(sys_get_temp_dir(), 'out'); | |
$bigout = fopen($bigoutname, 'w+'); | |
logstr("output name: " . $bigoutname); | |
$downmap = []; | |
foreach ($config['list'] as $item) { | |
list($url, $format, $compression, $desc) = $item; | |
$outfilename = tempnam(sys_get_temp_dir(), 'block-'); | |
$outfile = fopen($outfilename, 'w'); | |
$downmap[$url] = [$outfile, $outfilename]; | |
curl_setopt_array($curl, [ | |
CURLOPT_FILE => $outfile, | |
CURLOPT_FOLLOWLOCATION => true, | |
CURLOPT_URL => $url, | |
CURLOPT_TIMEOUT => 120, | |
]); | |
logstr("Downloading " . $desc . "...", false, false); | |
$result = curl_exec($curl); | |
logstr("done."); | |
$info = curl_getinfo($curl); | |
if (!$result) { | |
logstr("exec error: " . curl_error($curl), true); | |
logstr(print_r($info, true), true); | |
unlink($outfilename); | |
unset($downmap[$url]); | |
logstr("skipping."); | |
continue; | |
} | |
if ('gz' == $compression) { | |
logstr("decompressing"); | |
$gzh = gzopen($outfilename, 'r'); | |
while (!feof($gzh)) { | |
$buff = gzgets($gzh); | |
if ($buff[0] != '#' && !empty(trim($buff))) | |
fwrite($bigout, $buff); | |
logstr(".", false, false); | |
$linecount++; | |
} | |
echo "\n"; | |
fclose($outfile); | |
} | |
} | |
C_ALL='C' sort -u $bigoutname", 'r'); | |
while ( !feof($sorthandle) ) { | |
$buff = fread($sorthandle, 1024); | |
$res = fwrite($sortedouthandle, $buff); | |
if (!$res) throw new Exception("Write failure"); | |
logstr('.', false, false); | |
} | |
logstr('done'); | |
fclose($sortedouthandle); | |
unlink($bigoutname); | |
system('gzip -f ' . $config['sortOut']); | |
logstr("Total lines: " . $linecount); | |
logstr("DONE"); | |
interface LineSeekable { | |
/** | |
* @return int | |
*/ | |
function getCurrentLineNo(); | |
/** | |
* @param int $line | |
* @return void | |
*/ | |
function setLineNo(int $line); | |
/** | |
* @return int | |
*/ | |
function getTotalLines(); | |
/** | |
* @return string | |
*/ | |
function getLine(); | |
/** | |
* @param int $line1 | |
* @param int $line2 | |
* @return void | |
*/ | |
function exchangeLines( int $line1, int $line2 ); | |
/** | |
* @param int $from | |
* @param int $to | |
* @return void | |
*/ | |
function moveLine (int $from, int $to ); | |
} | |
class TextLineSeeker implements LineSeekable { | |
private $fileHandle; | |
private $currentLine; | |
private $totalLines; | |
/** | |
* TextLineSeeker constructor. | |
* @param $fileName | |
*/ | |
public function __construct( $fileName ) { | |
if (!is_readable($fileName)) throw new InvalidArgumentException("fileName must be a readable file"); | |
$this->fileHandle = fopen($fileName, 'r+'); | |
rewind($this->fileHandle); | |
$this->currentLine = 0; | |
$this->totalLines = 0; | |
while (!feof($this->fileHandle) && fgets($this->fileHandle) !== false) $this->totalLines++; | |
} | |
function getCurrentLineNo() { | |
return $this->currentLine; | |
} | |
function setLineNo( int $line ) { | |
rewind($this->fileHandle); | |
$cline = 0; | |
while (!feof($this->fileHandle) && $cline < $line) { | |
fgets( $this->fileHandle ); | |
$cline++; | |
} | |
if ($line != $cline) throw new Exception("Line not found"); | |
$this->currentLine = $line; | |
} | |
function getTotalLines() { | |
return $this->totalLines; | |
} | |
function getLine() { | |
$line = fgets($this->fileHandle); | |
$res = fseek($this->fileHandle, -(strlen($line) + 1), SEEK_CUR); | |
if (0 !== $res) throw new Exception("Failed to set file cursor"); | |
return $line; | |
} | |
function exchangeLines( int $line1, int $line2 ) { | |
$this->setLineNo($line1); | |
$str1 = $this->getLine(); | |
$this->setLineNo($line2); | |
$str2 = $this->getLine(); | |
fwrite($this->fileHandle, $str1); | |
$this->setLineNo($line1); | |
fwrite($this->fileHandle, $str2); | |
} | |
function moveLine( int $from, int $to ) { | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment