duplicate_search.php

<?php

$lines    = file($argv[1], FILE_IGNORE_NEW_LINES);
$buffer   = '';
$requests = array();

foreach ($lines as $line)
{
    if (empty($line)) {
        $requests[] = trim($buffer);
        $buffer     = '';
    }
    else {
        $buffer .= ' ' . $line;
    }
}


$readedRequests  = array();
$repeatedRequest = array();

foreach ($requests as $request)
{
    if (in_array($request, $readedRequests) && !in_array($request, $repeatedRequest)) {
        $repeatedRequest[] = $request;
    }
    else {
        $readedRequests[] = $request;
    }
}

$totalOcc = 0;
foreach ($repeatedRequest as $request)
{
    $occ = array_filter($requests, function($e) use ($request) {
        return $e == $request;
    });

    printf("--> The following request is repeated %d times.\n\n%s\n\n\n\n", count($occ), $request);
    $totalOcc += count($occ);
}

$total_requests = count($requests);
$duplicated_requests = $totalOcc - count($repeatedRequest);

printf("-----------------------------------------\n");
printf("--> %6d repeated request\n", count($repeatedRequest));
printf("--> %6d duplicated requests\n", $duplicated_requests);
printf("--> %6d unique requests\n", $total_requests - $duplicated_requests);
printf("--> %6d total requests\n", $total_requests);

$ratio = (double)1 - ((double)$duplicated_requests / (double)$total_requests);
printf("--> ratio %f \n", $ratio);