Magento shell script to parse system logs to CSV

djv-applog.php

<?php

require_once 'abstract.php';

class Djv_Shell_AppLog extends Mage_Shell_Abstract
{
    const DEFAULT_SOURCE_FILE = 'system.log';
    const LOG_LINE_REGEX = "/^(?P<date>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}([+-]\d{2}:\d{2})?)\b\s*\b(?P<errcode>[A-Z]+)\b\s\((?P<errno>\d)\):\s*(?P<message>.*?)(\s+in\s(?P<file>[^\s]+)\s+on\sline\s(?<line>\d+))?$/";

    /**
     * Djv_Shell_AppLog constructor.
     */
    public function __construct()
    {
        parent::__construct();

        $this->csv = new Varien_File_Csv();
    }

    /**
     * @throws Mage_Core_Exception
     */
    public function run()
    {
        switch (true) {
            case in_array('to-csv', $_SERVER['argv']) :
                $sourceFile = $this->getArg('source') ?? self::DEFAULT_SOURCE_FILE;
                $destFile = $this->getArg('dest');
                if (!$destFile) {
                    $extension = pathinfo($sourceFile, PATHINFO_EXTENSION);
                    $destFile = substr($sourceFile, 0, 0 - (strlen($extension) + 1)) . '.csv';
                }

                $this->toCsv($sourceFile, $destFile);
                echo PHP_EOL."Done".PHP_EOL;
                break;

            default :
                echo $this->usageHelp();
                break;
        }
    }

    /**
     * @param string $line
     * @return array
     */
    public function extractData($line)
    {
        $line = trim($line);

        $return = [
            'date' => null,
            'errcode' => null,
            'errno' => null,
            'type' => null,
            'subtype' => null,
            'message' => null,
            'file' => null,
            'lineno' => null,
//            'line' => $line,
        ];

        $matches = [];
        preg_match(self::LOG_LINE_REGEX, $line, $matches);
        if (empty($matches)) {
            return $return;
        }

        $return = array_merge($return, [
            'date' => $matches['date'],
            'errcode' => $matches['errcode'],
            'errno' => $matches['errno'],
            'message' => $matches['message'],
            'file' => $matches['file'],
            'lineno' => $matches['lineno'],
        ]);

        $return = array_merge($return, $this->getErrorClassification($matches['message']));

        return $return;
    }

    /**
     * @param string $message
     * @return array
     */
    public function getErrorClassification($message)
    {
        $return = [
            'type' => null,
            'subtype' => null,
        ];

        $matches = [];
        switch (true) {
            case preg_match("/SQLSTATE\[(\d+)\]/", $message,$matches) :
                $return['type'] = 'SQL Error';
                $sqlStateCode = $matches[1] ? (int)$matches[1] : null;
                switch (true) {
                    case $sqlStateCode >= 21000 && $sqlStateCode < 22000:
                        $return['subtype'] = "21xxx : cardinality violation";
                        break;
                    case $sqlStateCode >= 22000 && $sqlStateCode < 23000 :
                        $return['subtype'] = "22xxx : data exception";
                        break;
                    case $sqlStateCode >= 23000 && $sqlStateCode < 24000 :
                        $return['subtype'] = "23xxx : integrity constraint violation";
                        break;
                    case $sqlStateCode >= 24000 && $sqlStateCode < 25000 :
                        $return['subtype'] = "24xxx : invalid cursor state ";
                        break;
                    case $sqlStateCode >= 25000 && $sqlStateCode < 26000 :
                        $return['subtype'] = "25xxx :  invalid transaction state ";
                        break;
                    default :
                        $return['subtype'] = $matches[1];
                        break;
                }
                break;

            case false !== strpos($message, "Not valid template file:") :
                $return['type'] = 'Magento error';
                $return['subtype'] = 'Invalid template';
                break;

            case false !== strpos($message, "Notice: Undefined offset") :
            case false !== strpos($message, "Notice: Undefined index") :
            case false !== strpos($message, "Notice: Undefined variable") :
                $return['type'] = 'PHP Notice';
                $return['subtype'] = 'Undefined item';
                break;

            case preg_match('/Missing argument \d+ for/', $message) :
                $return['type'] = 'PHP Warning';
                $return['subtype'] = 'Missing argument';
                break;

            case preg_match("/(include|require)(_once)?\([^\)]+\): failed to open stream:/", $message) :
            case preg_match("/(include|require)(_once)?\(\): Failed opening/", $message) :
                $return['type'] = 'PHP Warning';
                $return['subtype'] = 'Missing include';
                break;

            case 0 === strpos($message, "Notice: ") :
                $return['type'] = 'PHP Notice';
                break;

            case 0 === strpos($message, "Warning: ") :
                $return['type'] = 'PHP Warning';
                break;
        }

        return $return;
    }

    /**
     * @param string $sourceFile
     * @param string $destFile
     * @return bool
     * @throws Mage_Core_Exception
     */
    public function toCsv($sourceFile, $destFile)
    {
        $sourceFile = $this->getJailedFilepath($sourceFile);
        if (!file_exists($sourceFile)) {
            throw new Mage_Core_Exception(__("Cannot find source file %s", $sourceFile));
        }

        $destFile = $this->getJailedFilepath($destFile);
        $addHeaders = true;
        if (file_exists($destFile)) {
            switch ($this->getArg('destAction')) {
                case 'append' :
                    $addHeaders = false;
                    break;

                case 'overwrite' :
                    unlink($destFile);
                    break;

                case 'archive' :
                default :
                    $this->archiveFile($destFile);
                    break;
            }
        }

        $sourceFileObject = new SplFileObject($sourceFile);
        $destFileHandle = fopen($destFile, 'a+');
        if (!$destFileHandle) {
            throw new RuntimeException(sprintf(
                "Cannot open '%s' for writing",
                $destFile
            ));
        }

        $rows = 0;
        while ($sourceFileObject->valid()) {
            $line = $sourceFileObject->fgets();
            $lineData = $this->extractData($line);
            if (!array_filter($lineData)) {
                continue;
            }

            if ($addHeaders && 0 === $rows) {
                $this->csv->fputcsv($destFileHandle, array_keys($lineData));
            }
            $this->csv->fputcsv($destFileHandle, $lineData);
            $rows++;
        }
        fclose($destFileHandle);

        return true;
    }

    /**
     * @param string $filepath
     * @return bool
     */
    public function archiveFile($filepath)
    {
        if (!file_exists($filepath)) {
            return false;
        }

        $io = new Varien_Io_File();
        $from = $filepath;
        $to = $filepath . '.' . date('YmdHis');

        return $io->mv($from, $to);
    }

    /**
     * @param string $filepath
     * @return string
     */
    public function getJailedFilepath($filepath)
    {
        return Mage::getBaseDir('log') . DS . preg_replace('/^[\.\/\\\\]+/', null, $filepath);
    }

    /**
     * @return string
     */
    public function usageHelp()
    {
        return <<<USAGE
Usage:  php -f djv-applog.php -- [options]
        php -f djv-applog.php -- to-csv --source system.log --dest system.csv
        
    to-csv      Parses a log file (not exception log at this time) and creates a csv version
                If destination exists, it will be archived first                
    
    --source <path> File from which to read (jailed to var/log)
    --dest <path>   File to which transformations are saved (jailed to var/log)
    --destAction <archive*,overwrite,append>
    

USAGE;
    }
}

$shell = new Djv_Shell_AppLog();
$shell->run();