A script to tokenize a given PHP script and display the list of tokens.

tokenize.php

#!/usr/bin/env php
<?php
if (!isset($_SERVER['argv'][1])) {
    echo "Usage: {$_SERVER['argv'][0]} <filename>" . PHP_EOL;
    exit;
}

$tokens = token_get_all(file_get_contents($_SERVER['argv'][1]));
foreach ($tokens as $token) {
    if (is_integer($token[0])) {
        echo str_pad(token_name($token[0]), 28);
        echo trim($token[1]);
    } else {
        echo str_repeat(" ", 28);
        echo $token[0];
    }
    echo PHP_EOL;
}
?>