PHP Snippets

PHPSnippets.php

    // Split the string into individual words
    $words = explode(' ', $string);

    // Initialize a counter
    $tokens = 0;

    // Iterate over the words
    foreach ($words as $word) {
        // Increment the counter by the ceil of the word's length divided by 4
        // GPT-3's tokenizer uses a byte pair encoding that won't split a word into more than one token if it's 4 bytes or less
        $tokens += ceil(mb_strlen($word) / 4);
    }

    // Return the total count
    return $tokens;
}

$string = "antidisestablishmentarianism";
echo countTokens($string);