AI generated Nepali syllables splitter

splitter-unicode.php

<?php

function isDevanagariConsonant($char)
{
    $code = uniord($char);
    return ($code >= 0x0915 && $code <= 0x0939); // क - ह
}

function isVowelSign($char)
{
    $vowelSigns = ["ा", "ि", "ी", "ु", "ू", "े", "ै", "ो", "ौ"];
    return in_array($char, $vowelSigns, true);
}

function isVirama($char)
{
    return $char === "्";
}

function isModifier($char)
{
    return in_array($char, ["ं", "ः", "ँ"], true); // anusvara, visarga, chandrabindu
}

// Unicode-safe ord
function uniord($c)
{
    $h = mb_convert_encoding($c, 'UCS-4BE', 'UTF-8');
    $v = unpack('N', $h);
    return $v[1];
}

// Split into akshara-like segments
function splitNepaliWord($word)
{
    $chars = preg_split('//u', $word, -1, PREG_SPLIT_NO_EMPTY);
    $segments = [];
    $current = "";

    $len = count($chars);

    for ($i = 0; $i < $len; $i++) {
        $char = $chars[$i];
        $current .= $char;

        // Handle consonant clusters (with virama)
        if (isDevanagariConsonant($char)) {
            while (
                $i + 1 < $len &&
                isVirama($chars[$i + 1]) &&
                $i + 2 < $len &&
                isDevanagariConsonant($chars[$i + 2])
            ) {
                $current .= $chars[$i + 1] . $chars[$i + 2];
                $i += 2;
            }

            // Attach vowel sign if present
            if ($i + 1 < $len && isVowelSign($chars[$i + 1])) {
                $current .= $chars[$i + 1];
                $i++;
            }

            // Attach modifiers (ं, ः,ँ)
            while ($i + 1 < $len && isModifier($chars[$i + 1])) {
                $current .= $chars[$i + 1];
                $i++;
            }

            // End of one segment
            $segments[] = $current;
            $current = "";
        }
    }

    // leftover safety
    if ($current !== "") {
        $segments[] = $current;
    }

    return $segments;
}


// Process full text (like your original)
function processLine($line)
{
    $pattern = '/([\p{Devanagari}]+|[^\p{Devanagari}]+)/u';
    preg_match_all($pattern, $line, $matches);

    $output = "";

    foreach ($matches[0] as $part) {
        if (preg_match('/^[\p{Devanagari}]+$/u', $part)) {
            $segments = splitNepaliWord($part);
            $output .= implode("   ", $segments); // 3 spaces
        } else {
            $output .= $part;
        }
    }

    return $output;
}


// Example
$line = "नेपाल राम्रो छ।";
echo processLine($line) . PHP_EOL;

splitter.cpp

// eg: diggaj => dig gaj
// eg. kathmandu => kath man du

#include <QCoreApplication>
#include <QString>
#include <QStringList>
#include <QDebug>
#include <QFile>
#include <QTextStream>
#include <QSet>
#include <QRegularExpression>

// Helper to determine the length of special Nepali consonant units
int getConsonantUnitSize(const QString &word, int index) {
    if (index >= word.length()) return 0;

    // Check 3-letter units
    QString tri = word.mid(index, 3).toLower();
    if (tri == "chh" || tri == "yan") return 3;

    // Check 2-letter units
    QString bi = word.mid(index, 2).toLower();
    static const QSet<QString> digraphs = {
        "bh", "ch", "dh", "gh", "jh", "kh", "ng", "ph", "th", "sh"
    };
    if (digraphs.contains(bi)) return 2;

    return 1;
}

// Logic to split a single word into segments
QStringList splitWordToSegments(QString word) {
    if (word.isEmpty()) return QStringList();
    
    QStringList segments;
    QString currentSegment;
    const QString vowels = "aeiouAEIOU";

    for (int i = 0; i < word.length(); ++i) {
        currentSegment.append(word.at(i));

        if (vowels.contains(word.at(i))) {
            // Keep double/triple vowels together
            if (i + 1 < word.length() && vowels.contains(word.at(i + 1))) {
                continue;
            }

            // Pull Rule: check next consonant unit
            int nextUnitSize = getConsonantUnitSize(word, i + 1);
            if (nextUnitSize > 0) {
                int posAfterUnit = i + 1 + nextUnitSize;
                
                // If followed by another consonant OR if it's a double consonant (e.g., pp in chappa)
                bool isDouble = (word.at(i+1).toLower() == word.mid(posAfterUnit, 1).toLower());
                bool nextIsConsonant = (posAfterUnit >= word.length() || !vowels.contains(word.at(posAfterUnit)));

                if (nextIsConsonant || isDouble) {
                    currentSegment.append(word.mid(i + 1, nextUnitSize));
                    i += nextUnitSize;
                }
            }
            segments.append(currentSegment);
            currentSegment.clear();
        }
    }

    // Attach leftovers to the last segment
    if (!currentSegment.isEmpty()) {
        if (!segments.isEmpty()) segments.append(segments.takeLast() + currentSegment);
        else segments.append(currentSegment);
    }
    return segments;
}

int main(int argc, char *argv[]) {
    QCoreApplication a(argc, argv);

    QFile file("words.txt");
    if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) {
        qCritical() << "Could not open words.txt";
        return 1;
    }

    QTextStream in(&file);
    // Regex to separate words from everything else (spaces, punctuation)
    QRegularExpression re("([a-zA-Z]+|[^a-zA-Z]+)");

    while (!in.atEnd()) {
        QString line = in.readLine();
        QString processedLine;

        QRegularExpressionMatchIterator it = re.globalMatch(line);
        while (it.hasNext()) {
            QRegularExpressionMatch match = it.next();
            QString part = match.captured(1);

            if (QRegularExpression("^[a-zA-Z]+$").match(part).hasMatch()) {
                // Split the word and join with a space as requested
                processedLine += splitWordToSegments(part).join("   "); // 3 spaces
            } else {
                // Keep spaces and punctuation exactly as they are
                processedLine += part;
            }
        }
        qDebug().noquote() << processedLine << "\n";
    }

    file.close();
    return 0;
}

splitter.php

<?php

// Helper to determine the length of special Nepali consonant units
function getConsonantUnitSize($word, $index) {
    $length = strlen($word);
    if ($index >= $length) return 0;

    // Check 3-letter units
    $tri = strtolower(substr($word, $index, 3));
    if ($tri === "chh" || $tri === "yan") return 3;

    // Check 2-letter units
    $bi = strtolower(substr($word, $index, 2));
    $digraphs = ["bh", "ch", "dh", "gh", "jh", "kh", "ng", "ph", "th", "sh"];

    if (in_array($bi, $digraphs)) return 2;

    return 1;
}

// Logic to split a single word into segments
function splitWordToSegments($word) {
    if ($word === "") return [];

    $segments = [];
    $currentSegment = "";
    $vowels = "aeiouAEIOU";
    $length = strlen($word);

    for ($i = 0; $i < $length; $i++) {
        $currentSegment .= $word[$i];

        if (strpos($vowels, $word[$i]) !== false) {

            // Keep double/triple vowels together
            if ($i + 1 < $length && strpos($vowels, $word[$i + 1]) !== false) {
                continue;
            }

            // Pull Rule: check next consonant unit
            $nextUnitSize = getConsonantUnitSize($word, $i + 1);
            if ($nextUnitSize > 0 && ($i + 1) < $length) {
                $posAfterUnit = $i + 1 + $nextUnitSize;

                $nextChar = ($i + 1 < $length) ? strtolower($word[$i + 1]) : '';
                $afterChar = ($posAfterUnit < $length) ? strtolower($word[$posAfterUnit]) : '';

                // Check conditions
                $isDouble = ($nextChar !== '' && $nextChar === $afterChar);
                $nextIsConsonant = ($posAfterUnit >= $length || strpos($vowels, $word[$posAfterUnit]) === false);

                if ($nextIsConsonant || $isDouble) {
                    $currentSegment .= substr($word, $i + 1, $nextUnitSize);
                    $i += $nextUnitSize;
                }
            }

            $segments[] = $currentSegment;
            $currentSegment = "";
        }
    }

    // Attach leftovers to the last segment
    if ($currentSegment !== "") {
        if (!empty($segments)) {
            $segments[count($segments) - 1] .= $currentSegment;
        } else {
            $segments[] = $currentSegment;
        }
    }

    return $segments;
}


// Main processing
$filename = "words.txt";

if (!file_exists($filename)) {
    fwrite(STDERR, "Could not open words.txt\n");
    exit(1);
}

$handle = fopen($filename, "r");

// Regex to separate words from everything else
$pattern = '/([a-zA-Z]+|[^a-zA-Z]+)/';

while (($line = fgets($handle)) !== false) {
    $processedLine = "";

    preg_match_all($pattern, $line, $matches);

    foreach ($matches[0] as $part) {
        if (preg_match('/^[a-zA-Z]+$/', $part)) {
            // Split the word and join with 3 spaces
            $segments = splitWordToSegments($part);
            $processedLine .= implode("   ", $segments);
        } else {
            // Keep spaces and punctuation as-is
            $processedLine .= $part;
        }
    }

    echo $processedLine . "\n";
}

fclose($handle);