Split Text into Sentences with Maximum Length

split_max_text.py

#!/usr/bin/env python3

#python3 -m pip install nltk
#chmod +x split_max_text.py
#./split_max_text.py "This is a test sentence. This is another test sentence. This is a long test sentence." 20

import nltk
from nltk.tokenize import sent_tokenize
from nltk.data import find
import sys

def get_nltk_punkt():
    try:
        # Check if the 'punkt' package has already been downloaded
        find('tokenizers/punkt')
    except LookupError:
        try:
            # Attempt to download the 'punkt' package
            nltk.download('punkt')
        except Exception as e:
            # Handle the download error
            print(f"Error downloading the 'punkt' package: {e}")
            sys.exit(1)

def split_sentences(text, max_length=512):
    get_nltk_punkt()
    sentences = sent_tokenize(text)
    short_sentences = []
    current_sentence = ""
    for sentence in sentences:
        if len(current_sentence) + len(sentence) + 1 <= max_length:
            if current_sentence:
                current_sentence += " " + sentence
            else:
                current_sentence = sentence
        else:
            if current_sentence:
                short_sentences.append(current_sentence)
            current_sentence = sentence
            while len(current_sentence) > max_length:
                # Find a space or newline near the maximum length to split the sentence
                split_point_space = current_sentence.rfind(' ', 0, max_length)
                split_point_newline = current_sentence.rfind('\n', 0, max_length)
                split_point = max(split_point_space, split_point_newline)
                if split_point == -1:  # Not found, break at the maximum limit
                    split_point = max_length
                short_sentences.append(current_sentence[:split_point])
                current_sentence = current_sentence[split_point:].lstrip()
    if current_sentence:
        short_sentences.append(current_sentence)
    return short_sentences

def main():
    if len(sys.argv) < 2:
        print("Usage: python script.py <text> [max_length]")
        sys.exit(1)
    
    text = sys.argv[1]
    max_length = int(sys.argv[2]) if len(sys.argv) > 2 else 512
    
    sentences = split_sentences(text, max_length)
    for sentence in sentences:
        print(sentence)

if __name__ == "__main__":
    main()