script to summarize a textfile chunked in openai

openai-chunky-boy.py

import openai

# Set up your OpenAI API credentials
openai.api_key = '<apikey>'

# Function to chunk text into parts of `max_tokens` tokens
def chunk_text(text, max_tokens):
    tokens = text.split()
    chunks = []
    current_chunk = ""

    for token in tokens:
        if len(current_chunk) + len(token) + 1 <= max_tokens:
            current_chunk += token + " "
        else:
            chunks.append(current_chunk.strip())
            current_chunk = token + " "

    if current_chunk:
        chunks.append(current_chunk.strip())

    return chunks

# Function to send each chunk to OpenAI API and append the output to a file
def send_chunks_to_api(chunks, output_file):
    with open(output_file, 'a') as file:
        for chunk in chunks:
            response = openai.Completion.create(
                engine="text-davinci-003",
                prompt=chunk,
                max_tokens=100,
                temperature=0.7
            )

            file.write(response.choices[0].text + "\n")

# Main function to process the text file
def process_text_file(filename, output_file):
    with open(filename, 'r', encoding='utf-8') as file:
        text = file.read()

    chunks = chunk_text(text, 4000)
    send_chunks_to_api(chunks, output_file)

# Example usage
filename = 'openai-input.txt'
output_file = 'output.txt'
process_text_file(filename, output_file)