SentenceGenerator.java

package org.acme;

import java.util.Random;

public class SentenceGenerator {
    private static final String[] subjects = {"The cat", "A wizard", "The dragon", "An astronaut", "A pirate"};
    private static final String[] verbs = {"jumps over", "casts a spell on", "breathes fire at", "explores", "steals"};
    private static final String[] objects = {"the moon", "a treasure chest", "a castle", "a spaceship", "a secret island"};

    public static String getSentenceWords(int wordLength) {
        Random random = new Random();
        StringBuilder sentence = new StringBuilder(wordLength);
        for (int i = 0; i < wordLength; i++) { 
            sentence.append(STR."\{subjects[random.nextInt(subjects.length)]} \{verbs[random.nextInt(verbs.length)]} \{objects[random.nextInt(objects.length)]}.");
        }
        return sentence.toString();
    }
}

TokenTest.java

package org.acme;

import com.knuddels.jtokkit.Encodings;
import com.knuddels.jtokkit.api.Encoding;
import com.knuddels.jtokkit.api.EncodingRegistry;
import com.knuddels.jtokkit.api.EncodingType;
import com.knuddels.jtokkit.api.ModelType;
import dev.langchain4j.data.message.ChatMessage;
import dev.langchain4j.data.message.UserMessage;
import dev.langchain4j.model.chat.ChatLanguageModel;
import dev.langchain4j.model.chat.request.ChatRequest;
import dev.langchain4j.model.chat.response.ChatResponse;
import dev.langchain4j.model.openai.OpenAiChatModel;
import dev.langchain4j.model.openai.OpenAiChatModelName;

import java.util.List;

public class TokenTest {


    public static void main(String args[]){

        ChatLanguageModel model = OpenAiChatModel.builder()
                .apiKey(System.getEnv("OPEN_API_KEY"))
                .modelName(OpenAiChatModelName.GPT_4_O_MINI) 
                .maxTokens(10000)
                .logRequests(true)
                .logResponses(true)
                .build();


        List<ChatMessage> messages = List.of(
                new UserMessage(SentenceGenerator.getSentenceWords(1000)),
                new UserMessage(SentenceGenerator.getSentenceWords(9000))
        );

        int totalTokens = messages.stream()
                .mapToInt(msg -> countTokens(msg.toString()))
                .sum();


        System.out.println(STR."Total tokens in conversation: \{totalTokens}");


        ChatRequest cr = ChatRequest.builder().messages(messages).build();
        ChatResponse response = model.chat(cr);

        System.out.println("Tokens used:" + response.tokenUsage());

    }

    public static int countTokens(String text) {

        EncodingRegistry registry = Encodings.newDefaultEncodingRegistry();
        Encoding enc = registry.getEncodingForModel(ModelType.GPT_4O_MINI);

        return enc.countTokens(text);
    }

}