Updating train_gpt2.cu for inference (llm.c)

README.md

• Run python train_gpt2.py
• Copy paste the .cu snippet below right above this line here:

https://github.com/karpathy/llm.c/blob/7ecd8906afe6ed7a2b2cdb731c042f26d525b820/train_gpt2.cu#L1761

• Use https://tiktokenizer.vercel.app/?model=gpt2 to generate tokens
• Copy and paste the values into tokens array. (Keep the -1 at the end - marks the end of the array)
• Run the .sh snippet listed below, linking your model binary

snippet.cu

            // The GitHub project llm.c is a
            int tokens[100] = {464, 21722, 1628, 32660, 76, 13, 66, 318, 257, -1};
            // fill up gen_tokens with the
            int prompt_length = 0;
            for(int i = 0; i < B * T && tokens[i] != -1; ++i) {
                gen_tokens[i] = tokens[i % 10];
                prompt_length = i;
            }
            prompt_length += 1;
            printf("prompt_length: %d\n", prompt_length);

            // Print out gen_tokens
            printf("gen_tokens: ");
            for(int i = 0; i < B * T; ++i) {
                printf("%d ", gen_tokens[i]);
            }
            printf("\n");

            printf("Prompt: ");
            for(int i = 0; i < prompt_length; ++i) {
                const char* token_str = tokenizer_decode(&tokenizer, gen_tokens[i]);
                safe_printf(token_str);
            }
            printf("\n");

            // now sample from the model autoregressively
            printf("generating:\n---\n");
            for (int t = prompt_length; t < genT; t++) {

snippet.sh

set -e

make train_gpt2cu NO_MULTI_GPU=1

./train_gpt2cu \
    -i "dev/data/fineweb10B/fineweb_train_*.bin" \
    -j "dev/data/fineweb10B/fineweb_val_*.bin" \
    # Link to your model here
    -e "log124M/model_00015000.bin" \
    -b 1 -t 1024 \
    -x 1 \
    -l 0.0 \
    -s 1 -g 256 | tee -a log124M/inference.log

https://tiktokenizer.vercel.app/?model=gpt2 to generate the tokens