parseMonsterManual.ts

import * as fs from 'fs';
import { OpenAI } from 'openai';

const openai = new OpenAI({
  apiKey:
    process.env.OPENAI_API_KEY ??
    '',
});

async function extractInfoFromImage(imagePath: string): Promise<any> {
  try {
    const image = fs.readFileSync(imagePath, { encoding: 'base64' });

    const response = await openai.chat.completions.create({
      model: 'gpt-4o',
      messages: [
        {
          role: 'user',
          content: [
            {
              type: 'text',
              text: `You will receive an image with one or more monster character sheets. Focus only on the character sheets. If there are no character sheets, return an empty array.

Abilities (STR, DEX, etc) show up in a table with the following format:

STR {total} {mod} {save}  DEX {total} {mod} {save}  CON {total} {mod} {save}
INT {total} {mod} {save}  WIS {total} {mod} {save}  CHA {total} {mod} {save}

Be precise with these values, DO NOT assume their values nor proficiencies, read them directly from the image. In the past you've made mistakes with the save value specifically, so be extra careful to get the right value from the image, which is the third number after each ability. If you are unsure about the save value, send an empty array, don't guess or assume anything.

Character sheets can optionally have Traits, Actions, Bonus Actions, Legendary Actions, and Reactions. If these are present, include them in the JSON in each property. If they are absent, simply don't include them in the JSON.

Extract each character sheet and return an array using the extract_monsters function.`,
            },
            {
              type: 'image_url',
              image_url: {
                url: `data:image/jpeg;base64,${image}`,
                detail: 'high',
              },
            },
          ],
        },
      ],
      tools: [
        {
          type: 'function',
          function: {
            name: 'extract_monsters',
            description: 'Extracts monster character sheets from an image',
            parameters: {
              type: 'object',
              properties: {
                monsters: {
                  type: 'array',
                  items: {
                    type: 'object',
                    properties: {
                      name: { type: 'string' },
                      type: { type: 'string' },
                      alignment: { type: 'string' },
                      AC: { type: 'string' },
                      HP: { type: 'string' },
                      initiative: { type: 'string' },
                      speed: { type: 'string' },
                      abilities: {
                        type: 'object',
                        properties: {
                          STR: {
                            type: 'object',
                            properties: {
                              total: { type: 'string' },
                              mod: { type: 'string' },
                              save: { type: 'string' },
                            },
                            required: ['total', 'mod', 'save'],
                          },
                          DEX: {
                            type: 'object',
                            properties: {
                              total: { type: 'string' },
                              mod: { type: 'string' },
                              save: { type: 'string' },
                            },
                            required: ['total', 'mod', 'save'],
                          },
                          CON: {
                            type: 'object',
                            properties: {
                              total: { type: 'string' },
                              mod: { type: 'string' },
                              save: { type: 'string' },
                            },
                            required: ['total', 'mod', 'save'],
                          },
                          INT: {
                            type: 'object',
                            properties: {
                              total: { type: 'string' },
                              mod: { type: 'string' },
                              save: { type: 'string' },
                            },
                            required: ['total', 'mod', 'save'],
                          },
                          WIS: {
                            type: 'object',
                            properties: {
                              total: { type: 'string' },
                              mod: { type: 'string' },
                              save: { type: 'string' },
                            },
                            required: ['total', 'mod', 'save'],
                          },
                          CHA: {
                            type: 'object',
                            properties: {
                              total: { type: 'string' },
                              mod: { type: 'string' },
                              save: { type: 'string' },
                            },
                            required: ['total', 'mod', 'save'],
                          },
                        },
                        required: ['STR', 'DEX', 'CON', 'INT', 'WIS', 'CHA'],
                      },
                      skills: { type: 'string' },
                      resistances: { type: 'string' },
                      immunities: { type: 'string' },
                      senses: { type: 'string' },
                      languages: { type: 'string' },
                      CR: { type: 'string' },
                      traits: {
                        type: 'string',
                        description:
                          'A string containing all traits, formatted with HTML if needed',
                      },
                      actions: {
                        type: 'string',
                        description:
                          'A string containing all actions, formatted with HTML if needed',
                      },
                      bonus_actions: {
                        type: 'string',
                        description:
                          'A string containing all bonus actions, formatted with HTML if needed',
                      },
                      legendary_actions: {
                        type: 'string',
                        description:
                          'A string containing all legendary actions, formatted with HTML if needed',
                      },
                      reactions: {
                        type: 'string',
                        description:
                          'A string containing all reactions, formatted with HTML if needed',
                      },
                    },
                    required: [
                      'name',
                      'type',
                      'alignment',
                      'AC',
                      'HP',
                      'initiative',
                      'speed',
                      'abilities',
                      'skills',
                      'senses',
                      'languages',
                      'CR',
                    ],
                  },
                },
              },
              required: ['monsters'],
            },
          },
        },
      ],
      tool_choice: { type: 'function', function: { name: 'extract_monsters' } },
    });

    const monsters = response.choices[0].message.tool_calls?.[0].function.arguments ?? '{}';
    return JSON.parse(monsters).monsters;
  } catch (error) {
    console.error('Error extracting information from image:', error);
    throw error;
  }
}

async function run() {
  try {
    for (let i = 1; i <= 364; i++) {
      const monsters = await extractInfoFromImage(`mm2024/page_${i}.png`);
      console.log('Extracted Information:');
      console.log(JSON.stringify(monsters, null, 2));
      fs.writeFileSync(`monsters/page_${i}.json`, JSON.stringify(monsters, null, 2));
    }
  } catch (error) {
    console.error('Error in run function:', error);
  }
}

run()
  .then(() => console.log('done'))
  .catch((err) => console.log('error', err));