Model instruction following

instruction-following.md

Result

json_object does not guarantee adherence to schema. Some models do better than others though.

 "schema": {
      "type": "object",
      "properties": {
          "title": {"type": "string"},
          "author": {"type": "string"},
          "publication_year": {"type": "integer"}
      },
      "required": ["title", "author", "publication_year"]
  }

Processing model: openai/o4-mini

  "title": "Echoes of Eternity",
  "author": {
    "firstName": "Lila",
    "lastName": "Morgan"
  },
  "isbn": "978-1-23456-789-7",
  "publicationDate": "2023-08-15",
  "publisher": "Aurora Press",
  "genre": [
    "Fantasy",
    "Adventure"
  ],
  "pages": 432,
  "language": "English",
  "description": "In a world where time weaves itself into the fabric of reality, two unlikely heroes embark on a perilous quest to uncover the secrets of the Eternity Wells.",
  "availableFormats": [
    "Hardcover",
    "Paperback",
    "eBook",
    "Audiobook"
  ],
  "rating": {
    "average": 4.7,
    "reviews": 389
  },
  "chapters": [
    {
      "number": 1,
      "title": "The Shattered Hourglass"
    },
    {
      "number": 2,
      "title": "Whispers in the Wind"
    },
    {
      "number": 3,
      "title": "Beneath the Crystal Spire"
    },
    {
      "number": 4,
      "title": "Echoes of the Past"
    },
    {
      "number": 5,
      "title": "Journey to the End of Time"
    }
  ]
}

Processing model: google/gemini-2.5-flash

  "title": "The Hitchhiker's Guide to the Galaxy",
  "author": "Douglas Adams",
  "publication_year": 1979,
  "genre": "Science Fiction",
  "pages": 193,
  "isbn": "978-0345391803"
}

Processing model: anthropic/claude-3.7-sonnet

jq: parse error: Invalid numeric literal at line 2, column 0

Processing model: cohere/command-a-03-2025

  "title": "The Great Gatsby",
  "author": "F. Scott Fitzgerald",
  "publication_date": "1925",
  "genre": "Fiction",
  "pages": 180,
  "ISBN": "978-0743273565"
}

Command

#!/bin/bash

# Check for API key
if [ -z "$OPENROUTER_API_KEY" ]; then
    echo "🚨 Error: OPENROUTER_API_KEY environment variable is not set." >&2
    exit 1
fi

# Template for the JSON payload (note the dynamic __MODEL_PLACEHOLDER__)
API_DATA_TEMPLATE='{
    "model": "__MODEL_PLACEHOLDER__",
    "messages": [
        {
            "role": "user",
            "content": "Generate a JSON describing a book. Only output the JSON object."
        }
    ],
    "response_format": {
        "type": "json_object",
        "schema": {
            "type": "object",
            "properties": {
                "title": {"type": "string"},
                "author": {"type": "string"},
                "publication_year": {"type": "integer"}
            },
            "required": ["title", "author", "publication_year"]
        }
    }
}'

# Define the function that curl and parses the response
# It accepts the model name as the first argument ($1)
process_model() {
    local MODEL="$1"
    echo "Processing model: $MODEL"

    # 1. Substitute the model name into the data template
    PAYLOAD=$(echo "$API_DATA_TEMPLATE" | sed "s|__MODEL_PLACEHOLDER__|$MODEL|")

    # 2. Execute the API call
    # The output is piped directly through two jq commands:
    # - First jq extracts the raw content string: .choices[0].message.content
    # - Second jq parses that raw string into a formatted JSON object: .
    curl --silent --request POST \
        --url https://openrouter.ai/api/v1/chat/completions \
        --header "Authorization: Bearer $OPENROUTER_API_KEY" \
        --header 'Content-Type: application/json' \
        --data "$PAYLOAD" \
    | jq -r '.choices[0].message.content' \
    | jq -r '.'
}

process_model "openai/o4-mini"
process_model "google/gemini-2.5-flash"
process_model "anthropic/claude-3.7-sonnet"
process_model "cohere/command-a-03-2025"