Bonsai 27B

promptfoo.yaml

description: Compare local LLMs in LM Studio

prompts:
  - |
    {{question}}

providers:
  - id: openai:chat:prism-ml/bonsai-27b
    label: Bonsai 27B
    config:
      apiBaseUrl: http://localhost:1234/v1
      apiKey: lm-studio
      temperature: 0
      max_tokens: 2048
      showThinking: false
      tool_choice: auto
      tools: &tools
        - type: function
          function:
            name: get_weather
            description: Get the current weather for a specified city
            parameters:
              type: object
              properties:
                city:
                  type: string
                  description: The city name
                unit:
                  type: string
                  enum:
                    - celsius
                    - fahrenheit
              required:
                - city
                - unit
              additionalProperties: false

  - id: openai:chat:qwen3-14b
    label: Qwen3 14B
    config:
      apiBaseUrl: http://localhost:1234/v1
      apiKey: lm-studio
      temperature: 0
      max_tokens: 2048
      showThinking: false
      tool_choice: auto
      tools: *tools

  # - id: openai:chat:gemma-4-e4b-it-mlx
  #   label: Gemma 4-e4b-it-mlx
  #   config:
  #     apiBaseUrl: http://localhost:1234/v1
  #     apiKey: lm-studio
  #     temperature: 0
  #     max_tokens: 2048
  #     showThinking: false
  #     tool_choice: auto
  #     tools: *tools

  - id: openai:chat:openai/gpt-oss-20b
    label: GPT-OSS 20B
    config:
      apiBaseUrl: http://localhost:1234/v1
      apiKey: lm-studio
      temperature: 0
      max_tokens: 2048
      showThinking: false
      tool_choice: auto
      tools: *tools

  - id: openai:chat:google/gemma-4-e4b
    label: Gemma 4-e4b
    config:
      apiBaseUrl: http://localhost:1234/v1
      apiKey: lm-studio
      temperature: 0
      max_tokens: 2048
      showThinking: false
      tool_choice: auto
      tools: *tools

  - id: openai:chat:qwen/qwen3.5-9b
    label: Qwen3.5 9B
    config:
      apiBaseUrl: http://localhost:1234/v1
      apiKey: lm-studio
      temperature: 0
      max_tokens: 2048
      showThinking: false
      tool_choice: auto
      tools: *tools

  - id: openai:chat:google/gemma-4-12b
    label: Gemma 4-12b
    config:
      apiBaseUrl: http://localhost:1234/v1
      apiKey: lm-studio
      temperature: 0
      max_tokens: 2048
      showThinking: false
      tool_choice: auto
      tools: *tools

defaultTest:
  options:
    provider:
      id: openai:responses:gpt-5-mini
      config:
        apiKey: "{{ env.OPENAI_API_KEY }}"

tests:
  - description: Basic arithmetic reasoning
    vars:
      question: |
        Peter had 17 apples. He gave away one third of them,
        rounding the number down. How many apples does he have left?
        Return only the number.
    assert:
      - type: contains
        value: "12"

  - description: Python coding
    vars:
      question: |
        Write a Python function called gt2(n).
        It must correctly handle integers greater than 2.
        Output only the function definition without any explanation or additional text.
    assert:
      - type: contains
        value: "def gt2"
      - type: contains
        value: "n > 2"

  - description: General knowledge
    vars:
      question: |
        What is the capital of Australia?
        Return only the city name.
    assert:
      - type: javascript
        value: |
          const answer = output.trim().toLowerCase();
          return answer === "canberra";

  - description: Strict JSON output
    vars:
      question: |
        Return exactly this JSON object without Markdown or explanation:
        {"status":"ok","value":42}
    assert:
      - type: is-json
      - type: javascript
        value: |
          const data = JSON.parse(output);
          return data.status === "ok" && data.value === 42;

  - description: Exact bullet count
    vars:
      question: |
        Explain how an electrical power transformer works.
        Use exactly three Markdown bullet points.
        Mention windings, magnetic induction, and voltage.
    assert:
      - type: javascript
        value: |
          const lines = output.split('\n');
          const bullets = lines.filter(line => /^\s*[-*]\s+/.test(line));
          const text = output.toLowerCase();

          return (
            bullets.length === 3 &&
            text.includes("winding") &&
            text.includes("magnetic") &&
            text.includes("voltage")
          );

  - description: Logical reasoning
    vars:
      question: |
        All sensors in system A are digital.
        Sensor X belongs to system A.
        Is sensor X digital?
        Answer only "yes" or "no".
    assert:
      - type: javascript
        value: |
          const answer = output.trim().toLowerCase();
          return answer === "yes";

  - description: Hallucination resistance
    vars:
      question: |
        Provide the publication date of the fictional paper
        "Quantum Bonsai Networks for Lunar Agriculture" by John Smith.
        Do not invent information.
    assert:
      - type: llm-rubric
        value: |
          Pass if the response clearly indicates that the paper is fictional,
          nonexistent, unverifiable, or has no real publication date.
          The response must not provide, imply, or invent any publication date
          or other bibliographic information.
          Different wording is acceptable.

  - description: Technical explanation
    vars:
      question: |
        Explain the difference between concurrency and parallelism
        in no more than 80 words.
    assert:
      - type: javascript
        value: |
          const wordCount = output.trim().split(/\s+/).length;
          return wordCount <= 80;
      - type: llm-rubric
        value: |
          The explanation must correctly distinguish concurrency
          from parallel execution.

  - description: Tool calling
    vars:
      question: |
        What is the current weather in Rome?
        Use the get_weather tool with Celsius units.
        Do not answer from your internal knowledge.
    assert:
      - type: is-valid-openai-tools-call
      - type: javascript
        value: |
          let data;

          try {
            data = typeof output === "string"
              ? JSON.parse(output)
              : output;
          } catch {
            return {
              pass: false,
              score: 0,
              reason: "The output is not a valid JSON tool call."
            };
          }

          const calls =
            data.tool_calls ||
            data.toolCalls ||
            data.message?.tool_calls ||
            [];

          const call = calls.find(
            item => item.function?.name === "get_weather"
          );

          if (!call) {
            return {
              pass: false,
              score: 0,
              reason: "The get_weather tool was not called."
            };
          }

          let args;

          try {
            args = typeof call.function.arguments === "string"
              ? JSON.parse(call.function.arguments)
              : call.function.arguments;
          } catch {
            return {
              pass: false,
              score: 0,
              reason: "The tool arguments are not valid JSON."
            };
          }

          const city = String(args?.city || "").toLowerCase();
          const validCity = city === "rome" || city === "roma";
          const validUnit = args?.unit === "celsius";

          return {
            pass: validCity && validUnit,
            score: validCity && validUnit ? 1 : 0,
            reason: validCity && validUnit
              ? "Correct tool and arguments."
              : `Unexpected arguments: ${JSON.stringify(args)}`
          };

  - description: Deterministic rule-following benchmark
    vars:
      question: |
        Execute the task exactly as specified.

        Rules:

        1. A record is eligible only if:
           - active=true
           - trial=false
           - incidents <= 2

        2. net = floor(seats * price * (100 - discount) / 100)

        3. total = floor(net * (100 + tax) / 100)

           Tax rates:
           EU=20, US=8, UK=15, CA=13.

        4. penalty = incidents * 37 + tierPenalty

           tierPenalty:
           basic=23, pro=11, enterprise=0.

        5. score = total - penalty.

        6. Return ONLY eligible records whose score is between
           400 and 500 inclusive.

        7. Sort:
           - score descending
           - if scores are equal, id ascending.

        8. checksum = sum of all returned scores.

        Return ONLY valid JSON, with no markdown or explanation:

        {
          "items":[
            {"id":"...","score":0}
          ],
          "checksum":0
        }

        Dataset:

        [
          {"id":"A","active":true,"tier":"pro","seats":17,"price":29,"discount":12,"region":"EU","incidents":1,"trial":false},
          {"id":"B","active":true,"tier":"basic","seats":31,"price":11,"discount":0,"region":"US","incidents":0,"trial":false},
          {"id":"C","active":true,"tier":"pro","seats":9,"price":41,"discount":7,"region":"UK","incidents":3,"trial":false},
          {"id":"D","active":false,"tier":"enterprise","seats":40,"price":23,"discount":15,"region":"EU","incidents":0,"trial":false},
          {"id":"E","active":true,"tier":"enterprise","seats":12,"price":53,"discount":18,"region":"US","incidents":2,"trial":false},
          {"id":"F","active":true,"tier":"basic","seats":22,"price":17,"discount":5,"region":"EU","incidents":2,"trial":true},
          {"id":"G","active":true,"tier":"pro","seats":14,"price":37,"discount":9,"region":"CA","incidents":2,"trial":false},
          {"id":"H","active":true,"tier":"enterprise","seats":7,"price":79,"discount":20,"region":"UK","incidents":1,"trial":false}
        ]
    assert:
      - type: is-json
      - type: javascript
        value: |
          const expected = {
            items: [
              { id: "E", score: 488 },
              { id: "A", score: 471 },
              { id: "H", score: 471 },
              { id: "G", score: 447 }
            ],
            checksum: 1877
          };
          const itemToString = ({ id, score }) => JSON.stringify([id, score]);

          try {
            const actual = JSON.parse(output.trim());

            return actual.checksum === expected.checksum &&
              JSON.stringify(actual.items.map(itemToString).sort()) ===
              JSON.stringify(expected.items.map(itemToString).sort());
          } catch {
            return false;
          }