Image Classifier

Gemfile

# frozen_string_literal: true

ruby "3.3.4"

source "https://rubygems.org"

gem "langchainrb"
gem "ruby-openai"
gem "pry-byebug"
gem "dotenv"

Gemfile.lock

GEM
  remote: https://rubygems.org/
  specs:
    addressable (2.8.7)
      public_suffix (>= 2.0.2, < 7.0)
    baran (0.1.12)
    byebug (11.1.3)
    coderay (1.1.3)
    dotenv (3.1.4)
    event_stream_parser (1.0.0)
    faraday (2.12.0)
      faraday-net_http (>= 2.0, < 3.4)
      json
      logger
    faraday-multipart (1.0.4)
      multipart-post (~> 2)
    faraday-net_http (3.3.0)
      net-http
    json (2.7.2)
    json-schema (4.3.1)
      addressable (>= 2.8)
    langchainrb (0.17.0)
      baran (~> 0.1.9)
      json-schema (~> 4)
      matrix
      pragmatic_segmenter (~> 0.3.0)
      zeitwerk (~> 2.5)
    logger (1.6.1)
    matrix (0.4.2)
    method_source (1.1.0)
    multipart-post (2.4.1)
    net-http (0.4.1)
      uri
    pragmatic_segmenter (0.3.24)
    pry (0.14.2)
      coderay (~> 1.1)
      method_source (~> 1.0)
    pry-byebug (3.10.1)
      byebug (~> 11.0)
      pry (>= 0.13, < 0.15)
    public_suffix (6.0.1)
    ruby-openai (7.1.0)
      event_stream_parser (>= 0.3.0, < 2.0.0)
      faraday (>= 1)
      faraday-multipart (>= 1)
    uri (0.13.1)
    zeitwerk (2.6.18)

PLATFORMS
  arm64-darwin-23
  ruby

DEPENDENCIES
  dotenv
  langchainrb
  pry-byebug
  ruby-openai

RUBY VERSION
   ruby 3.3.4p94

BUNDLED WITH
   2.5.17

image-classifier.rb

require "langchain"

class ImageClassifier
  extend Langchain::ToolDefinition

    define_function :classify, description: "Classify photo" do
      property :main_subject, 
        description: "Primary subject of the photo",
        type: "string",
        required: true
      property :scene_type,
        description: "Scene setting in the photo",
        type: "string",
        required: true,
        enum: ["urban", "rural", "indoor", "outdoor"]
      property :colors,
        description: "Dominant colors in the photo",
        type: "array",
        required: true do
          item type: "string"
        end
      property :objects,
        description: "List of key objects visible in the image",
        type: "array",
        required: true do
          item type: "string"
        end
      property :actions,
        description: "Activities or movements captured",
        type: "array",
        required: true do
          item type: "string"
        end
      property :number_of_humans,
        description: "Number of people if present",
        type: "number",
        required: true
      property :weather_conditions,
        description: "Weather conditions",
        type: "string",
        required: true,
        enum: ["sunny", "cloudy", "rainy", "snowy", "foggy"]
      property :time_of_day,
        description: "Time of day",
        type: "string",
        required: true,
        enum: ["morning", "afternoon", "evening", "night"]
      property :mood,
        description: "Overall feeling conveyed by the photo",
        type: "string",
        required: true
  end
end

main.rb

require "langchain"
require "./image-classifier.rb"
require "bundler/setup"
Bundler.require
require "dotenv/load"

#Langchain.logger.level = Logger::ERROR

llm = Langchain::LLM::OpenAI.new(api_key: ENV["OPENAI_API_KEY"])

assistant = Langchain::Assistant.new(
  llm: llm,
  instructions: "You're a helpful AI assistant",
  tools: [ImageClassifier.new],
  tool_choice: "image_classifier__classify"
)

puts "Processing image..."

messages = assistant.add_message_and_run image_url: "https://gist.githubusercontent.com/andreibondarev/b6f444194d0ee7ab7302a4d83184e53e/raw/db2e55ce2d29f3f6a7e3540e5abfd451c6a3d2a8/sf-cable-car.jpg"

puts messages.last.tool_calls

sf-cable-car.jpg