LiteLLM install with ansible playbook

config.yaml.j2

# https://docs.litellm.ai/docs/proxy/configs
# https://docs.litellm.ai/docs/proxy/quickstart

model_list:
  # https://docs.lambdalabs.com/public-cloud/lambda-inference-api/#listing-models
  - model_name: hermes3-70b
    litellm_params:
      model: openai/hermes3-70b
      api_key: "os.environ/OPENAI_API_KEY"
      api_base: "os.environ/OPENAI_API_BASE"

  - model_name: claude-3.5
    litellm_params:
      # https://docs.litellm.ai/docs/providers/anthropic
      model: anthropic/claude-3-5-sonnet-20240620
      api_key: "os.environ/ANTHROPIC_API_KEY"

  # https://docs.litellm.ai/docs/providers/mistral
  - model_name: mistral-large-latest
    litellm_params:
      model:  mistral/mistral-large-latest
      api_key: "os.environ/MISTRAL_API_KEY"

  # https://docs.litellm.ai/docs/providers/mistral
  - model_name: pixtral-large-latest
    litellm_params:
      model:  mistral/pixtral-large-latest
      api_key: "os.environ/MISTRAL_API_KEY"

  # https://openrouter.ai/cognitivecomputations/dolphin3.0-r1-mistral-24b:free
  - model_name: dolphin3.0-r1-mistral-24b
    litellm_params:
      model:  openrouter/cognitivecomputations/dolphin3.0-r1-mistral-24b:free
      api_key: "os.environ/OPENROUTER_API_KEY"
  
  # https://openrouter.ai/deepseek/deepseek-r1-distill-llama-70b:free
  - model_name: deepseek-r1-distill-llama-70b
    litellm_params:
      model: openrouter/deepseek/deepseek-r1-distill-llama-70b:free
      api_key: "os.environ/OPENROUTER_API_KEY"
  
  # https://docs.litellm.ai/docs/providers/groq
  - model_name: qwen-2.5-coder-32b
    litellm_params:
      model: groq/qwen-2.5-coder-32b
      api_key: "os.environ/GROQ_API_KEY"

litellm_settings:
  request_timeout: 600
  set_verbose: False

playbook.yml

---
- name: Install LiteLLM with Docker
  hosts: all
  become: true
  vars_files:
    - vars/main.yml

  pre_tasks:
    - name: Install system dependencies
      import_tasks: tasks/system_deps.yml

  tasks:
    - name: Install Tailscale
      import_tasks: tasks/tailscale_setup.yml

    - name: Setup Docker
      import_tasks: tasks/docker_setup.yml

    - name: Pull LiteLLM Docker image
      docker_image:
        name: ghcr.io/berriai/litellm
        source: pull
        tag: main-latest
        pull:
          platform: linux/amd64

    - name: Create LiteLLM directory
      file:
        path: "{{ litellm_config_dir }}"
        state: directory
        mode: '0755'

    - name: Create config.yaml for LiteLLM
      template:
        src: templates/config.yaml.j2
        dest: "{{ litellm_config_dir }}/config.yaml"
        mode: '0644'

    # https://www.papertrail.com/solution/tips/logging-in-docker-how-to-get-it-right/
    # https://www.papertrail.com/help/configuring-centralized-logging-from-docker/#logspout
    - name: Run Logspout container
      docker_container:
        name: logspout
        image: gliderlabs/logspout
        network_mode: host
        state: started
        restart_policy: always
        pull: no
        volumes:
          - "/var/run/docker.sock:/var/run/docker.sock"
        command: {{ papertrail_command }}

    - name: Run LiteLLM container
      docker_container:
        name: litellm
        image: ghcr.io/berriai/litellm:main-latest
        network_mode: host
        state: started
        restart_policy: always
        platform: linux/amd64 
        env:
          ANTHROPIC_API_KEY: "{{ anthropic_api_key }}"          
          OLLAMA_BASE_URL: "{{ ollama_base_url }}"
          OPENAI_API_KEY: "{{ openai_api_key }}"
          OPENAI_API_BASE: "{{ openai_api_base }}"
          MISTRAL_API_KEY: "{{ mistral_api_key }}"
          GROQ_API_KEY: "{{ groq_api_key }}"
          OPENROUTER_API_KEY: "{{ openrouter_api_key }}"
        volumes:
          - "{{ litellm_config_dir }}:/data"
        command: "--config /data/config.yaml --num_workers 4 --port 4000"

    - name: Set tailscale serve of litellm
      ansible.builtin.command: tailscale serve --bg --https 443 localhost:4000
      become: true

  handlers:
    - name: reload systemd
      systemd:
        daemon_reload: yes