Skip to content

Instantly share code, notes, and snippets.

View toranb's full-sized avatar

Toran Billups toranb

354 followers · 15 following
View GitHub Profile
@toranb
toranb / turbo.elixir
Created November 9, 2024 19:11
elixir and bumblebee with turbo v3 large
Mix.install([
{:axon, "~> 0.7"},
{:bumblebee, "~> 0.6"},
{:exla, "~> 0.9"},
{:nx, "~> 0.9"},
{:mp3_duration, "~> 0.1.0"},
{:plug_cowboy, "~> 2.6"},
{:jason, "~> 1.4"},
{:phoenix, "~> 1.7.14"},
{:phoenix_html, "~> 4.1"},
@toranb
toranb / Nx_Decision_Trees.livemd
Created October 1, 2024 15:41 — forked from acalejos/Nx_Decision_Trees.livemd
Serving Spam Detection With XGBoost and Elixir

Nx-Powered Decision Trees

Mix.install(
  [
    {:exgboost, "~> 0.3.1", override: true},
    {:nx, "~> 0.6"},
    {:exla, "~> 0.5"},
@toranb
toranb / llama_three.ex
Created April 20, 2024 15:17
bumblebee hack to run llama 3
def llama() do
llama = {:hf, "meta-llama/Meta-Llama-3-8B-Instruct", auth_token: "abc123"}
{:ok, model_info} = Bumblebee.load_model(llama, type: :bf16, backend: {EXLA.Backend, client: :cuda})
{:ok, tokenizer} = Bumblebee.load_tokenizer(llama)
{:ok, generation_config} = Bumblebee.load_generation_config(llama)
tokenizer =
tokenizer
|> Map.put(:special_tokens, %{
pad: "<|eot_id|>",
@toranb
toranb / gemma.exs
Last active March 25, 2024 13:16
Single file elixir app to chat with Gemma 7B using the RTX 4090
Mix.install([
{:bumblebee, git: "https://github.com/toranb/bumblebee", branch: "main"},
{:nx, "~> 0.7", override: true},
{:exla, ">= 0.0.0"},
{:plug_cowboy, "~> 2.6"},
{:jason, "~> 1.4"},
{:bandit, "~> 1.2"},
{:phoenix, "~> 1.7"},
{:phoenix_live_view, "~> 0.20.0"}
])
@toranb
toranb / example.exs
Created November 17, 2023 18:07
Single file elixir web app with self hosted Zephyr LLM
Mix.install([
{:bumblebee, git: "https://github.com/toranb/bumblebee", branch: "main"},
{:nx, "~> 0.6.2", override: true},
{:exla, "~> 0.6.1"},
{:plug_cowboy, "~> 2.6"},
{:jason, "~> 1.4"},
{:phoenix, "~> 1.7"},
{:phoenix_live_view, "~> 0.20.0"}
])
@toranb
toranb / Dockerfile
Created November 3, 2023 11:14
Dockerfile with OTP 26 and Elixir 1.15
ARG ELIXIR_VERSION=1.15.6
ARG OTP_VERSION=26.1.2
ARG DEBIAN_VERSION=buster-20230612-slim
ARG BUILDER_IMAGE="hexpm/elixir:${ELIXIR_VERSION}-erlang-${OTP_VERSION}-debian-${DEBIAN_VERSION}"
ARG RUNNER_IMAGE="debian:${DEBIAN_VERSION}"
FROM ${BUILDER_IMAGE} as builder
# install build dependencies
@toranb
toranb / application.ex
Last active October 21, 2023 14:34
Zephyr 7B with bumblebee when PR 264 lands
def serving() do
mistral = {:hf, "HuggingFaceH4/zephyr-7b-alpha"}
{:ok, spec} = Bumblebee.load_spec(mistral, module: Bumblebee.Text.Mistral, architecture: :for_causal_language_modeling)
{:ok, model_info} = Bumblebee.load_model(mistral, spec: spec, backend: {EXLA.Backend, client: :host})
{:ok, tokenizer} = Bumblebee.load_tokenizer(mistral, module: Bumblebee.Text.LlamaTokenizer)
{:ok, generation_config} = Bumblebee.load_generation_config(mistral, spec_module: Bumblebee.Text.Mistral)
generation_config = Bumblebee.configure(generation_config, max_new_tokens: 500)
Bumblebee.Text.generation(model_info, tokenizer, generation_config, defn_options: [compiler: EXLA])
@toranb
toranb / bert_medium_training.ex
Last active November 14, 2023 15:56
The bumblebee fine tuning example with one of the smaller Pytorch pre-trained BERT variants
defmodule Training.Example do
def train() do
Nx.default_backend(EXLA.Backend)
{:ok, spec} =
Bumblebee.load_spec({:hf, "prajjwal1/bert-medium"},
module: Bumblebee.Text.Bert,
architecture: :for_sequence_classification
)
@toranb
toranb / install_asdf.linux
Last active April 30, 2023 13:30
install asdf with elixir and erlang for ubuntu or popos
sudo apt install git
sudo apt install curl
sudo apt install build-essential
sudo apt install autoconf
sudo apt install m4
sudo apt install libncurses5-dev
sudo apt install libssh-dev
sudo apt install unixodbc-dev
git clone https://github.com/asdf-vm/asdf.git ~/.asdf --branch v0.11.3
@toranb
toranb / first_speech_to_text.exs
Created April 23, 2023 20:50
my first working liveview mp3 upload and transcription example
Application.put_env(:sample, PhoenixDemo.Endpoint,
http: [ip: {127, 0, 0, 1}, port: 8080],
server: true,
live_view: [signing_salt: "bumblebee"],
secret_key_base: String.duplicate("b", 64),
pubsub_server: PhoenixDemo.PubSub
)
Mix.install([
{:plug_cowboy, "~> 2.6"},