lucataco

""" To use: install Ollama, clone OpenVoice, run this script in the OpenVoice directory
    brew install portaudio
    brew install git-lfs
    git lfs install

    git clone https://github.com/myshell-ai/OpenVoice
    cd OpenVoice
    git clone https://huggingface.co/myshell-ai/OpenVoice
    cp -r OpenVoice/* .
    

lucataco

# from cog import BasePredictor, Input, Path
import os
import cv2
import time
import math
import torch
import numpy as np
from PIL import Image
from glob import glob
from typing import Optional

lucataco

**Goal**: Run benchmarks of SDXL, SVD, and Llama 13B on an L40S test node

**TL;DR**:

- L40S has same inference speeds as A40 for SDXL
- L40S has 10% faster inference speeds than A40S for llama2
- L40S are ~9% faster at Video rendering than A40s

**Process**: Run non-docker/cog python code for fp16

lucataco

from diffusers import DiffusionPipeline
import torch
import time

# load both base & refiner
t1 = time.time()
base = DiffusionPipeline.from_pretrained(
    "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16, variant="fp16", use_safetensors=True
)
base.to("cuda")

lucataco

import os
import time
import torch
from typing import Iterator
from threading import Thread
from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer

#Change this to 512, 1024, 2048
MAX_NEW_TOKENS = 512
MAX_INPUT_TOKEN_LENGTH = int(os.getenv("MAX_INPUT_TOKEN_LENGTH", "4096"))

lucataco

import time
import json
import requests


# Start Llama2 13b locally:
# docker run -d -p 5000:5000 --gpus=all r8.im/meta/llama-2-13b@sha256:078d7a002387bd96d93b0302a4c03b3f15824b63104034bfa943c63a8f208c38


url = "http://localhost:5000/predictions"

lucataco

import io
import time
import json
import base64
import requests

# Start SDXL locally:
# docker run -d -p 5000:5000 --gpus=all r8.im/stability-ai/stable-video-diffusion@sha256:3f0457e4619daac51203dedb472816fd4af51f3149fa7a9e0b5ffcf1b8172438

lucataco

import io
import time
import json
import base64
import requests
from PIL import Image
# Start SDXL locally:
# docker run -d -p 5000:5000 --gpus=all r8.im/stability-ai/sdxl@sha256:39ed52f2a78e934b3ba6e2a89f5b1c712de7dfea535525255b1aa35c5565e08b

url = "http://localhost:5000/predictions"

lucataco

from cog import BasePredictor, Input, Path
import os
import torch
import time
from diffusers import (DDIMScheduler, 
    DiffusionPipeline,
    DPMSolverMultistepScheduler,
    EulerAncestralDiscreteScheduler, 
    EulerDiscreteScheduler,
    HeunDiscreteScheduler, 

lucataco

from transformers import AutoTokenizer, AutoModelForCausalLM
import transformers
import torch
import time

model = "tiiuae/falcon-7b"

tokenizer = AutoTokenizer.from_pretrained(model)
pipeline = transformers.pipeline(
    "text-generation",