grahama1970

from types import SimpleNamespace
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm import completion, acompletion, token_counter
import asyncio
from functools import wraps
from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential
from litellm import RateLimitError, APIError
import os
from dotenv import load_dotenv

grahama1970

services:
  # WORKS: Loads successfully on an A5000 GPU
  sglang_QWEN_32B_Int4:
    image: lmsysorg/sglang:latest
    container_name: sglang_QWEN_32B_Int4
    volumes:
      - ${HOME}/.cache/huggingface:/root/.cache/huggingface
    restart: always
    ports:
      - "30004:30000"  # Adjust port as needed

grahama1970

import asyncio
import os
import runpod
from datetime import datetime, timedelta, timezone
from dotenv import load_dotenv
from loguru import logger
from tenacity import retry, stop_after_attempt, wait_fixed, retry_if_exception_type
from verifaix.llm_client.get_litellm_response import get_litellm_response
from verifaix.arangodb_helper.arango_client import (
    connect_to_arango_client,

grahama1970

import asyncio
from loguru import logger
from verifaix.arangodb_helper.arango_client import connect_to_arango_client

async def truncate_cache_collection(arango_config, db=None):
    logger.info(f"Attempting to truncate cache collection '{arango_config['cache_collection_name']}'")

    if db is None:
        logger.info(f"Connecting to ArangoDB at {arango_config['host']}")
        db = await asyncio.to_thread(connect_to_arango_client, arango_config)

grahama1970

import asyncio
from loguru import logger
from verifaix.arangodb_helper.arango_client import connect_to_arango_client

async def truncate_cache_collection(arango_config, db=None):
    logger.info(f"Attempting to truncate cache collection '{arango_config['cache_collection_name']}'")

    if db is None:
        logger.info(f"Connecting to ArangoDB at {arango_config['host']}")
        db = await asyncio.to_thread(connect_to_arango_client, arango_config)

grahama1970

from pathlib import Path
from dotenv import load_dotenv


def get_project_root(marker_file=".git"):
    """
    Find the project root directory by looking for a marker file.
    
    Args:
        marker_file (str): File/directory to look for (default: ".git")

grahama1970

2024-12-20 09:39:32.640 | INFO     | __main__:run_curl_version:10 - 
=== Running curl version ===
2024-12-20 09:39:32.641 | INFO     | __main__:run_curl_version:43 - Initial request time: 0.00 seconds
2024-12-20 09:39:32.641 | INFO     | __main__:run_curl_version:47 - Response tokens:
To determine the number of rugby players on a touch rugby team, we can refer to the relevant section of the document. 

1. **Understanding Team Composition**: The document states that a team consists of a maximum of 14 players. However, this number includes reserves, meaning that only six (6) players are allowed on the field at any given time during a match.

2. **Player Limitation**: Additionally, teams are encouraged to include mixed genders (four males and four females), indicating that2024-12-20 09:39:43.862 | INFO     | __main__:run_curl_version:68 - 
Tokens generated: 100

grahama1970

import os
import torch
import gc
from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
from peft import PeftModel
from huggingface_hub import snapshot_download
from transformers import (
    AutoModelForCausalLM, 
    AutoTokenizer, 
    pipeline,

grahama1970

# Doesn't Work. Outputs are garbled
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
from peft import PeftModel
from loguru import logger

# Configuration
BASE_MODEL_NAME = "unsloth/Phi-3.5-mini-instruct"
ADAPTER_PATH = "/home/grahama/dev/vllm_lora/training_output/Phi-3.5-mini-instruct_touch-rugby-rules_adapter/final_model"

grahama1970

import os
from loguru import logger
from typing import List, Dict

def load_aql_query(filename: str) -> str:
    """
    Load an AQL query from a file.
    """
    try:
        file_path = os.path.join("app/backend/vllm/beta/utils/aql", filename)