grahama1970

import speech_recognition as sr
import librosa
import numpy as np
from transformers import pipeline, AutoModelForAudioClassification, AutoFeatureExtractor
import openai
import sounddevice as sd
import io
import os
import requests
from dotenv import load_dotenv

grahama1970

print('hello')

grahama1970

from pydantic import BaseModel, create_model, ValidationError
from typing import Dict, Type, Any, List, Union
import json

from beta.llm_client.helpers.json_cleaner import clean_json_string

def infer_type(value: Any) -> Type:
    """
    Infers the type of a given value or type string.
    

grahama1970

import os
import requests
import asyncio
import regex as re
from requests.exceptions import RequestException
import json
from bs4 import BeautifulSoup
from pandas import read_html
from uuid import uuid4
import pandas as pd

grahama1970

import importlib
import os
import json
import asyncio
import sys
from arango import ArangoClient
from arango.exceptions import ArangoError, CollectionCreateError
import datetime
import logging
from typing import List, Dict, Optional, Any, Union

grahama1970

import asyncio
import datetime
import os
import sys
import json
from typing import Optional, Dict, List, Any
from concurrent.futures import ThreadPoolExecutor
from arango import ArangoClient, CollectionCreateError
from arango.exceptions import ArangoError
from loguru import logger

grahama1970

import regex as re
from typing import Dict, Optional
import unicodedata
import html
from dateutil.parser import parse as date_parser
from better_profanity import profanity
from bs4 import BeautifulSoup, MarkupResemblesLocatorWarning
import warnings

import emoji

grahama1970

Similarity Reranker

The Similarity Reranker is a Python-based module designed to analyze and rank the similarity between documents. It utilizes advanced techniques such as BERT embeddings, BM25 scoring, and Language Model (LLM) refinement to provide a comprehensive similarity analysis. The module is configurable and can handle large document sets efficiently, making it suitable for various use cases like document retrieval, comparison, and clustering.

Key Features

• Multi-stage Similarity Analysis: Combines embeddings, BM25, and LLM scoring to provide refined similarity rankings.
• Dimensionality Reduction: Uses random projection to reduce the dimensionality of BERT embeddings, improving computational efficiency.
• LLM-Based Refinement: Ranks document similarity using LLMs, with configurable models and parameters.

grahama1970

import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm import completion, acompletion, Cache
import asyncio
from functools import wraps
from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential
from litellm import RateLimitError, APIError, ModelResponse
import os
from dotenv import load_dotenv
from loguru import logger

grahama1970

from types import SimpleNamespace
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm import completion, acompletion
import asyncio
from functools import wraps
from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential
from litellm import RateLimitError, APIError
import os
from dotenv import load_dotenv