Skip to content

Instantly share code, notes, and snippets.

@grahama1970
grahama1970 / cleaning_utils.py
Created October 12, 2024 17:58
text_normalizer
import regex as re
from typing import Dict, Optional
import unicodedata
import html
from dateutil.parser import parse as date_parser
from better_profanity import profanity
from bs4 import BeautifulSoup, MarkupResemblesLocatorWarning
import warnings
import emoji
@grahama1970
grahama1970 / arango_client.py
Created October 5, 2024 22:03
test for 2 files
import asyncio
import datetime
import os
import sys
import json
from typing import Optional, Dict, List, Any
from concurrent.futures import ThreadPoolExecutor
from arango import ArangoClient, CollectionCreateError
from arango.exceptions import ArangoError
from loguru import logger
@grahama1970
grahama1970 / arango_db_helper.py
Last active October 12, 2024 17:53
The ArangoDBHelper class provides comprehensive management for an ArangoDB instance, handling initialization, connection, schema retrieval, and collection management. It integrates LLM-based metadata generation, ensuring structured data for collections. The class supports asynchronous database initialization, embedding storage, AQL query execut…
import importlib
import os
import json
import asyncio
import sys
from arango import ArangoClient
from arango.exceptions import ArangoError, CollectionCreateError
import datetime
import logging
from typing import List, Dict, Optional, Any, Union
import os
import requests
import asyncio
import regex as re
from requests.exceptions import RequestException
import json
from bs4 import BeautifulSoup
from pandas import read_html
from uuid import uuid4
import pandas as pd
@grahama1970
grahama1970 / create_pydantic_model_from_schema.py
Created August 9, 2024 23:08
create_pydantic_model_from_schema for dynamic openai structured response
from pydantic import BaseModel, create_model, ValidationError
from typing import Dict, Type, Any, List, Union
import json
from beta.llm_client.helpers.json_cleaner import clean_json_string
def infer_type(value: Any) -> Type:
"""
Infers the type of a given value or type string.
@grahama1970
grahama1970 / test.py
Created August 9, 2024 10:33
test.py
print('hello')
@grahama1970
grahama1970 / chatbot_emotion.py
Last active May 8, 2024 14:38
A chatbot that generates an response based on user emotion
import speech_recognition as sr
import librosa
import numpy as np
from transformers import pipeline, AutoModelForAudioClassification, AutoFeatureExtractor
import openai
import sounddevice as sd
import io
import os
import requests
from dotenv import load_dotenv