do-me

import re
from bs4 import BeautifulSoup

def remove_consecutive_whitespaces(text):
    # Define the regex pattern
    pattern = r'[^\S\r\n]*(\r\n|\n|\r)[^\S\r\n]*|([^\S\r\n]){2,}'

    # Use re.sub to replace matches with a single whitespace
    result = re.sub(pattern, lambda match: ' ' if match.group(2) else match.group(1), text)

do-me

from lxml import etree
import pyarrow as pa
import pyarrow.parquet as pq
import pandas as pd
from tqdm import tqdm
import os
tqdm.pandas()
import lxml
import cchardet
import re

do-me

# Input: folder chunks/chunk_*
# Output: folder chunks/chunk_*_embeddings
# Deletes trash/cache for each iteration to free disk space

import subprocess

def run_shell_command(command):
    process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
    output, error = process.communicate()
    

do-me

from pysal.viz.mapclassify import NaturalBreaks as nb
import pandas as pd

# import your df here

num_classes = 5
classifier = nb(list(df["score"]), k=num_classes)
classifications = list(classifier.bins)

classifications = [0] + classifications

do-me

import hashlib
# Function to generate hash for each row
def generate_hash(row):
    hash_object = hashlib.sha256()
    # Convert each element in the row to a string and update the hash object
    for value in row.values:
        hash_object.update(str(value).encode('utf-8'))
    # Return the hexadecimal representation of the hash
    return hash_object.hexdigest()[:30]

do-me

import pandas as pd 
import numpy as np

# Useful when you indxed document chunks and now need to create average embeddings per document
df = pd.read_parquet("chunk_embeddings.parquet") # ~3GB file

# Convert the embeddings column to a NumPy array
df['embeddings'] = df['embeddings'].apply(np.array)

# Group by filename and calculate the mean of the embeddings

do-me

import requests
from bs4 import BeautifulSoup
import json

def extract_dataset_info(html):
    soup = BeautifulSoup(html, 'html.parser')

    dataset_info_list = []

    # Find all occurrences of 'h3' and 'p' tags

do-me

html_table_string = """
<table class="table svelte-1jok1de" style="height: 100%; --bw-svt-p-top: 0px; --bw-svt-p-bottom: 3808.18359375px; --bw-svt-head-height: 37px; --bw-svt-foot-height: 0px; --bw-svt-avg-row-height: 36.97265625px;"><thead class="thead svelte-1jok1de"><tr slot="thead" class="svelte-1bvc1p0"><th aria-sort="none" class="svelte-1bvc1p0" style="width: var(--cell-width-0);"><div class="cell-wrap svelte-1bvc1p0"> <span tabindex="-1" role="button" style="" class="svelte-q8uklq">Rank</span> <div class="sort-button undefined svelte-1bvc1p0"><svg width="1em" height="1em" viewBox="0 0 9 7" fill="none" xmlns="http://www.w3.org/2000/svg" class="svelte-1bvc1p0"><path d="M4.49999 0L8.3971 6.75H0.602875L4.49999 0Z"></path></svg></div></div> </th><th aria-sort="none" class="svelte-1bvc1p0" style="width: var(--cell-width-1);"><div class="cell-wrap svelte-1bvc1p0"> <span tabindex="-1" role="button" style="" class="svelte-q8uklq">Model</span> <div class="sort-button undefined svelte-1bvc1p0"><svg width="1em" he

do-me

def separate_street_and_number(address):
    '''
    Split by " " and check for the index of the first element that STARTSWITH a number.
    Everything before that index is the street name, so " ".join()
    Everything after is the house number.

    # K1 | 1-4
    # E1 | 15
    # Bahnhofstr. | 27
    # Marienbrunnen | 10 a

do-me

from collections import Counter
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from tqdm import tqdm 
import pandas as pd
import string

# Download NLTK stopwords
import nltk
nltk.download('stopwords')