filter df dataset col for LLM refusals in instruct data

filter_refusals.py

# !pip install -q sentence-splitter
import os
from sentence_splitter import split_text_into_sentences

REFUSAL_TERMS = [
    "sorry",
    "i can't",
    "unfortunately,",
    "as a language model",
    "as an ai language model",
    "i cannot",
]

def not_refusal(example) -> bool:
    first_sentence = split_text_into_sentences(
        example["response"],
        language="en",
    )[0]
    return not any(term in first_sentence.lower() for term in REFUSAL_TERMS)

ds = ds.filter(not_refusal, num_proc=os.cpu_count())
ds