Stage all text-encoded knowledge base files in a directory for RAG ingestion

rag-build-knowledge.bash

#!/usr/bin/env bash

set -euo pipefail

KNOW_SRC="$HOME/Sites/netfoundry/github"
: "${TMPDIR:=$(mktemp -d)}"
cd "$TMPDIR"
KNOW_DST="$TMPDIR/knowledge"

mkdir -p "$KNOW_DST"

find "$KNOW_SRC" \
    \( \
        -type d \( \
            -name ".npm" -o \
            -name "build" -o \
            -name ".cache" -o \
            -name ".git" -o \
            -name ".github" -o \
            -name ".docusaurus" -o \
            -name "node_modules" -o \
            -name "_deps" -o \
            -name "_remotes" \
        \) -o \
        -name "*.svg" \
    \) -prune -o \
    -type f -size +0c -print0 | \
xargs -0r file --mime-encoding | \
grep -v 'binary$' | \
cut -d: -f1 | \
tee text_files.list

rsync -aiP --stats --files-from=text_files.list \
    --prune-empty-dirs --delete \
    / "$KNOW_DST/"

echo "Text knowledge files have been copied to: $(realpath "$KNOW_DST")"