A gist that concatenate the content of a repo into a single txt file for LLM inference

github-repo-content.txt.sh

#!/bin/bash

# Define output file
OUTPUT="big.txt"

# Define excluded directories
EXCLUDED_DIRS=(
    ".git"
    ".venv"
    "__pycache__"
)

# Define excluded file extensions
EXCLUDED_EXTENSIONS=(
    "jpg"
    "jpeg"
    "png"
    "gif"
    "bmp"
    "tiff"
    "webp"
    "svg"
    "ico"
    "heic"
    "raw"
    "cr2"
    "nef"
    "ai"
    "psd"
    "xcf"
)

# Build the find command arguments for directory exclusions
DIR_EXCLUDES=$(printf "! -path \"*/%s/*\" " "${EXCLUDED_DIRS[@]}")

# Build the find command arguments for file extension exclusions
FILE_EXCLUDES=$(printf "! -name \"*.%s\" " "${EXCLUDED_EXTENSIONS[@]}")

# Clear or create the output file
> "$OUTPUT"

# Execute find command with all exclusions
eval "find . -type f \
    $DIR_EXCLUDES \
    $FILE_EXCLUDES \
    ! -name \"$OUTPUT\" \
    -print0" | while IFS= read -r -d $'\0' file; do
    # Add file path as a header
    echo "=== $file ===" >> "$OUTPUT"
    # Add a blank line
    echo >> "$OUTPUT"
    # Add file contents
    cat "$file" >> "$OUTPUT"
    # Add two blank lines as separator
    echo -e "\n\n" >> "$OUTPUT"
done

v2.sh

#!/bin/bash

# Define output file
OUTPUT="big.txt"

# Define excluded directories
EXCLUDED_DIRS=(
    ".git"
    ".venv"
    "__pycache__"
)

# Define excluded file extensions
EXCLUDED_EXTENSIONS=(
    "jpg"
    "jpeg"
    "png"
    "gif"
    "bmp"
    "tiff"
    "webp"
    "svg"
    "ico"
    "heic"
    "raw"
    "cr2"
    "nef"
    "ai"
    "psd"
    "xcf"
)

# Function to check if a file is ignored by git
is_ignored_by_git() {
    local file="$1"
    if command -v git >/dev/null 2>&1; then
        # Check if we're in a git repository
        if git rev-parse --is-inside-work-tree >/dev/null 2>&1; then
            # Use git check-ignore to see if the file is ignored
            if git check-ignore -q "$file"; then
                return 0  # File is ignored
            fi
        fi
    fi
    return 1  # File is not ignored
}

# Build the find command arguments for directory exclusions
DIR_EXCLUDES=$(printf "! -path \"*/%s/*\" " "${EXCLUDED_DIRS[@]}")

# Build the find command arguments for file extension exclusions
FILE_EXCLUDES=$(printf "! -name \"*.%s\" " "${EXCLUDED_EXTENSIONS[@]}")

# Clear or create the output file
> "$OUTPUT"

# Execute find command with all exclusions
eval "find . -type f \
    $DIR_EXCLUDES \
    $FILE_EXCLUDES \
    ! -name \"$OUTPUT\" \
    -print0" | while IFS= read -r -d $'\0' file; do
    # Skip if file is ignored by git
    if ! is_ignored_by_git "$file"; then
        # Add file path as a header
        echo "=== $file ===" >> "$OUTPUT"
        # Add a blank line
        echo >> "$OUTPUT"
        # Add file contents
        cat "$file" >> "$OUTPUT"
        # Add two blank lines as separator
        echo -e "\n\n" >> "$OUTPUT"
    fi
done