Create a dataset folder with all your github code

Prepare_Code_Dataset.sh

#!/usr/bin/env bash
# Constants
GREEN='\033[0;32m'
RED='\033[0;31m'
YELLOW='\033[0;33m'
NC='\033[0m' # No Color
# ---
CODE_DIRECTORY='code_dataset'

# Get GitHub repo information
response=$(curl -s https://abhinavkm.com/github_repos)
# Replace this with Github API + Token or your own API like I did 😎
parsed_resp=$(echo "$response" | jq '.')

# Count the number of repositories found
repo_count=$(echo "$parsed_resp" | jq '. | length')

# Ask user for confirmation
echo -e "${YELLOW}Found $repo_count repositories under user account. Do you want to continue? (yes/no)${NC}"
read user_input

# Check if the user answered yes or no
if [[ "$user_input" != "yes" ]]; then
    echo -e "${RED}Operation cancelled by user.${NC}"
    exit 1
fi

# Check if the dataset directory exists
if [ -d "$CODE_DIRECTORY" ]; then
    echo -e "${GREEN}Dataset directory already exists.${NC}"
else
    echo -e "${RED}Dataset directory does not exist. Creating a new one${NC}"
    mkdir "$CODE_DIRECTORY"
fi

# Move into the dataset directory
cd "$CODE_DIRECTORY"

# Iterate through each repository in the parsed response
echo "$parsed_resp" | jq -c '.[]' | while read repo; do
    # Extract repo name and URL using jq
    repo_name=$(echo "$repo" | jq -r '.name')
    repo_url=$(echo "$repo" | jq -r '.url')
    
    # Print the repo name in yellow
    echo -e "${YELLOW}Cloning repository: $repo_name${NC}"
    
    # Clone the repository into a directory named after the repo
    git clone --depth=1 "$repo_url" "./$repo_name"
    
    # Remove the .git directory to clean up the repo
    rm -rf "./$repo_name/.git"
    
    # Print confirmation message
    echo -e "${GREEN}Repository $repo_name cloned and cleaned.${NC}"
done