Last active
February 17, 2021 23:17
-
-
Save emadpres/46ef5107c443694fef23e0aea63e3231 to your computer and use it in GitHub Desktop.
Shell scripts related to git repositories: (1) Given a list (of repos): copy/delete, (2) Given a list of repos clone/pull #script #git
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| ############### Author: Emad Aghajani | |
| ############### What does this script do? | |
| ############### - Given a list of repos in form of "owner/repo" at "repos_list", clone them to "clone_to". | |
| ############### - Remove "--depth 1" if you need history of repos. | |
| ############### - Remove "--single-branch" if needed | |
| ############### | |
| ############### Source: https://gist.github.com/emadpres/46ef5107c443694fef23e0aea63e3231 | |
| # Define a timestamp function | |
| timestamp() { | |
| date +"%D-%T" | |
| } | |
| repos_list="repos.txt" | |
| clone_to="repos" | |
| nTotal=0 | |
| nSuccess=0 | |
| nFail=0 | |
| log_output="${PWD}/clone.log" | |
| rm -f $log_output | |
| mkdir -p $clone_to | |
| while IFS= read -r owner_repo ############### For each item in "repos_list" | |
| do | |
| nTotal=$((nTotal+1)) | |
| ############### Break each item to "owner" and "repo" | |
| owner=`dirname "$owner_repo"` | |
| repo=`basename "$owner_repo"` | |
| echo -e "$(timestamp)\t$nTotal $owner/$repo" >> "$log_output" | |
| mkdir -p "$clone_to/$owner" | |
| pushd "$clone_to/$owner" > /dev/null | |
| if [ -d $repo ]; then ############### if repo already exists, delete it | |
| echo -e "$(timestamp)\tRemoving old $repo at $PWD" >> "$log_output" | |
| rm -rf $repo | |
| fi | |
| ############### Clone | |
| # GIT_TERMINAL_PROMPT=0: will not prompt for authentication. See https://git-scm.com/docs/git#GIT_TERMINAL_PROMPT | |
| # Below line is correct and doen't need && before "git clone ..." | |
| GIT_TERMINAL_PROMPT=0 git clone --depth 1 --single-branch --quiet "https://github.com/$owner/$repo" 1>>"$log_output" 2>&1 | |
| # only primary branch: --single-branch | |
| # with submodules: --recurse-submodules --shallow-submodules | |
| ############### Check clone result | |
| res=$? | |
| if [ $res -eq 0 ]; then | |
| echo -e "$(timestamp)\t$nTotal $owner/$repo: [SUCCESS]" | |
| echo -e "$(timestamp)\tCloned successfully" >> "$log_output" | |
| nSuccess=$((nSuccess+1)) | |
| else | |
| echo -e "$(timestamp)\t$nTotal $owner/$repo: [FAIL] - $res" | |
| echo -e "$(timestamp)\tCloned failed - return code: $res" >> "$log_output" | |
| nFail=$((nFail+1)) | |
| if [ -z "$(ls -A -- ./)" ]; then # if cur dir (=owner) is empty | |
| ## If from this "owner" no other repo cloned before, delete empty 'owner" dir | |
| cd .. | |
| rm -rf "$owner" | |
| fi | |
| fi | |
| popd > /dev/null | |
| echo "" >> "$log_output" | |
| done < "${repos_list}" | |
| echo "------------------------------------------" | |
| echo "#Repo Total : $nTotal" | |
| echo "#Repo Clone Successful: $nSuccess" | |
| echo "#Repo Clone Failed: $nFail" | |
| echo "------------------------------------------$(timestamp)" |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| ############### Author: Emad Aghajani | |
| ############### What does this script do? | |
| ############### Given a list of files (their relative path to current directory) in "input_list", copy them from | |
| ############### "./source_dir" to a "./dest_dir". | |
| ############### You can delete copied file using `deleteList.sh` script. | |
| ############### | |
| ############### Source: https://gist.github.com/emadpres/46ef5107c443694fef23e0aea63e3231 | |
| input_list="matchingRepos.txt" | |
| source_dir="ROOT" | |
| dest_dir="920Repos" | |
| mkdir -p $dest_dir | |
| while IFS= read -r file | |
| do | |
| owner=`dirname "$file"` | |
| repo=`basename "$file"` | |
| mkdir -p "$dest_dir/$owner" | |
| cp -rp "$source_dir/$owner/$repo" "$dest_dir/$owner" # -p: preserve owners and permissions | |
| done < "${input_list}" |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| ############### Author: Emad Aghajani | |
| ############### What does this script do? | |
| ############### - Sample code below, count total Java ELOC for each repo in "repos/*/*" | |
| ############### Dependencies | |
| ############### - ~/bin/cloc (https://github.com/AlDanial/cloc) | |
| ############### | |
| ############### Source: https://gist.github.com/emadpres/46ef5107c443694fef23e0aea63e3231 | |
| n_repo=$(echo repos/*/*/ | wc -w) | |
| echo N_REPO: $n_repo | |
| counter=1 | |
| out='eloc.txt' | |
| echo "" > $out | |
| for repo in repos/*/*/; do | |
| echo "#${counter}: $repo" | |
| eloc=$(~/bin/cloc --timeout 0 --include-lang=Java --csv --sum-one $repo | tail -1 | awk -F ',' '{print $5}') | |
| echo "${repo:6: -1},$eloc" >> $out | |
| counter=$(($counter+1)) | |
| done | |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| ############### Author: Emad Aghajani | |
| ############### What does this script do? | |
| ############### Given a list of files (their relative path to current directory) in "input_list", delete them from "./target_dir". | |
| ############### This script usually come in handy after `copyList.sh` script. | |
| ############### | |
| ############### Source: https://gist.github.com/emadpres/46ef5107c443694fef23e0aea63e3231 | |
| input_list="matchingRepos.txt" | |
| target_dir="ROOT" | |
| while IFS= read -r file | |
| do | |
| rm -rf "$target_dir/$file" #-f: force it + ignore file does not exist | |
| done < "${input_list}" |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| ############### Author: Emad Aghajani | |
| ############### What does this script do? | |
| ############### Run "git pull" on all subdirectories of `target_dir_name` | |
| ############### Assumption: Script is being run on a directory containing target_dir_name/owner/repo/[GIT PROJECT] subfolders. | |
| ############### | |
| ############### Note: | |
| ############### The script doesn't work if repos are at target_dir_name/owner_repo | |
| ############### | |
| ############### Source: https://gist.github.com/emadpres/46ef5107c443694fef23e0aea63e3231 | |
| # Define a timestamp function | |
| timestamp() { | |
| date +"%D-%T" | |
| } | |
| target_dir="920Repos" | |
| nRepo_total=0 | |
| nRepo_success=0 | |
| nRepo_notgit=0 | |
| nRepo_fail=0 | |
| ########### Path to output logs | |
| log_output="${PWD}/pull_out.log" | |
| log_error="${PWD}/pull_error.log" | |
| rm -f $log_output | |
| rm -f $log_error | |
| cd "$target_dir" | |
| ########### Loop over "owners" directories | |
| for owner in */ ; do | |
| ########### enter owner directory | |
| #echo "$owner" | |
| cd "$owner" | |
| ########### Loop over repositories inside current owener directory | |
| for repo in */ ; do | |
| ########### enter repo directory | |
| cd "$repo" | |
| nRepo_total=$((nRepo_total+1)) | |
| echo -e "$(timestamp)\t$owner/$repo" >> "$log_output" | |
| echo -e "$(timestamp)\t$owner/$repo" >> "$log_error" | |
| if [ -d .git ]; then | |
| ########### Perform "git pull" and check result | |
| git pull 1>>"$log_output" 2>>"$log_error" | |
| res=$? | |
| if [ $res -eq 0 ]; then | |
| echo -e "$(timestamp)\t$owner/$repo: [SUCCESS]" | |
| echo -e "$(timestamp)\t$owner/$repo: [SUCCESS]" >> "$log_output" | |
| nRepo_success=$((nRepo_success+1)) | |
| else | |
| echo -e "$(timestamp)\t$owner/$repo: [FAIL] - $res" | |
| echo -e "$(timestamp)\t$owner/$repo: [FAIL] - $res" >> "$log_output" | |
| nRepo_fail=$((nRepo_fail+1)) | |
| fi | |
| else | |
| echo -e "$(timestamp)\t$owner/$repo: [SKIP] - not git" | |
| echo -e "$(timestamp)\t[SKIP] - no .git " >> "$log_output" | |
| nRepo_notgit=$((nRepo_notgit+1)) | |
| fi | |
| echo "" >> "$log_output" | |
| echo "" >> "$log_error" | |
| cd .. # exit repo directory | |
| done | |
| cd .. # exit owner directory | |
| done | |
| cd .. # exit target_dir | |
| echo "------------------------------------------" | |
| echo "#Repo Total : $nRepo_total" | |
| echo "#Repo Successful: $nRepo_success" | |
| echo "#Repo Failed: $nRepo_fail" | |
| echo "#Repo Skipped (not git): $nRepo_notgit" | |
| echo "------------------------------------------$(timestamp)" |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Gist Title |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| ############### Author: Emad Aghajani | |
| ############### What does this script do? | |
| ############### Run "git pull" or "git fetch --unshallow" on all subdirectories of `target_dir_name` | |
| ############### Assumption: Script is being run on a directory containing target_dir_name/owner/repo/[GIT PROJECT] subfolders. | |
| ############### | |
| ############### Note: | |
| ############### The script doesn't work if repos are at target_dir_name/owner_repo | |
| ############### | |
| # Define a timestamp function | |
| timestamp() { | |
| date +"%D-%T" | |
| } | |
| target_dir=$1 | |
| char_range_from=$2 | |
| char_range_to=$3 | |
| if [ -z $target_dir ] || [ ! -d $target_dir ]; then | |
| echo "No target dir entered!" | |
| exit 1 | |
| fi | |
| nRepo_total=0 | |
| nRepo_success=0 | |
| nRepo_notgit=0 | |
| nRepo_fail=0 | |
| ########### Path to output logs | |
| log_output="${PWD}/pull_out.log" | |
| log_error="${PWD}/pull_error.log" | |
| rm -f $log_output | |
| rm -f $log_error | |
| cd "$target_dir" | |
| owner_loop_text="*/" | |
| if [ ! -z $char_range_from ] && [ ! -z $char_range_to ]; then | |
| owner_loop_text="[${char_range_from}-${char_range_to}]*/" | |
| fi | |
| echo "---------------------------" | |
| echo "Searching: $owner_loop_text (use A-z for full coverage)" | |
| echo "---------------------------" | |
| # Check target directory (with character range filter) is empty or not | |
| if [ -z "$(ls -A $owner_loop_text)" ]; then | |
| echo "Not directory matched!" | |
| exit 0 | |
| fi | |
| ########### Loop over "owners" directories | |
| for owner in $owner_loop_text ; do | |
| ########### enter owner directory if not empty | |
| if [ -z "$(ls -A ./$owner)" ]; then | |
| #echo "Empty owner dir!" | |
| continue | |
| fi | |
| cd "$owner" | |
| ########### Loop over repositories inside current owener directory | |
| for repo in */ ; do | |
| ########### enter repo directory | |
| cd "$repo" | |
| nRepo_total=$((nRepo_total+1)) | |
| echo -e "$(timestamp)\t$owner/$repo" >> "$log_output" | |
| echo -e "$(timestamp)\t$owner/$repo" >> "$log_error" | |
| if [ -d .git ]; then | |
| ########### Perform "git pull" and check result | |
| #GIT_TERMINAL_PROMPT=0 git pull 1>>"$log_output" 2>>"$log_error" | |
| GIT_TERMINAL_PROMPT=0 git fetch --unshallow 1>>"$log_output" 2>>"$log_error" | |
| ## If unshallow on full repo => "fatal: --unshallow on a complete repository does not make sense" | |
| res=$? | |
| if [ $res -eq 0 ]; then | |
| echo -e "$(timestamp)\t$owner/$repo: [SUCCESS]" | |
| echo -e "$(timestamp)\t$owner/$repo: [SUCCESS]" >> "$log_output" | |
| nRepo_success=$((nRepo_success+1)) | |
| else | |
| echo -e "$(timestamp)\t$owner/$repo: [FAIL] - $res" | |
| echo -e "$(timestamp)\t$owner/$repo: [FAIL] - $res" >> "$log_output" | |
| nRepo_fail=$((nRepo_fail+1)) | |
| fi | |
| else | |
| echo -e "$(timestamp)\t$owner/$repo: [SKIP] - not git" | |
| echo -e "$(timestamp)\t[SKIP] - no .git " >> "$log_output" | |
| nRepo_notgit=$((nRepo_notgit+1)) | |
| fi | |
| echo "" >> "$log_output" | |
| echo "" >> "$log_error" | |
| cd .. # exit repo directory | |
| done | |
| cd .. # exit owner directory | |
| done | |
| cd .. # exit target_dir | |
| echo "------------------------------------------" | |
| echo "#Repo Total : $nRepo_total" | |
| echo "#Repo Successful: $nRepo_success" | |
| echo "#Repo Failed: $nRepo_fail" | |
| echo "#Repo Skipped (not git): $nRepo_notgit" | |
| echo "------------------------------------------$(timestamp)" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment