Last active
August 29, 2015 13:56
-
-
Save IndrekV/8819012 to your computer and use it in GitHub Desktop.
Advanced link checker. Takes in a path to a directory, then finds all urls in all the files inside this directory and all it's subdirectories. Then checks if urls are valid. Has nice progressbar.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| if [[ -z "$1" ]] | |
| then | |
| echo "Specify path as argument!" | |
| exit | |
| fi | |
| ERRORS=() | |
| COUNT=1 | |
| function showProgress { | |
| TEXT="" | |
| for i in $(eval echo "{1..$2}"); do | |
| if [[ "$i" -le "$1" ]]; then | |
| TEXT+="|" | |
| else | |
| TEXT+="." | |
| fi | |
| done | |
| echo -ne "Checking links $1/$2 [$TEXT]\r" | |
| } | |
| function showErrors { | |
| if [ ${#ERRORS[@]} -eq 0 ]; then | |
| echo -e "\n/---NO ERRORS FOUND!---/\n" | |
| else | |
| echo -e "\n/---THERE ARE [${#ERRORS[@]}] ERRORS---/\n" | |
| for err in "${ERRORS[@]}"; do | |
| echo -e "$err" | |
| done | |
| fi | |
| } | |
| function parseFiles { | |
| FILES=($(find $1 -type f)) | |
| for f in "${FILES[@]}"; do | |
| if [[ -d $f ]]; then | |
| echo "||| DIRECTORY ||| $f" | |
| parseFiles $f | |
| fi | |
| echo -e "<--\tFile [ $f ]\t-->" | |
| URLS=($(grep -oE '\b(https?|ftp|file)://[-A-Za-z0-9+&@#/%?=~_|!:,.;]*[-A-Za-z0-9+&@#/%=~_|]' $f)) | |
| COUNT=1 | |
| for i in "${URLS[@]}" | |
| do | |
| makeRequest $i $f | |
| done | |
| wait | |
| echo -ne "\n" | |
| done | |
| wait | |
| showErrors | |
| } | |
| function makeRequest { | |
| if [[ $1 == *http* ]] | |
| then | |
| # request without following the redirects | |
| RESPONSE=($(curl -s -w "%{http_code} %{url_effective}\\n" $1 -o /dev/null)) | |
| if [[ ${RESPONSE::1} == "4" ]] | |
| then | |
| echo -ne "\n" | |
| echo -e "$RESPONSE \tURL[ $1 ]" | |
| ERRORS+=("$RESPONSE \n\tURL[ $1 ] \n\tIN [$2] ") | |
| fi | |
| # To follow redirects use | |
| #curl -sL -w "%{http_code} %{url_effective}\\n" $i -o /dev/null | |
| fi | |
| showProgress $COUNT ${#URLS[@]} | |
| COUNT=$((COUNT+1)) | |
| } | |
| parseFiles $1 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment