sitemapandrobots.sh

#!/bin/bash
###############################################################
######################### FIND SOME URLS ######################
###############################################################
###fist parameter $1 is the url (i.e. google.com)
###second paramter $2 is the github token
mkdir output/recon/$1
#FILE structure should be
## /output/recon/domain/
#--->
#	->ports.txt
# 	->subdomains.txt
# 	->urls.txt
# the .txt files also include that subdomains ports and urls
# urls.txt also include web resources such as javascript,images,css.
echo "[sitemap 80] ____________________________________" | tee output/recon/$1/urls.txt
/usr/src/app/pythob/content_discovery/sitemap_urls/sitemap-urls.sh http://www.$1/sitemap.xml | tee -a output/recon/$1/urls.txt
echo "[sitemap 443] ____________________________________" | tee -a output/recon/$1/urls.txt
/usr/src/app/pythob/content_discovery/sitemap_urls/sitemap-urls.sh https://www.$1/sitemap.xml | tee -a output/recon/$1/urls.txt
###GEt the content of robosts if exists >> $1
echo "[robots 80] ____________________________________" | tee -a output/recon/$1/urls.txt
curl http://$1/robots.txt --output output/recon/robots.txt
cat output/recon/robots.txt >> output/recon/$1/urls.txt
echo "[robots 443] ____________________________________" | tee -a output/recon/$1/urls.txt
curl -k https://$1/robots.txt --output output/recon/robots.txt
cat output/recon/robots.txt >> output/recon/$1/urls.txt
rm output/recon/robots.txt
##
echo "[waybackurls] ____________________________________" | tee -a output/recon/$1/urls.txt
waybackurls.py $1
cat output/waybackurls/$1-waybackurls.json | tee -a output/recon/$1/urls.txt
echo "[github-endpoints] ____________________________________" | tee -a output/recon/$1/urls.txt
github-endpoints.py -d $1 -t $2 | tee -a output/recon/$1/urls.txt
echo "[paramSpider] ____________________________________" | tee -a output/recon/$1/urls.txt
paramspider.py --domain $1 | tee -a output/recon/$1/urls.txt
echo "[gau] ____________________________________" | tee -a output/recon/$1/urls.txt
gau $1 | tee -a output/recon/$1/urls.txt
echo "[hakrawler] ____________________________________" | tee -a output/recon/$1/urls.txt
echo $1 hakrawler | tee -a output/recon/$1/urls.txt
#TODO: galer and gospider does not work
#echo "[galer] ____________________________________" | tee -a output/recon/$1/urls.txt 
#galer -u "https://$1" | tee -a output/recon/$1/urls.txt
#echo "[gospider] ____________________________________" | tee -a output/recon/$1/urls.txt
#gospider -q -s $1 | tee -a output/recon/$1/urls.txt
###############################################################
######################### CLEAN UP ############################
###############################################################
#filtering repeated urls.
urldedupe -s -u output/recon/$1/urls.txt | tee output/recon/$1/urls.txt

##Removing repeated URls by removing URLS with diff. params. mostly js file. then `sort -u` to remove same entries.
sed 's/?.*//' output/recon/$1/urls.txt | sort -u > output/recon/$1/woParamsUrls.txt
#use woParamsUrls.txt and grep to retrive js files.
cat output/recon/$1/woParamsUrls.txt | grep .js$ > output/recon/$1/jsfiles.txt
##Removing resources (e.g. js,images and css files. first remove using file extension then second regex(sed) removes data uri formated resources )
sed -E -e 's:.*\.(png|svg|jpeg|jpg|js|css|bmp|ico|woff|mp4|webm|pdf).*::' output/recon/$1/urls.txt | sed -E -e 's:.*image\/(png|svg|jpeg|jpg|js|css|bmp|ico|woff|mp4|webm|pdf).*::' | grep "\S" > output/recon/$1/urlswoResources.txt
###############################################################
################# FIND URL fragments in js files ##########################
###############################################################
linkfinder.py -i https://$1 -o cli > output/recon/$1/urlfragments.txt
while read line; do echo "###"$line >> urlfragments.txt;linkfinder.py -i $line -o cli >> output/recon/$1/urlfragments.txt; done < output/recon/$1/jsfiles.txt
###############################################################
################# FIND SECRETS in js and github ###############
###############################################################
echo "[zile] ____________________________________" | tee output/recon/$1/secrets.txt
cat output/recon/$1/jsfiles.txt | python /usr/src/app/pythob/content_discovery/zile/zile.py >> output/recon/$1/secrets.txt
echo "[github-search] ____________________________________" | tee -a output/recon/$1/secrets.txt
github-dorks.py -o $3 -d /usr/src/app/pythob/subdomain_discovery/github-search/dorks.txt -t $2 | tee -a output/recon/$1/secrets.txt
sed 's/debug\://' output/recon/$1/secrets.txt | grep "\S" > output/recon/$1/secrets.txt

###enumerating subdomain
echo 'SUBDOMAIN ENUMERATION'
echo "[amass] ____________________________________" | tee output/recon/$1/subdomains.txt
pathtoamass='/usr/src/app/pythob/subdomain_discovery/amass_linux_amd64'
amass enum -active -d $1 -brute -w $pathtoamass/examples/wordlists/deepmagic.com_top50kprefixes.txt -src -ip -dir $pathtoamass/amass4owasp$1 -config $pathtoamass/examples/config.ini -o output/amass/amass_results_$1.txt
cat output/amass/amass_results_$1.txt | grep -o '\s.*\s' | sed 's: ::g' | tee output/recon/$1/subdomains.txt
echo "[sublert] ____________________________________" | tee -a output/recon/$1/subdomains.txt
sublert.py -u $1
cat output/$1 | tee output/recon/$1/subdomains.txt
echo "[sublist3r] ____________________________________" | tee -a output/recon/$1/subdomains.txt
sublist3r.py -d bbp.ph -o output/sublist34/sub34$1.txt
cat output/sublist34/sub34$1.txt | tee output/recon/$1/subdomains.txt
echo "[github-search] ____________________________________" | tee -a output/recon/$1/subdomains.txt
github-subdomains.py -d $1 -t $2 | tee -a output/recon/$1/subdomains.txt
echo "[crt.sh] ____________________________________" | tee -a output/recon/$1/subdomains.txt
curl -s https://crt.sh/?q=$1  | grep $1  | grep TD | sed 's/<BR>.*//g' | sed 's/<\/TD>.*$//' | sed 's/<TD>//g' | sed 's/\*.//g' | sed 's/ //g' | grep -v 'TDclass' | sort -u > output/crt/$1.txt
cat output/crt/$1.txt | tee output/recon/$1/subdomains.txt
sort -u output/recon/$1/subdomains.txt | tee output/recon/$1/subdomains.txt
cat output/recon/$1/subdomains.txt | grep -o '.*'$1 | tee output/recon/$1/cleansubdomains.txt
mkdir output/recon/$1/screenshots
webscreenshot.py -i output/recon/$1/cleansubdomains.txt -o output/recon/$1/screenshots
#port scan the main domainf and subdomain
#outputting the mani domain
cat $1 | tee -a output/recon/$1/cleansubdomains.txt
echo "[masscan] ____________________________________" | tee output/recon/$1/ports.txt
mkdir output/masscan
dnsmasscan.sh output/recon/$1/cleansubdomains.txt output/masscan/dns.log -p80,443 -oG output/masscan/masscan.log
cat output/masscan/masscan.log | tee -a output/recon/$1/ports.txt
rm output/recon/$1/cleansubdomains.txt