gwen001

firefox `cat urls.txt | awk '{if(index($1,"http")){print $1}else{print "http://"$1;print "https://"$1}}'
firefox `cat urls.txt | awk '{if(index($1,"http")){print $1}else{print "http://"$1;print "https://"$1}}' | tr "\n" " "`

chromium-browser `cat urls.txt | awk '{if(index($1,"http")){print $1}else{print "http://"$1;print "https://"$1}}'
chromium-browser `cat urls.txt | awk '{if(index($1,"http")){print $1}else{print "http://"$1;print "https://"$1}}' | tr "\n" " "`

function oopen() {
  firefox `cat $1 | awk '{if(index($1,"http")){print $1}else{print "http://"$1;print "https://"$1}}'
}

gwen001

(ls /tmp/GeoLite2-ASN-CSV.zip||wget --quiet -P /tmp "https://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN-CSV.zip") 2>&1|grep X; unzip -c /tmp/GeoLite2-ASN-CSV.zip|grep -i airbnb|cut -d ',' -f 2|sort -fu

amass intel -org paypal | cut -d ',' -f 1

output:
 
133850
137437
206695
206704

gwen001

cat plainurls.txt | while read u;do echo "$(printf '%-100s' "$u")-> $((curl -I -s -m 5 -k "$u"||echo KO)|head -n 1 -)"; done

From a json file:
cat waybackurls.json|jq -r '.[]'|grep 'http'|cut -d '"' -f 2 | while read u;do echo "$(printf '%-100s' "$u")-> $((curl -I -s -m 5 -k "$u"||echo KO)|head -n 1 -)"; done

Using parallel to speed up the process:
cat plainurls.txt | parallel -j 20 -I# 'echo "$(printf "%-100s" "#") -> $((curl -I -s -m 5 -k "#"||echo KO)|head -n 1 -)"'

function otestu {
  cat $1 | parallel -j 20 -I# 'echo "$(printf "%-100s" "#") -> $((curl -I -s -m 5 -k "#"||echo KO)|head -n 1 -)"'

gwen001

host `echo https://pass:[email protected]:2996/utils/|sed "s/.*:\/\///"|cut -d '/' -f 1|cut -d '@' -f 2|cut -d':' -f 1`

function ohost {
  host `echo $1|sed "s/.*:\/\///"|cut -d '/' -f 1|cut -d '@' -f 2|cut -d':' -f 1`
}

function ohosts {
  while read u; do host `echo $u|sed "s/.*:\/\///"|cut -d '/' -f 1|cut -d '@' -f 2|cut -d':' -f 1`; done < $1
}

gwen001

while read h; do curl -siL https://$h|sed -n -E "s/.*<.*(href|src|url)[=:]['\"]?([^'\">]+).*/\2/p" ; done < hosts.txt

function olink {
  curl -siL https://$1|sed -n -E "s/.*(href|src|url)[=:]['\"]?([^'\">]+).*/\2/p"
}

function olinks {
  while read h; do curl -siL https://$h|sed -n -E "s/.*<.*(href|src|url)[=:]['\"]?([^'\">]+).*/\2/p" ; done < $1
}

gwen001

while read h; do curl -siL https://$h|egrep -io  "[0-9a-z_\-\.]+\.([0-9a-z_\-]+)?`echo $h|awk -F '.' '{print $(NF-1)}'`([0-9a-z_\-\.]+)?\.[a-z]{1,5}"|sort -fu ; done < hosts.txt

function osub {
  curl -siL https://$h1egrep -io "[0-9a-z_\-\.]+\.([0-9a-z_\-]+)?`echo $h|awk -F '.' '{print $(NF-1)}'`([0-9a-z_\-\.]+)?\.[a-z]{1,5}"|sort -fu
}

function osubs {
  while read h; do curl -siL https://$h|egrep -io  "[0-9a-z_\-\.]+\.([0-9a-z_\-]+)?`echo $h|awk -F '.' '{print $(NF-1)}'`([0-9a-z_\-\.]+)?\.[a-z]{1,5}"|sort -fu ; done < $1
}

gwen001

echo "https://www.example.com/?aaa=bbb&ccc=ddd" | tr '?' '&' | awk -F '&' '{for(i=2;i<=NF;i++){split($i,t,"=");print t[1]}}' 

while read u; do echo $u | tr '?' '&' | awk -F '&' '{for(i=2;i<=NF;i++){split($i,t,"=");print t[1]}}'; done < plainurls.txt | sort -fu

From wayback json file:
cat waybackurls.json|jq -r '.[]'|grep 'http'|cut -d '"' -f 2 | while read u; do echo $u | tr '?' '&' | awk -F '&' '{for(i=2;i<=NF;i++){split($i,t,"=");print t[1]}}'; done | sort -fu

function oparam {
  echo $1 | tr '?' '&' | awk -F '&' '{for(i=2;i<=NF;i++){split($i,t,"=");print t[1]}}' 
}

gwen001

curl -L -k -s https://www.example.com | tac | sed "s#\\\/#\/#g" | egrep -o "src['\"]?\s*[=:]\s*['\"]?[^'\"]+.js[^'\"> ]*" | awk -F '//' '{if(length($2))print "https://"$2}' | sort -fu | xargs -I '%' sh -c "curl -k -s \"%\" | sed \"s/[;}\)>]/\n/g\" | grep -Po \"(['\\\"](https?:)?[/]{1,2}[^'\\\"> ]{5,})|(\.(get|post|ajax|load)\s*\(\s*['\\\"](https?:)?[/]{1,2}[^'\\\"> ]{5,})\"" | awk -F "['\"]" '{print $2}' | sort -fu

# using linkfinder
function ejs() {
   URL=$1;
   curl -Lks $URL | tac | sed "s#\\\/#\/#g" | egrep -o "src['\"]?\s*[=:]\s*['\"]?[^'\"]+.js[^'\"> ]*" | sed -r "s/^src['\"]?[=:]['\"]//g" | awk -v url=$URL '{if(length($1)) if($1 ~/^http/) print $1; else if($1 ~/^\/\//) print "https:"$1; else print url"/"$1}' | sort -fu | xargs -I '%' sh -c "echo \"\n##### %\";wget --no-check-certificate --quiet \"%\"; basename \"%\" | xargs -I \"#\" sh -c 'linkfinder.py -o cli -i #'"
}

# with file download (the new best one):
# but there is a bug if you don't provide a root url

gwen001

#using cewl
wordgrab() {
  url=$1
  cewl.rb -u "Mozilla/5.0 (X11; Linux; rv:74.0) Gecko/20100101 Firefox/74.0" -d 0 -m 3 https://www.$1 | tr '[:upper:]' '[:lower:]' |sort -fu | grep -v "robin wood"
}

# added min length 3
wordgrab() {
  url=$1
  tmpfile="$(date "+%s")"

gwen001

function sslsub() {
  timeout 3 openssl s_client -showcerts -servername $1 -connect $1:443 <<< "Q"  2>/dev/null | openssl x509 -text -noout | grep DNS | tr ',' '\n' | cut -d ':' -f 2 | sort -fu
}