incrementscanner.py

#!/usr/bin/env python3
#
# IncrementalScanner version 2.3 (14/04/2023)
# A wrapper for nmap and other Kali enumeration tools
#
# Written by Andy Tyler (@ticarpi)
# Please use responsibly...
# Software URL: https://gist.github.com/ticarpi/b1acf632bde307ba37e5a19af06901c2
# Web: https://www.ticarpi.com
# Twitter: @ticarpi
#
# Feel free to modify nmap commandline options, default ports and other flags 
# by searching the code for 'scan_string' in the functions below.

import argparse
import os
import sys
import subprocess
import re
import ipaddress
import datetime


date_time = datetime.datetime.now()
str_date_time = date_time.strftime("%d-%m-%Y-%H_%M")
alivehosts = []
allports ={}
allportdict ={}

def discovery_scan():
    targetfile = path+"/scans/scan_disco.gnmap"
    scan_string = "nmap -PS21-23,25,53,80,110-111,135,139,143,199,443,445,587,993,995,1025,1720,1723,3306,3389,5900,8080,8888,10000 -PE -PP -PM -PU53,69,111,123,137,161,500,514,520 -sn -T4 -n -oG "+targetfile+" "+target+" -d -vv"+exclusions
    print("\nStarting discovery scan for target: "+target)
    if os.path.isfile(targetfile):
        os.remove(targetfile)
    disco = subprocess.run(['bash','-c',scan_string], capture_output=True)
    if disco.stderr:
        print("[-] ERRORS (discovery scan):\n" + disco.stderr.decode('utf-8'))
    print("[*] ("+re.search("scanned in .*",disco.stdout.decode('utf-8'))[0]+")")
    with open(targetfile, "r", encoding='utf-8', errors='ignore') as discoLst:
        nextHost = discoLst.readline()
        while nextHost:
            if re.search('Up', nextHost):
                alivehosts.append(ipaddress.ip_address(re.search(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}[^0-9]", nextHost)[0].rstrip()))
            nextHost = discoLst.readline()
        if len(alivehosts) > 0:
            print("[+] Found "+str(len(alivehosts))+" alive hosts")
            alivehosts.sort()
            with open(path+"/alivehosts.txt", 'w') as alivehoststxt:
                for host in alivehosts:
            	    alivehoststxt.write(str(host)+"\n")
        else:
            exit("[-] No alive hosts found in range")


def quick_scan(service, port_list=[]):
    ports = ""
    for port in port_list:
        ports = ports+","+str(port)
    ports = ports.lstrip(',')
    targetfile = path+"/scans/scan_"+service+"_alive.gnmap"
    scan_string = "sudo nmap -p"+ports+" --open -oG "+targetfile+" "+target
    print("\nStarting "+service+" quickscan (TCP ports "+ports+") on alive targets")
    if os.path.isfile(targetfile):
        os.remove(targetfile)
    quickscan = subprocess.run(['bash','-c',scan_string], capture_output=True)
    if quickscan.stderr:
        print("[-] ERRORS (quick scan):\n" + quickscan.stderr.decode('utf-8'))
    print("[*] ("+re.search("scanned in .*",quickscan.stdout.decode('utf-8'))[0]+")")
    quickPorts, portDict = parse_scan_gnmap(targetfile, service, port_list)
    if len(quickPorts) > 0 and not args.bare_nmap:
        if service == 'smb':
            smb_post_scan()
        elif service == 'web':
            web_post_scan()
    return quickPorts, portDict

def tcp_script_scan():
    targetfile = path+"/scans/scan_tcp_scripts_alive"
    scan_string = "sudo nmap -sC -sV --open -oA "+targetfile+" "+target
    print("\nStarting script scan (common TCP ports) on alive targets\nNOTE: this may take a while...")
    if os.path.isfile(targetfile):
        os.remove(targetfile)
    tcpscan = subprocess.run(['bash','-c',scan_string], capture_output=True)
    if tcpscan.stderr:
        print("[-] ERRORS (script scan):\n" + tcpscan.stderr.decode('utf-8'))
    print("[*] ("+re.search("scanned in .*",tcpscan.stdout.decode('utf-8'))[0]+")")
    quickPorts, portDict = parse_scan_gnmap(targetfile+".gnmap", "tcp_common_ports", [])
    return quickPorts, portDict

def tcp_allports_scan():
    targetfile = path+"/scans/scan_tcp_allports_alive.gnmap"
    scan_string = "sudo nmap -p- --open -T4 -oG "+targetfile+" "+target
    print("\nStarting all ports scan (all TCP ports) on alive targets\nNOTE: this may take a while...")
    if os.path.isfile(targetfile):
        os.remove(targetfile)
    tcpallscan = subprocess.run(['bash','-c',scan_string], capture_output=True)
    if tcpallscan.stderr:
        print("[-] ERRORS (all ports scan):\n" + tcpallscan.stderr.decode('utf-8'))
    print("[*] ("+re.search("scanned in .*",tcpallscan.stdout.decode('utf-8'))[0]+")")
    quickPorts, portDict = parse_scan_gnmap(targetfile, "tcp_all_ports", [])
    return quickPorts, portDict

def udp_scan():
    targetfile = path+"/scans/scan_udp_alive.gnmap"
    scan_string = "sudo nmap -sU --top-ports 100 --open -oG "+targetfile+" "+target
    print("\nStarting UDP scan (top 100 UDP ports) on alive targets\nNOTE: this may take a while...")
    if os.path.isfile(targetfile):
        os.remove(targetfile)
    udpscan = subprocess.run(['bash','-c',scan_string], capture_output=True)
    if udpscan.stderr:
        print("[-] ERRORS (UDP scan):\n" + udpscan.stderr.decode('utf-8'))
    print("[*] ("+re.search("scanned in .*",udpscan.stdout.decode('utf-8'))[0]+")")
    quickPorts, portDict = parse_scan_gnmap(targetfile, "udp_ports", [])
    return quickPorts, portDict

def parse_scan_gnmap(targetfile, service, port_list):
    with open(targetfile, "r", encoding='utf-8', errors='ignore') as quickLst:
        nextHost = quickLst.readline()
        quickPorts = {}
        portDict = {}
        serviceCount = 0
        while nextHost:
            if re.search("(\d+\/open\/(tcp|udp))", nextHost):
                host = re.search(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}[^0-9]", nextHost)[0].rstrip()
                quickPorts[ipaddress.ip_address(host)] = []
                for port in re.findall('(\d+\/open\/(tcp|udp))',nextHost):
                    quickPorts[ipaddress.ip_address(host)].append(port[0].replace("open/",""))
                    serviceCount += 1
            nextHost = quickLst.readline()
        if len(quickPorts) > 0:
            print("[+] Found "+str(len(quickPorts))+" hosts with open "+service+" ports ("+str(serviceCount)+" ports)") 
            for host in quickPorts:
                for port in quickPorts[host]:
                    try:
                        if portDict[port] and host not in portDict[port]:
                            portDict[port].append(host)
                    except:
                        portDict[port] = [host]
            if len(port_list) != 1:
                with open(path+"/lists/list_"+service+"_host_port.txt", 'w') as quickportstxt:
                    for host in quickPorts:
                        for port in quickPorts[host]:
                            if re.search("\/tcp",port):
                                quickportstxt.write(str(host)+":"+port.rstrip("/tcp")+"\n")
                            elif re.search("\/udp",port):
                                quickportstxt.write(str(host)+":"+port.rstrip("/udp")+"\n")
                with open(path+"/lists/list_"+service+"_port_ordered.txt", 'w') as orderedportstxt:
                    for port in portDict:
                        orderedportstxt.write("\n"+str(port)+":\n")
                        for host in portDict[port]:
                            orderedportstxt.write(str(host)+"\n")
            else:
                with open(path+"/lists/list_"+service+".txt", 'w') as quickportstxt:
                    for host in quickPorts:
            	        quickportstxt.write(str(host)+"\n")
        else:
            print("[-] No open "+service+" ports found in alive range")
    return quickPorts, portDict

def smb_post_scan():
    print("\nGenerating CrackMapExec report for alive SMB hosts")
    cmd_string = "crackmapexec smb "+path+"/lists/list_smb.txt 2>/dev/null | tee "+path+"/report_CME_SMB_alive.txt"
    cmereport = subprocess.Popen(['bash','-c',cmd_string], stdout = subprocess.PIPE, stderr = subprocess.PIPE, text = True)
    print("[+] EXPORTED: "+path+"/report_CME_SMB_alive.txt")
    print("\nGenerating Relay List from alive hosts with SMB Signing not enforced")
    cmd_string = "crackmapexec smb "+path+"/lists/list_smb.txt --gen-relay-list "+path+"/lists/list_SMB_Relay.txt"
    cmerelaylist = subprocess.Popen(['bash','-c',cmd_string], stdout = subprocess.PIPE, stderr = subprocess.PIPE, text = True)
    print("[+] EXPORTED: "+path+"/lists/list_SMB_Relay.txt")
    print("\nChecking RPC on alive hosts for protocols vulnerable to NTLM coercion")
    cmd_string = 'for i in `cat '+path+'/lists/list_smb.txt`; do impacket-rpcdump @$i | grep "MS-RPRN\|MS-PAR\|MS-ICPR\|MS-EFSR" && echo ^Found on $i^;done > '+path+"/lists/list_RPC_Coercible_Protocols.txt"
    rpccoercionlist = subprocess.Popen(['bash','-c',cmd_string], stdout = subprocess.PIPE, stderr = subprocess.PIPE, text = True)
    print("[+] EXPORTED: "+path+"/lists/list_RPC_Coercible_Protocols.txt")

def web_post_scan():
    print("\nBuilding URL list") # open web list, prepend proto, save as urllist
    with open(path+"/lists/list_web_host_port.txt", 'r') as listhostporttxt:
        with open(path+"/lists/list_URLs.txt", 'w') as listurlstxt:
            nextHostPort = listhostporttxt.readline()
            while nextHostPort:
                [host,port] = nextHostPort.split(":")
                if port.rstrip() in ['832', '981', '1311', '7002', '7021', '7023', '7025', '7777', '8333', '8531', '8888']:
                    listurlstxt.write("https://"+host+":"+port)
                else:
                    listurlstxt.write("http://"+host+":"+port)
                nextHostPort = listhostporttxt.readline()
    print("[+] EXPORTED: "+path+"/lists/list_URLs.txt")
    print("\nRunning gowitness (docker) against identified URLs")
    cmd_string = "docker run --rm -v \""+path+"\":/data -p7171:7171 leonjza/gowitness gowitness file -f /data/lists/list_URLs.txt"
    gowitness = subprocess.Popen(['bash','-c',cmd_string], stdout = subprocess.PIPE, stderr = subprocess.PIPE, text = True)
    serve_gowitness = "docker run --rm -v \""+path+"\":/data -p7171:7171 leonjza/gowitness gowitness report serve --address :7171"
    report_gowitness = "docker run --rm -v \""+path+"\":/data -p7171:7171 leonjza/gowitness gowitness report export -f /data/gowitness_report.zip"
    with open(path+"/gowitness_commands.txt","w") as gowitnesscmd:
        gowitnesscmd.write("Serve gowitness (docker) report on http://localhost:7171 by running:\n"+serve_gowitness+"\n\nGenerate report from gowitness (docker) by running:\n"+report_gowitness)
    print("[+] EXPORTED: "+path+"/gowitness_commands.txt")

def script_post_scan():
    print("\nCreating HTML report for script scan (using xsltproc)")
    cmd_string = "xsltproc "+path+"/scans/scan_tcp_scripts_alive.xml --output "+path+"/report_scan_tcp_scripts_alive.html"
    xsltproc = subprocess.Popen(['bash','-c',cmd_string], stdout = subprocess.PIPE, stderr = subprocess.PIPE, text = True)
    print("[+] EXPORTED: "+path+"/report_scan_tcp_scripts_alive.html")

def mergeports(portlists):
    quickPorts, portDict = portlists
    for key in quickPorts.keys():
        for port in quickPorts[key]:
            try:
                if allports[key] and port not in allports[key]:
                    allports[key].append(port)
            except:
                allports[key] = [port]
    for port in portDict:
        for host in portDict[port]:
            try:
                if allportdict[port] and host not in allportdict[port]:
                    allportdict[port].append(host)
            except:
                allportdict[port] = [host]

def csvoutput(targetcsv,currentdict,headers):
    with open(targetcsv, 'w') as targetcsv:
        targetcsv.write(headers+"\n")
        for key in currentdict.keys():
            keyprint = str(key)
            for value in currentdict[key]:
                targetcsv.write(keyprint+","+str(value)+"\n")
                keyprint = "``"

def sort_IPs(templist):
    tmpkeys = []
    sortedtemplist = {}
    for key in templist:
        tmpkeys.append(key)
    tmpkeys.sort()
    for key in tmpkeys:
        tempvalues = []
        temptcp = []
        tempudp = []
        for value in templist[key]:
            if re.search("/tcp",value):
                temptcp.append(int(value.rstrip("/tcp")))
            else:
                tempudp.append(int(value.rstrip("/udp")))
        temptcp.sort()
        tempudp.sort()
        for value in temptcp:
            tempvalues.append(str(value)+"/tcp")
        for value in tempudp:
            tempvalues.append(str(value)+"/udp")
        sortedtemplist[key] = tempvalues
    return sortedtemplist

def sort_ports(templist):
    tmpkeys = []
    temptcp = []
    tempudp = []
    sortedtemplist = {}
    for key in templist:
        if re.search("/tcp",key):
            temptcp.append(int(key.rstrip("/tcp")))
        else:
            tempudp.append(int(key.rstrip("/udp")))
    temptcp.sort()
    tempudp.sort()
    for key in temptcp:
        tmpkeys.append(str(key)+"/tcp")
    for key in tempudp:
        tmpkeys.append(str(key)+"/udp")
    for key in tmpkeys:
        tempvalues = []
        for value in templist[key]:
            tempvalues.append(value)
        tempvalues.sort()
        sortedtemplist[key] = tempvalues
    return sortedtemplist

def export_ports():
    sortedallports = sort_IPs(allports)
    sortedallportdict = sort_ports(allportdict)
    csvoutput(path+"/allports_by_host.csv",sortedallports,"host,port")
    csvoutput(path+"/allports_by_port.csv",sortedallportdict,"port,host")
    print("\n[+] EXPORTED: "+path+"/allports_by_host.csv")
    print("[+] EXPORTED: "+path+"/allports_by_port.csv")


banner = "#### ##    ##  ######  ########  ######## ##     ## ######## ##    ## ######## \n ##  ###   ## ##    ## ##     ## ##       ###   ### ##       ###   ##    ##    \n ##  ####  ## ##       ##     ## ##       #### #### ##       ####  ##    ##    \n ##  ## ## ## ##       ########  ######   ## ### ## ######   ## ## ##    ##    \n ##  ##  #### ##       ##   ##   ##       ##     ## ##       ##  ####    ##    \n ##  ##   ### ##    ## ##    ##  ##       ##     ## ##       ##   ###    ##    \n#### ##    ##  ######  ##     ## ######## ##     ## ######## ##    ##    ##    \n       ######   ######     ###    ##    ## ##    ## ######## ########          \n      ##    ## ##    ##   ## ##   ###   ## ###   ## ##       ##     ##         \n      ##       ##        ##   ##  ####  ## ####  ## ##       ##     ##         \n       ######  ##       ##     ## ## ## ## ## ## ## ######   ########          \n            ## ##       ######### ##  #### ##  #### ##       ##   ##           \n      ##    ## ##    ## ##     ## ##   ### ##   ### ##       ##    ##          \n       ######   ######  ##     ## ##    ## ##    ## ######## ##     ##\n       version 2.3                                            @ticarpi\n\n"


if __name__ == '__main__':
    print(banner)
    parser = argparse.ArgumentParser(epilog="Thanks for scanning.", formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument("-t", "--target", action="store",
                        help="IP list/ranges for the scan")
    parser.add_argument("-n", "--name", action="store",
                        help="name to give the target scan/range (used in filenames)")
    parser.add_argument("-b", "--bare-nmap", action="store_true",
                        help="only run nmap scans and report output (no follow-on enumeration)")
    parser.add_argument("-a", "--allports-tcp", action="store_true",
                        help="scan for all open TCP ports (not just top 1000) - may take a long time")
    parser.add_argument("-e", "--exclusions", action="store",
                        help="IPs to exclude from scans (your own, perchance?)")
    args = parser.parse_args()
    if os.geteuid() != 0:
        exit("You need to have root privileges to run this script.\nPlease try again, this time using 'sudo'. Exiting.")
    try:
        sudouser = os.environ['SUDO_USER']
    except:
        sudouser = os.environ['USER']
    if args.name:
        name = args.name
    else:
        name = "incremscan"
    path = sys.path[0]+"/"+name+"_"+str_date_time
    try:
        if not os.path.exists(path):
            os.makedirs(path)
            os.makedirs(path+"/lists/")
            os.makedirs(path+"/scans/")
    except:
        exit("Permissions or directory issue with specified output paths")
    if args.target:
    	target = args.target
    else:
        exit("No target set.\nSet with -t")
    if args.exclusions:
        exclusions = " --exclude "+args.exclusions
    else:
        exclusions = ""
    discovery_scan()
    target = "-iL "+path+"/alivehosts.txt"
    mergeports(quick_scan("smb", [445]))
    mergeports(quick_scan("kerberos", [88]))
    mergeports(quick_scan("dns", [53]))
    mergeports(quick_scan("db", [1433,1434,3306,1521,1830,5432,8529,7000,7001,9042,5984,9200,9300,27017,27018,27019,28017,7473,7474,6379,8087,8098,28015,29015,7574,8983]))
    mergeports(quick_scan("web", [80,280,81,591,593,2080,2480,3080,4080,4567,5080,5104,5800,6080,7001,7080,7777,8000,8008,8042,8080,8081,8082,8088,8180,8222,8280,8281,8530,8887,9000,9080,9090,16080,443,832,981,1311,7002,7021,7023,7025,8333,8443,8531,8888]))
    mergeports(tcp_script_scan())
    script_post_scan()
    mergeports(udp_scan())
    if args.allports_tcp:
        mergeports(tcp_allports_scan())
    export_ports()
    os.system("chown -R "+sudouser+":"+sudouser+" "+path)
    exit("\nFinished scanning - quitting")