whey-cewler.py

'''
Based on the initial work of Digininja at https://github.com/digininja/CeWL. While CeWL is a script written
in Ruby that requires an independent crawl of a website in order to build a custom wordlist, Whey CeWLer
runs within Portswigger's Burp Suite and parses an already crawled sitemap to build a custom wordlist. It
does not have the meta data parsing capabilities that CeWL does, but it more than makes up for it in
convenience.

The name gets its origins from the CeWLer portion of the CO2 Burp extension by Jason Gillam, which is written
in Java and does something similar, but Whey CeWLer is a completely reimagined extension written in Python,
making it "way cooler".

Usage:
1. Point Burp Suite to Jython in the Extender > Options tab.
2. Install this extension manually in the Extender > Extensions tab.
3. Select an option for extension output (File, Console or UI).
4. Right-click on any element in the Target tab's hierarchical sitemap.
5. Select the Extensions > Create wordlist context menu item.

The wordlist is created to wherever you have the extension configured for output.
'''

from burp import IBurpExtender
from burp import IContextMenuFactory
from javax.swing import JMenuItem
from java.util import ArrayList, List
from HTMLParser import HTMLParser
from datetime import datetime
import re

COMMON_PASSWORDS = ['password']
TEXT_CONTENT_TYPES = ['text/html', 'application/xml', 'application/json', 'text/plain']

# helpful resource
# https://github.com/laconicwolf/burp-extensions/blob/master/GenerateForcedBrowseWordlist.py

class TagStripper(HTMLParser):
    '''
    Attempts to strip all tags from an HTML page recieved in the HTTP response. The remaining text
    is appended to an array and then joined with " " for regex parsing.
    '''

    def __init__(self):
        HTMLParser.__init__(self)
        self.page_text = []

    def handle_data(self, data):
        self.page_text.append(data)

    def handle_comment(self, data):
        self.page_text.append(data)

    def strip(self, html_page):
        self.feed(html_page)
        return " ".join(self.page_text)


class BurpExtender(IBurpExtender, IContextMenuFactory):
    '''
    BurpExtender Class as per Reference API.
    '''

    def registerExtenderCallbacks(self, callbacks):
        '''
        Registers the extension and initializes the root URLs and word list sets.
        '''
        self._callbacks = callbacks
        self._helpers = callbacks.getHelpers()
        self.context = None
        self.roots = set()
        self.word_list = set(COMMON_PASSWORDS)
        callbacks.setExtensionName("Whey CeWLer")
        callbacks.registerContextMenuFactory(self)
        return

    def createMenuItems(self, context):
        '''
        Invokes the "Create Wordlist" Menu.
        '''
        self.context = context
        if context.getInvocationContext() == context.CONTEXT_TARGET_SITE_MAP_TREE:
            menu_list = ArrayList()
            menu_item = JMenuItem("Create Wordlist", actionPerformed=self.menu_action)
            menu_list.add(menu_item)
            return menu_list
        return

    def menu_action(self, event):
        '''
        Obtains the selected messages from the interface. Filters the sitmap for all messages containing
        URLs within the selected messages' hierarchy. If so, the message is analyzed to create a word list.
        '''
        # get all first-level selected messages and store the URLs as roots to filter the sitemap
        http_messages = self.context.getSelectedMessages()
        for http_message in http_messages:
            root = str(http_message.getUrl())
            self.roots.add(root)
        # get all sitemap entries associated with the selected messages and scrape them for words
        for http_message in self._callbacks.getSiteMap(None):
            url = http_message.getUrl().toString()
            for root in self.roots:
                # will scrape the same URL multiple times if the site map has stored multiple instances
                # the site map stores multiple instances if it detects differences, so this is desirable
                if url.startswith(root):
                    # only scrape if there is a response to scrape
                    http_response = http_message.getResponse()
                    if http_response:
                        self.get_words(url, http_response)
        self.display_words()
        return

    def get_words(self, url, http_response):
        '''
        Checks the header for a text-based content type. If the content type is text-based, uses
        the TagStripper class to parse out the text and runs a regex to create a wordlist based on
        the regex criteria. The resulting words are added to the word_list set.
        '''
        response = self._helpers.analyzeResponse(http_response)
        headers = response.getHeaders()[1:]
        body = self._helpers.bytesToString(http_response[response.getBodyOffset():])
        for header in headers:
            name, value = [x.strip() for x in header.split(':', 1)]
            if name.lower() == 'content-type':
                content_type = value.split(';')[0].strip()
                if content_type.lower() not in TEXT_CONTENT_TYPES:
                    return
        tag_stripper = TagStripper()
        page_text = tag_stripper.strip(body)
        # alpha numerics and apostrophes
        # at least 3 characters in length
        word_candidates = re.findall(r"[\w']{3,}", page_text)
        for word in word_candidates:
            # strip apostrophes
            word = word.replace("'", "")
            # add the word to the list
            self.word_list.add(word)
        return

    def display_words(self):
        '''
        Displays the word list to whatever Burp is configured for stdout.
        '''
        for word in sorted(self.word_list):
            print word
        return