Brief mitmdump module for dumping content of responses to a file, and keeping track of what files are which URIs in a map JSON.

contentdump.py

"""
Mitmdump extension script that can be used to dump content to disk in a way that can be processed after the fact by arbitrary tools.
"""

import time
import uuid
import json
import os.path

from mitmproxy import http


class ContentToFlatFile(object):
    def __init__(self, data_dir="content"):
        self.last_dump = 0
        self.dump_interval = 10
        self.request_mapping = dict()
        self.data_dir = data_dir

        if not os.path.exists(self.data_dir):
            os.mkdir(self.data_dir)
            with open(os.path.join(self.data_dir, ".map.json"), "w") as fp:
                fp.write(json.dumps(self.request_mapping))
        else:
            if not os.path.isdir(self.data_dir):
                raise Exception(
                    "Target output location exists, but is not a directory")

        if os.path.exists(os.path.join(self.data_dir, ".map.json")):
            with open(os.path.join(self.data_dir, ".map.json"), "r") as fp:
                self.request_mapping = json.loads(fp.read())

    def response(self, flow: http.HTTPFlow) -> None:
        uri = flow.request.data.path.decode("utf-8")
        sni_host = flow.client_conn.sni
        content_bytes = flow.response.content
        request_uuid = str(uuid.uuid4())
        self.request_mapping[request_uuid] = {"sni_host": sni_host, "uri": uri}
        with open(os.path.join(self.data_dir, request_uuid), "wb") as fp:
            fp.write(content_bytes)

        if time.time() - self.last_dump > self.dump_interval:
            self.last_dump = time.time()
            with open(os.path.join(self.data_dir, ".map.json"), "w") as fp:
                fp.write(json.dumps(self.request_mapping))

    def request(self, flow: http.HTTPFlow) -> None:
        # request_content = flow.request.content
        pass


addons = [ContentToFlatFile()]