fetch_proxy.py

       def __fetchContent_(self, url, method):
                # TODO get actual character encoding from the request           

                #  Extract the request Util.headers from the RequestUtil.headers super-global (this -does- unfortunatly mean that any Util.header that php doesn't understand won't be proxied thru though)
                #  if this turns out to be a problem we could add support for HTTP_RAW_Util.headerS, but this depends on a php.ini setting, so i'd rather prevent that from being required
                headers = '';
                context = GadgetContext(self.request, self.response,'GADGET');
                requestheaders = self.__request_headers_();
                for key in requestheaders.keys():
                        val = requestheaders[key];
                        if (key != 'Keep-alive' and key != 'Connection' and key != 'Host' and key != 'Accept' and key != 'Accept-Encoding'):
                                #  propper curl header format according to http://www.php.net/manual/en/def.curl-setopt.php#80099
                                headers = headers  + key + ": " + val + "\n";

                from lib import RequestUtil;
                requestUtilHandler = RequestUtil.getInstance();         

                if (method == 'POST'):
                        data = requestUtilHandler.GETORPOST['postData'] if requestUtilHandler.GETORPOST.has_key('postData') else False;
                        #if (not data):
                        #       data = isset(requestUtilHandler.GETORPOST['postData']) ? requestUtilHandler.GETORPOST['postData'] : False;
                        
                        postData = '';
                        if (data):
                                data = urllib.unquote(data);
                                entries = data.split('&');
                                for entry in entries:
                                        parts = entry.split('=');
                                        #  Process only if its a valid value=something pair
                                        if (len(parts) == 2):
                                                postData = postData + urllib.quote(parts[0]) + '=' + urllib.quote(parts[1]) + '&';
                                        
                                
                                #  chop of the trailing &
                                if (len(postData)):
                                        postData = Util.substr(postData, 0, len(postData) - 1);
                                
                        
                        #  even if postData is an empty string, it will still post (since RemoteContentRquest checks if its False)
                        #  so the request to POST is still honored

                        from src.common import RemoteContentRequest;
                        request = RemoteContentRequest(url, headers, postData);
                        request = self.__context_.getHttpFetcher().fetch(request, context);
                else:
                        from src.common import RemoteContentRequest;
                        request = RemoteContentRequest(url, headers);
                        request = self.__context_.getHttpFetcher().fetch(request, context);
                
                return request;

RemoteContentRequest.py

class RemoteContentRequest():
    # these are used for making the request
    SC_OK = 200; #Please, use only for testing!
    DEFAULT_CONTENT_TYPE = "application/x-www-form-urlencoded; charset=utf-8";

    def __init__(self,uri, headers = {}, postBody = None):
        self.uri = uri;
        self.headers = headers;
        self.postBody = postBody;
        import time;
        self.created = time.time();
        self.__method = '';
        # these fields are filled in once the request has completed
        self.__responseContent = False;
        self.__responseSize = False;
        self.__responseHeaders = False;
        self.__httpCode = False;
        self.__contentType = None;
        self.__options = '';
        self.handle = False;

    def createRemoteContentRequest(self, method, uri, headers, postBody, options):
        self.__method = method;
        self.uri = uri;
        self.options = options;
        # Copy the headers
        from lib import Util
        if (not Util.is_set(headers)):
            self.headers = '';
        else:
            setPragmaHeader = False;
            tmpHeaders = '';
            for key, value in enumerate(headers):
                # Proxies should be bypassed with the Pragma: no-cache check.
                if (key == "Pragma" and options.ignoreCache):
                    value = "no-cache";
                    setPragmaHeader = True;
                tmpHeaders = tmpHeaders + key + ":" + value + "\n";
            # Bypass caching in proxies as well.
            if (not setPragmaHeader and options.ignoreCache):
                tmpHeaders = tmpHeaders + "Pragma:no-cache\n";
            self.headers = tmpHeaders;
        if (postBody == None):
            self.postBody = None;
        else:
            self.postBody = Util.array_merge(postBody, self.postBody);

        type = self.getHeader("Content-Type");
        if (not Util.is_set(type)):
            self.__contentType = RemoteContentRequest.DEFAULT_CONTENT_TYPE;
        else:
            self.__contentType = type;

    #
    # Creates a new request to a different URL using all request data from
    # an existing request.
    #
    # @param uri
    # @param base The base request to copy data from.
    #
    @staticmethod
    def createRemoteContentRequestWithUriBase(self, uri, base):
        self.uri = uri;
        self.__method = base.method;
        self.options = base.options;
        self.headers = base.headers;
        self.__contentType = base.contentType;
        self.postBody = base.postBody;

    #
    # Basic GET request.
    #
    # @param uri
    #
    def createRemoteContentRequestWithUri(self, uri):
        self.createRemoteContentRequest(self, "GET", uri, None, None, RemoteContentRequest.getDefaultOptions());

    #
    # GET with options
    #
    # @param uri
    # @param options
    #
    def createRemoteContentRequestWithUriOptions(self, uri, options):
        self.createRemoteContentRequest(self, "GET", uri, None, None, options);

    #
    # GET request with custom headers and default options
    # @param uri
    # @param headers
    #
    def RemoteContentRequestWithUriHeaders(self, uri, headers):
        self.createRemoteContentRequest(self,"GET", uri, headers, None, RemoteContentRequest.getDefaultOptions());

    #
    # GET request with custom headers + options
    # @param uri
    # @param headers
    # @param options
    #
    def createRemoteContentRequestWithUriHeadersOptions(self, uri, headers, options):
        self.createRemoteContentRequest(self, "GET", uri, headers, None, options);

    #
    # Basic POST request
    # @param uri
    # @param postBody
    #
    def RemoteContentRequestWithUriPostBody(self, uri, postBody):
        self.createRemoteContentRequest(self,"POST", uri, None, postBody, RemoteContentRequest.getDefaultOptions());

    #
    # POST request with options
    # @param uri
    # @param postBody
    # @param options
    #
    def createRemoteContentRequestWithUriPostBodyOptions(self, uri, postBody, options):
        self.createRemoteContentRequest(self, "POST", uri, None, postBody, options);

    #
    # POST request with headers
    # @param uri
    # @param headers
    # @param postBody
    #
    def createRemoteContentRequestWithUriHeadersPostBody(self, uri, headers, postBody):
        self.createRemoteContentRequest(self, "POST", uri, headers, postBody, RemoteContentRequest.getDefaultOptions());

    #
    # POST request with options + headers
    # @param uri
    # @param headers
    # @param postBody
    # @param options
    #
    def createRemoteContentRequestWithUriHeadersPostBodyOptions(self, uri, headers, postBody, options):
        self.createRemoteContentRequest(self,"POST", uri, headers, postBody, options);

    #
    # Creates a simple GET request
    #
    # @param uri
    # @param ignoreCache
    #
    def getRequest(self, uri, ignoreCache):
        
        options = Options();
        options.ignoreCache = ignoreCache;
        return self.createRemoteContentRequestWithUriOptions(self, uri, options);

    #
    # Simple constructor for setting a basic response from a string. Mostly used
    # for testing.
    #
    # @param body
    #
    def getHttpFalseResponseBody(self, body):
        return self.createFalseResponse(self, RemoteContentRequest.SC_OK, body, None);

    def __createFalseResponse_(self, httpCode, body, headers):
        self.__httpCode = httpCode;
        self.__responseContent = body;
        self.headers = headers;
        return self;

    # returns a hash code which identifies this request, used for caching
    # takes url and postbody into account for constructing the sha1 checksum
    def toHash(self):
        import md5
        
        return md5.new(self.uri + self.postBody).hexdigest();

    @staticmethod
    def getDefaultOptions():

        return Options();


    def getContentType(self):

        return self.__contentType;


    def getHttpCode(self):

        return self.__httpCode;


    def getResponseContent(self):

        return self.__responseContent;


    def getResponseHeaders(self):

        return self.__responseHeaders;


    def getResponseSize(self):

        return self.__responseSize;


    def getHeaders(self):

        return self.headers;


    def isPost(self):

        return (self.postBody != None);


    def hasHeaders(self):

        return (len(self.headers) > 0);


    def getPostBody(self):

        return self.postBody;


    def getUrl(self):

        return self.uri;


    def getMethod(self):

        return self.__method;


    def getOptions(self):

        if (len(self.options) == 0):
            import Options;
            return Options();
    
        return self.options;


    def setContentType(self, type):

        self.__contentType = type;


    def setHttpCode(self, code):

        self.__httpCode = int(code);


    def setResponseContent(self, content):

        self.__responseContent = content;


    def setResponseHeaders(self, headers):

        self.__responseHeaders = headers;


    def setResponseSize(self, size):

        self.__responseSize = int(size);


    def setHeaders(self, headers):

        self.headers = headers;


    #FIXME: Find a better way to do this
    # The headers can be an array of elements.
    def getHeader(self, headerName):

        headers = self.headers.split("\n");
        for header in headers:
            key = header.split(":", 2);
            if (key[0] == headerName):
                return key[1].strip();
    
        return None;


    #FIXME: Find a better way to do this
    # The headers can be an array of elements.
    def getResponseHeader(self, headerName):
        #headers = self.responseHeaders.split(":");
        headers = self.__responseHeaders;
        for header in headers:
            if (header == headerName):
                return headers[header].strip();
        
    
        return None;


    def getCreated(self):

        return self.created;


    def setPostBody(self, postBody):

        self.postBody = postBody;


    def setUri(self, uri):

        self.uri = uri;

RequestUtil.py

class RequestUtil:
        def __init__(self):
                self.GETORPOST = {};
                self.despatcher = None;
                self.headers = {};
                self.SERVER = {};
                self.__response_ = [];
                self.__initialized_ = False;

        @staticmethod
        def setParams(keys, values, headers, caller):
                global requestHandler;
                requestHandler.GETORPOST = {};

                
                
                #if (requestHandler.__initialized_):
                #       return;

                i = 0;

                for key in keys:
                        requestHandler.GETORPOST[key] = values[i];
                        i = i+1;

                


                for keys in headers:
                        requestHandler.headers[keys] = headers[keys];

                requestHandler.despatcher = caller;
                import os;
                requestHandler.SERVER = os.environ;

                requestHandler.__initialized_ = True;


        @staticmethod
        def getDespatcher():
                global requestHandler;
                return requestHandler.despatcher;


        @staticmethod
        def addResponseData(response):
                global requestHandler;
                requestHandler.__response_.append(response);

        @staticmethod
        def clearResponseData():
                global requestHandler;
                requestHandler.__response_ = [];

        @staticmethod
        def getResponseData():
                global requestHandler;
                return ''.join(requestHandler.__response_);

        @staticmethod
        def flushResponseData():
                global requestHandler;

                import time;

                for data in requestHandler.__response_:
                        requestHandler.getDespatcher().response.out.write(data);

                time.sleep(10);
                requestHandler.__response_ = [];

        @staticmethod
        def getInstance():
                global requestHandler;
                return requestHandler;
            
requestHandler = RequestUtil();