gurl.py

# http://httpstat.us - for test error codes
# http://uri-labs.com/macosx_headers/NSURLConnection_h/Protocols/NSURLConnectionDelegate/index.html
 
# Notes:
# - Errors are only thrown when the connection:
#   - Is interrupted before the headers can complete
#   - SSL couldn't happen correctly
#   - The connection never happens
# - A delegate.response always has a NSHTTPURLResponse key for HTTP/HTTPS
#   For a file:// transfer, it's NSURLResponse


import os
import sys
import xattr

from Foundation import NSRunLoop, NSDate
from Foundation import NSObject, NSURL, NSURLConnection
from Foundation import NSMutableURLRequest
from Foundation import NSURLRequestReloadIgnoringLocalCacheData
from Foundation import NSURLResponseUnknownLength
from Foundation import NSLog
from Foundation import NSString, NSUTF8StringEncoding
from Foundation import NSURLCredential, NSURLCredentialPersistenceNone

from Foundation import NSData, \
                       NSPropertyListSerialization, \
                       NSPropertyListMutableContainersAndLeaves, \
                       NSPropertyListXMLFormat_v1_0


ssl_error_codes = {-9800: u'SSL protocol error',
                   -9801: u'Cipher Suite negotiation failure',
                   -9802: u'Fatal alert',
                   -9803: u'I/O would block (not fatal)',
                   -9804: u'Attempt to restore an unknown session',
                   -9805: u'Connection closed gracefully',
                   -9806: u'Connection closed via error',
                   -9807: u'Invalid certificate chain',
                   -9808: u'Bad certificate format',
                   -9809: u'Underlying cryptographic error',
                   -9810: u'Internal error',
                   -9811: u'Module attach failure',
                   -9812: u'Valid cert chain, untrusted root',
                   -9813: u'Cert chain not verified by root',
                   -9814: u'Chain had an expired cert',
                   -9815: u'Chain had a cert not yet valid',
                   -9816: u'Server closed session with no notification',
                   -9817: u'Insufficient buffer provided',
                   -9818: u'Bad SSLCipherSuite',
                   -9819: u'Unexpected message received',
                   -9820: u'Bad MAC',
                   -9821: u'Decryption failed',
                   -9822: u'Record overflow',
                   -9823: u'Decompression failure',
                   -9824: u'Handshake failure',
                   -9825: u'Misc. bad certificate',
                   -9826: u'Bad unsupported cert format',
                   -9827: u'Certificate revoked',
                   -9828: u'Certificate expired',
                   -9829: u'Unknown certificate',
                   -9830: u'Illegal parameter',
                   -9831: u'Unknown Cert Authority',
                   -9832: u'Access denied',
                   -9833: u'Decoding error',
                   -9834: u'Decryption error',
                   -9835: u'Export restriction',
                   -9836: u'Bad protocol version',
                   -9837: u'Insufficient security',
                   -9838: u'Internal error',
                   -9839: u'User canceled',
                   -9840: u'No renegotiation allowed',
                   -9841: u'Peer cert is valid, or was ignored if verification disabled',
                   -9842: u'Server has requested a client cert',
                   -9843: u'Peer host name mismatch',
                   -9844: u'Peer dropped connection before responding',
                   -9845: u'Decryption failure',
                   -9846: u'Bad MAC',
                   -9847: u'Record overflow',
                   -9848: u'Configuration error',
                   -9849: u'Unexpected (skipped) record in DTLS'}


class Gurl(NSObject):

    GURL_XATTR = 'com.googlecode.munki.downloadData'

    def initWithOptions_(self, options):
        self = super(Gurl, self).init()
        if not self:
            return

        self.follow_redirects = options.get('follow_redirects') or False
        self.destination_path = options.get('file')
        self.can_resume = options.get('can_resume') or False
        self.url = options.get('url')
        self.additional_headers = options.get('additional_headers') or {}
        self.username = options.get('username')
        self.password = options.get('password')
        self.download_only_if_changed = options.get('download_only_if_changed') or False
        self.connection_timeout = options.get('connection_timeout') or 10

        self.resume = False
        self.response = None
        self.headers = None
        self.status = None
        self.error = None
        self.SSLerror = None
        self.done = False
        self.redirection = []
        self.destination = None
        self.bytesReceived = 0
        self.expectedLength = -1
        self.percentComplete = 0
        self.connection = None
        return self

    def start(self):
        if not self.destination_path:
            NSLog('No output file specified.')
            self.done = True
            return
        url = NSURL.URLWithString_(self.url)
        request = NSMutableURLRequest.requestWithURL_cachePolicy_timeoutInterval_(
            url, NSURLRequestReloadIgnoringLocalCacheData, self.connection_timeout)
        if self.additional_headers:
            for header, value in self.additional_headers.items():
                request.setValue_forHTTPHeaderField_(value, header)
        # does the file already exist?
        if os.path.isfile(self.destination_path):
            stored_data = self.get_stored_headers()
            if (self.can_resume and 'expected-length' in stored_data
                and ('last-modified' in stored_data or 'etag' in stored_data)):
                # we have a partial file and we're allowed to resume
                self.resume = True
                local_filesize = os.path.getsize(self.destination_path)
                byte_range = 'bytes=%s-' % local_filesize
                request.setValue_forHTTPHeaderField_(byte_range, 'Range')
            elif self.download_only_if_changed:
                if 'last-modified' in stored_data:
                    request.setValue_forHTTPHeaderField_(stored_data['last-modified'], 'if-modified-since')
                if 'etag' in stored_data:
                    request.setValue_forHTTPHeaderField_(stored_data['etag'], 'if-none-match')
        self.connection = NSURLConnection.alloc().initWithRequest_delegate_(request, self)

    def cancel(self):
        if self.connection:
            self.connection.cancel()
            self.done = True

    def isDone(self):
        if self.done:
            return self.done
        # let the delegates do their thing
        NSRunLoop.currentRunLoop().runUntilDate_(NSDate.dateWithTimeIntervalSinceNow_(.1))
        return self.done

    def get_stored_headers(self):
        '''Returns any stored headers for self.destination_path'''
        # try to read stored headers
        try:
            stored_plist_str = xattr.getxattr(self.destination_path, self.GURL_XATTR)
        except (KeyError, IOError):
            return {}
        data = buffer(stored_plist_str)
        dataObject, plistFormat, error = \
        NSPropertyListSerialization.propertyListFromData_mutabilityOption_format_errorDescription_(
                       data, NSPropertyListMutableContainersAndLeaves, None, None)
        if error:
            return {}
        else:
            return dataObject

    def store_headers(self, headers):
        '''Store dictionary data as an xattr for self.destination_path'''
        plistData, error = \
         NSPropertyListSerialization.dataFromPropertyList_format_errorDescription_(
                                headers, NSPropertyListXMLFormat_v1_0, None)
        if error:
            string = ''
        else:
            string = str(plistData)
        xattr.setxattr(self.destination_path, self.GURL_XATTR, string)

    def normalize_header_dict(self, a_dict):
        '''Since HTTP header names are not case-sensitive, we normalize a
        dictionary of HTTP headers by converting all the key names to lower case'''
        new_dict = {}
        for key, value in a_dict.items():
            new_dict[key.lower()] = value
        return new_dict

    def connection_didFailWithError_(self, connection, error):
        self.error = error
        # If this was an SSL error, try to extract the SSL error code.
        if 'NSUnderlyingError' in error.userInfo():
            ssl_code = error.userInfo()['NSUnderlyingError'].userInfo().get('_kCFNetworkCFStreamSSLErrorOriginalValue', None)
            if ssl_code:
                self.SSLerror = (ssl_code, ssl_error_codes.get(ssl_code, 'Unknown SSL error'))
        else:
            print error.userInfo()
        self.done = True
        if self.destination and self.destination_path:
            self.destination.close()
            # delete it? Might not want to...

    def connectionDidFinishLoading_(self, connection):
        self.done = True
        if self.destination and self.destination_path:
            self.destination.close()
        # remove the expected-size from the stored headers
        headers = self.get_stored_headers()
        if 'expected-length' in headers:
            del headers['expected-length']
        self.store_headers(headers)

    def connection_didReceiveResponse_(self, connection, response):
        self.response = response
        self.bytesReceived = 0
        self.percentComplete = -1
        self.expectedLength = response.expectedContentLength()

        download_data = {}
        if response.className() == u'NSHTTPURLResponse':
            # Headers and status code only available for HTTP/S transfers
            self.status = response.statusCode()
            self.headers = dict(response.allHeaderFields())
            normalized_headers = self.normalize_header_dict(self.headers)
            if 'last-modified' in normalized_headers:
                download_data['last-modified'] = normalized_headers['last-modified']
            if 'etag' in normalized_headers:
                download_data['etag'] = normalized_headers['etag']
            download_data['expected-length'] = self.expectedLength

        if not self.destination and self.destination_path:
            if self.resume:
                stored_data = self.get_stored_headers()
                if (not stored_data
                    or stored_data.get('etag') != download_data.get('etag')
                    or stored_data.get('last-modified') != download_data.get('last-modified')):
                    # file on server is different than the one we have a partial for
                    NSLog('Can\'t resume download; file on server has changed.')
                    connection.cancel()
                    self.done = True
                    NSLog('Removing %s' % self.destination_path)
                    os.unlink(self.destination_path)
                    # need to set some error
                    return
                # try to resume
                NSLog('Resuming download for %s' % self.destination_path)
                # add existing file size to bytesReceived so far
                local_filesize = os.path.getsize(self.destination_path)
                self.bytesReceived = local_filesize
                self.expectedLength += local_filesize
                # open file for append
                self.destination = open(self.destination_path, 'a')
                
            else:
                # not resuming, just open the file for writing
                self.destination = open(self.destination_path, 'w')
                # store some headers with the file for use if we need to resume the download
                # and for future checking if the file on the server has changed
                self.store_headers(download_data)

    def connection_willSendRequest_redirectResponse_(self, connection, request, response):
        if response == None:
            # This isn't a real redirect, this is without talking to a server.
            # Pass it back as-is
            return request
        # But if we're here, it appears to be a real redirect attempt
        # Annoyingly, we apparently can't get access to the headers from the site that told us to redirect.
        # All we know is that we were told to redirect and where the new location is.
        newURL = request.URL().absoluteString()
        self.redirection.append([newURL, dict(response.allHeaderFields())])
        if self.follow_redirects:
            # Allow the redirect
            NSLog('Allowing redirect to: %s' % newURL)
            return request
        else:
            # Deny the redirect
            NSLog('Denying redirect to: %s' % newURL)
            return None

    def connection_canAuthenticateAgainstProtectionSpace_(self, connection, protectionSpace):
        # this is not called in 10.5.x.
        NSLog('connection_canAuthenticateAgainstProtectionSpace_')
        if protectionSpace:
            host = protectionSpace.host()
            realm = protectionSpace.realm()
            authenticationMethod = protectionSpace.authenticationMethod()
            NSLog('Protection space found. Host: %s Realm: %s AuthMethod: %s'
                  % (host, realm, authenticationMethod))
            if self.username and self.password and authenticationMethod in [
                'NSURLAuthenticationMethodDefault', 
                'NSURLAuthenticationMethodHTTPBasic', 
                'NSURLAuthenticationMethodHTTPDigest']:
                # we know how to handle this
                NSLog('Can handle this authentication request')
                return True
        # we don't know how to handle this; let the OS try
        NSLog('Allowing OS to handle authentication request')
        return False

    def connection_didReceiveAuthenticationChallenge_(self, connection, challenge):
        protectionSpace = challenge.protectionSpace()
        host = protectionSpace.host()
        realm = protectionSpace.realm()
        authenticationMethod = protectionSpace.authenticationMethod()
        NSLog('Authentication challenge for Host: %s Realm: %s AuthMethod: %s'
              % (host, realm, authenticationMethod))
        if challenge.previousFailureCount() > 0:
            # we have the wrong credentials. just fail
            NSLog('Previous authentication attempt failed.')
            challenge.sender().cancelAuthenticationChallenge_(challenge)
        if self.username and self.password and authenticationMethod in [
            'NSURLAuthenticationMethodDefault', 
            'NSURLAuthenticationMethodHTTPBasic', 
            'NSURLAuthenticationMethodHTTPDigest']:
            NSLog('Will attempt to authenticate.')
            NSLog('Username: %s Password: %s' % (self.username, ('*' * len(self.password or ''))))
            credential = NSURLCredential.credentialWithUser_password_persistence_(
                self.username, self.password, NSURLCredentialPersistenceNone)
            challenge.sender().useCredential_forAuthenticationChallenge_(credential, challenge)
        else:
            # fall back to system-provided default behavior
            NSLog('Continuing without credential.')
            challenge.sender().continueWithoutCredentialForAuthenticationChallenge_(challenge)

    def connection_didReceiveData_(self, connection, data):
        if self.destination_path:
            self.destination.write(str(data))
        else:
            print NSString.alloc().initWithData_encoding_(data, NSUTF8StringEncoding)
        self.bytesReceived += len(data)
        if self.expectedLength != NSURLResponseUnknownLength:
            self.percentComplete = int(float(self.bytesReceived)/float(self.expectedLength)*100.0)


def run_connection(options):
    print "\n***Testing %s (%s)...***" % (options['url'], options.get('note'))
    connection = Gurl.alloc().initWithOptions_(options)
    percent_complete = -1
    bytes_received = 0
    connection.start()
    try:
        while not connection.isDone():
            if connection.destination_path:
                # only print progress info if we are writing to a file
                if connection.percentComplete != -1:
                    if connection.percentComplete != percent_complete:
                        percent_complete = connection.percentComplete
                        print 'Percent complete: %s' % percent_complete
                elif connection.bytesReceived != bytes_received:
                    bytes_received = connection.bytesReceived
                    print 'Bytes received: %s' % bytes_received
                    
    except (KeyboardInterrupt, SystemExit):
        # safely kill the connection then fall through
        connection.cancel()
    except Exception: # too general, I know
        # Let us out! ... Safely! Unexpectedly quit dialogs are annoying ...
        connection.cancel()
        # Re-raise the error
        raise

    if connection.error != None:
        print 'Error:', (connection.error.code(), connection.error.localizedDescription())
        if connection.SSLerror:
            print 'SSL error:', connection.SSLerror
    if connection.response != None:
        # print 'Response:', dir(connection.response)
        print 'Status:', connection.status
        print 'Headers:', connection.headers
    if connection.redirection != []:
        print 'Redirection:', connection.redirection


# Stuff we might need to specify:
# url
# file_destination
# resume_partial_download? 
# username
# password
# additional_headers
# follow_redirects?
# progress info?
# client cert (to verify client to server)
# cacert(s)  (to help client verify server)

# some URLs for testing
# https://tidia.ita.br/ - Untrusted CA
# https://testssl-expire.disig.sk/ - Expired cert
# https://tv.eurosport.com/ - Wrong hostname for cert
# https://test-sspev.verisign.com:2443/ - Revoked cert
# http://httpstat.us/404 - Status codes on-demand
# http://github.com/404 - HTTP to HTTPS redirect
# http://jigsaw.w3.org/HTTP/300/301.html
# http://jigsaw.w3.org/HTTP/300/302.html
# http://jigsaw.w3.org/HTTP/300/307.html
# http://browserspy.dk/password-ok.php -- HTTP Basic Authentication; username test password test
# file:///Users/someone/Desktop/something - a file URL"""

#'Authorization': 'Basic dGVzdDp0ZXN0' (corresponds to username:test, password:test)

def main():

    test_list = [
        {
            'note': 'Untrusted CA',
            'url': 'https://tidia.ita.br/',
            'file': '/tmp/foo',
        },
        {
            'note': 'Expired cert',
            'url': 'https://testssl-expire.disig.sk/',
            'file': '/tmp/foo',
        },
        {
            'note': 'Wrong hostname for cert',
            'url': 'https://tv.eurosport.com/',
            'file': '/tmp/foo',
        },
        {
            'note': 'Revoked cert',
            'url': 'https://test-sspev.verisign.com:2443/',
            'file': '/tmp/foo',
        },
        {
            'note': 'HTTP to HTTPS redirect',
            'url': 'http://github.com/404',
            'file': '/tmp/foo',
            'follow_redirects': True,
        },
        {
            'note': '301 redirect',
            'url': 'http://jigsaw.w3.org/HTTP/300/301.html',
            'file': '/tmp/foo',
            'follow_redirects': True,
        },
        {
            'note': '302 redirect',
            'url': 'http://jigsaw.w3.org/HTTP/300/302.html',
            'file': '/tmp/foo',
            'follow_redirects': True,
        },
        {
            'note': '307 redirect',
            'url': 'http://jigsaw.w3.org/HTTP/300/307.html',
            'file': '/tmp/foo',
            'follow_redirects': True,
        },
        {
            'note': 'HTTP Basic Authentication with username and password',
            'url': 'http://browserspy.dk/password-ok.php',
            'file': '/tmp/foo',
            'username': 'test',
            'password': 'test',
        },
        {
            'note': 'HTTP Basic Authentication with header',
            'url': 'http://browserspy.dk/password-ok.php',
            'file': '/tmp/foo',
            'additional_headers': {'Authorization': 'Basic dGVzdDp0ZXN0'},
        },
        {
            'note': 'HTTP Digest Authentication',
            'url': 'http://httpbin.org/digest-auth/auth/user/passwd',
            'file': '/tmp/foo',
            'username': 'user',
            'password': 'passwd',
        },
        {
            'note': 'Downloading a big file',
            'url': 'https://dl.google.com/chrome/mac/stable/GGRO/googlechrome.dmg',
            'file': '/tmp/foo',
            'can_resume': False,
            'download_only_if_changed': False,
        },
        {
            'note': 'Downloading a big file only if changed.',
            'url': 'https://dl.google.com/chrome/mac/stable/GGRO/googlechrome.dmg',
            'file': '/tmp/foo',
            'can_resume': True,
            'download_only_if_changed': True,
        },
    ]
    
    for options in test_list:
        run_connection(options)

if __name__ == '__main__':
    main()