gistfile1.txt

import re 
from furl import furl

def param_extract(response, level, black_list, placeholder):

    ''' 
    regexp : r'.*?:\/\/.*\?.*\=[^$]'
    regexp : r'.*?:\/\/.*\?.*\='
    '''

    parsed = list(set(re.findall(r'.*?:\/\/.*\?.*\=[^$]' , response)))
    final_uris = []        
    for i in parsed:
        delim = i.find('=')
        second_delim = i.find('=', i.find('=') + 1)
        if len(black_list) > 0:
            words_re = re.compile("|".join(black_list))
            if not words_re.search(i):
                f = furl(i) 
                if len(f.args) > 0 :
                    for k in f.args:
                        f.args[k] = placeholder
                final_uris.append(f.url)
                #final_uris.append((i[:delim+1] + placeholder))
                if level == 'high':
                    final_uris.append(i[:second_delim+1] + placeholder)
        else:
            final_uris.append((i[:delim+1] + placeholder))
            if level == 'high':
                final_uris.append(i[:second_delim+1] + placeholder)

    # for i in final_uris:
    #     k = [ele for ele in black_list if(ele in i)]
    
    return list(set(final_uris))