CharlyWargnier

start_date = '2016-10-16' #@param {type:"date"}
end_date = '2016-10-17' #@param {type:"date"}

print('start date for the exported csv is: ' + start_date)
print('end date for the exported csv is: ' + end_date)
print ('')

CharlyWargnier

print ('FYI, the dataset spans the following time intervals:')
print (df['Date'].min(), df['Date'].max())

CharlyWargnier

#Define the function

def reverse_dns(ip_address):
    '''
    This method returns the true host name for a 
    given IP address
    '''
    host_name = socket.gethostbyaddr(ip_address)
    reversed_dns = host_name[0]
    return reversed_dns

CharlyWargnier

df['SEBotClass'] = pd.np.where(df.userAgent.str.contains("YandexBot"), "YandexBot",
  pd.np.where(df.userAgent.str.contains("bingbot"), "BingBot",
    pd.np.where(df.userAgent.str.contains("DuckDuckBot"), "DuckDuckGo",
      pd.np.where(df.userAgent.str.contains("Baiduspider"), "Baidu",
        pd.np.where(df.userAgent.str.contains("Googlebot/2.1"), "GoogleBot", "Else")))))

CharlyWargnier

df['httpCodeClass'] = pd.np.where(df.httpCode.str.contains("^1.*"), 'Info (1XX)',
  pd.np.where(df.httpCode.str.contains("^2.*"), 'Success (2XX)',
    pd.np.where(df.httpCode.str.contains("^3.*"), 'Redirects (3XX)',
      pd.np.where(df.httpCode.str.contains("^4.*"), 'Client errors (4XX)', 'Server errors (5XX)'))))

CharlyWargnier

df = df[['IP','Date','Domain','Method','url','httpProto','httpCode','timeTaken','urlRef','userAgent','header']]
df.rename(columns={"header": "redirURL"}, inplace=True)

CharlyWargnier

df[['remainder','header']] = df["header"].str.split('".*LOCATION=30.-', 1, expand=True)
df['header'] = df['header'].str.replace('\"', '')

CharlyWargnier

df[['userAgent','header']] = df["header"].str.split("\" ", 1, expand=True)

CharlyWargnier

df[['urlRef','header']] = df["header"].str.split(" ", 1, expand=True)
df['urlRef'] = df['urlRef'].str.replace('\"', '')

CharlyWargnier

df[['timeTaken','header']] = df["header"].str.split(" ", 1, expand=True)