merge_ddos_fingerpring_log_summary.py

gistfile1.txt

mypath ="/Users/santannajj/Desktop"

import os
import pandas as pd
import numpy as np


for file in os.listdir(mypath):
    if file.endswith(".log"):
        path_file = os.path.join(mypath, file)
        print(path_file)
        with open(path_file, 'r') as f:
            lines = f.read().splitlines()
            last_line = lines[-1]
            print(last_line)
            with open("all_summaries.csv", 'a') as out:
                out.write(last_line + '\n')


columns = ['raw_filename','multivector_key', 'singlevector_keys', 'filters', '#_src_ips']

df = pd.read_csv("all_summaries.csv", sep=";", names=columns)

df.replace(np.nan, "[]", regex=True)

df['singlevector_keys'] = df['singlevector_keys'].apply(lambda x: str(x).strip('[]').split(', '))
singlevector_keys = df.apply(lambda x: pd.Series(x['singlevector_keys']),axis=1).stack().reset_index(level=1, drop=True)
singlevector_keys.name = 'singlevector_keys'

df['filters'] = df['filters'].apply(lambda x: str(x).strip('[]').split(', '))
filters = df.apply(lambda x: pd.Series(x['filters']),axis=1).stack().reset_index(level=1, drop=True)
filters.name = 'filters'

df['#_src_ips'] = df['#_src_ips'].apply(lambda x: str(x).strip('[]').split(', '))
src_ips = df.apply(lambda x: pd.Series(x['#_src_ips']),axis=1).stack().reset_index(level=1, drop=True)
src_ips.name = '#_src_ips'

df_temp = pd.DataFrame({'singlevector_key':singlevector_keys, 'filters':filters, '#_src_ips':src_ips})

df_extended = df.drop(['singlevector_keys','filters','#_src_ips'], axis=1).join(df_temp)

df_extended.to_csv('summary.csv')

merge_fingerprint_log_summaries.py

mypath ="/Users/santannajj/Desktop"

import os
import pandas as pd
import numpy as np


for file in os.listdir(mypath):
    if file.endswith(".log"):
        path_file = os.path.join(mypath, file)
        print(path_file)
        with open(path_file, 'r') as f:
            lines = f.read().splitlines()
            last_line = lines[-1]
            print(last_line)
            with open("all_summaries.csv", 'a') as out:
                out.write(last_line + '\n')


columns = ['raw_filename','multivector_key', 'singlevector_keys', 'filters', '#_src_ips']

df = pd.read_csv("all_summaries.csv", sep=";", names=columns)

df.replace(np.nan, "[]", regex=True)

df['singlevector_keys'] = df['singlevector_keys'].apply(lambda x: str(x).strip('[]').split(', '))
singlevector_keys = df.apply(lambda x: pd.Series(x['singlevector_keys']),axis=1).stack().reset_index(level=1, drop=True)
singlevector_keys.name = 'singlevector_keys'

df['filters'] = df['filters'].apply(lambda x: str(x).strip('[]').split(', '))
filters = df.apply(lambda x: pd.Series(x['filters']),axis=1).stack().reset_index(level=1, drop=True)
filters.name = 'filters'

df['#_src_ips'] = df['#_src_ips'].apply(lambda x: str(x).strip('[]').split(', '))
src_ips = df.apply(lambda x: pd.Series(x['#_src_ips']),axis=1).stack().reset_index(level=1, drop=True)
src_ips.name = '#_src_ips'

df_temp = pd.DataFrame({'singlevector_key':singlevector_keys, 'filters':filters, '#_src_ips':src_ips})

df_extended = df.drop(['singlevector_keys','filters','#_src_ips'], axis=1).join(df_temp)

df_extended.to_csv('summary.csv')