Convert stealthwatch CSV to data for analysis

stealth_watch_convert.py

import datetime
from dateutil import parser
import csv

def swc_csv_data(filename):
    data = []
    d = csv.DictReader(open(filename))
    for entry in d:
        new_entry = {}
        for k, v in entry.items():
            nk, pfn = SW_KEYS.get(k, (None, None))
            # print(k, nk, v)
            if nk is None:
                continue
            new_entry[nk] = pfn(v)
        data.append(new_entry)
    return data

def swc_to_float(val):
    if val.find('--') > -1:
        return float('nan')

    if val.strip().find(' ') == -1:
        return float(val)

    v, m = val.strip().split(' ')
    m_f = float(1)
    if m in MULTIPLIER:
        m_f = float(MULTIPLIER[m])

    return float(v.strip()) * m_f

def swc_to_int(val):
    if val.find('--') > -1:
        return 0

    if val.strip().find(' ') == -1:
        return int(val)

    v, m = val.strip().split(' ')
    m_i = int(1)
    if m in MULTIPLIER:
        m_i = MULTIPLIER[m]

    return int(v.strip()) * m_i

def swc_to_string(val):
    if val.find('--') > -1:
        return ''
    return val

def swc_to_delta(val):
    convert = lambda x: 0 if len(x) == 0 else int(x)
    
    if val.find('--') > -1:
        return None

    s = 0
    if val.find('H') > -1:
        h = v.split('H')[0].strip()
        s = s + convert(h)


    if val.find('M') > -1:
        h = v.split('M')[0].strip().split()[-1]
        s = s + convert(h)

    if val.find('M') > -1:
        h = v.split('S')[0].strip().split()[-1]
        s = s + convert(h)

    return datetime.timedelta(seconds=s)

def swc_to_date(val):
    if val.find('--'):
        return None
    return parser.parse(val)


SW_KEYS = {
    "Flow ID" : ('flow_id', swc_to_string), # str
    "Domain" : ('domain', swc_to_string), # str
    "Start" : ('start', swc_to_date), # date
    "End" : ('end', swc_to_date), # date
    "Duration" : ('duration', swc_to_delta), # str hr min s
    "Flow Action" : ('flow_action', swc_to_string), # str or '--' 
    "Subject ASN" : ('subject_asn', swc_to_string), # str or '--' 
    "Subject ASN Assignment" : ('subject_asn_assignment', swc_to_string), # str or '--' 
    "Subject Byte Ratio" : ('subject_byte_ratio', swc_to_float), # float or '--'? 
    "Subject IP Address" : ('subject_ip_address', swc_to_string), # str
    "Subject Hostname" : ('subject_hostname', swc_to_string), # str
    "Subject MAC Address" : ('subject_mac_address', swc_to_string), # str
    "Subject MAC Vendor" : ('subject_mac_vendor', swc_to_string), # str
    "Subject NAT" : ('subject_nat', swc_to_string), # str
    "Subject NAT Hostname" : ('subject_nat_hostname', swc_to_string), # str
    "Subject NAT Port" : ('subject_nat_port', swc_to_int), # int
    "Subject Orientation" : ('subject_orientation', swc_to_string), # str
    "Subject Port/Protocol" : ('subject_port_protocol', swc_to_string), #str
    "Subject Host Groups" : ('subject_host_groups', swc_to_string), # str
    "Subject Location" : ('subject_location', swc_to_string), # str
    "Subject User" : ('subject_user', swc_to_string),
    "Subject Bytes" : ('subject_bytes', swc_to_float), # int G * 10^6, M * 10^6, K 10^3
    "Subject Byte Rate" : ('subject_byte_rate', swc_to_float), # float
    "Subject Interfaces" : ('subject_interfaces', swc_to_string), # str
    "Subject Packets" : ('subject_packets', swc_to_float), # int
    "Subject Packet Rate" : ('subject_packet_rate', swc_to_float), # float 
    "Subject Payload" : ('subject_payload', swc_to_string), # str
    "Subject Process Account" : ('subject_process_account', swc_to_string), # str
    "Subject Process Name" : ('subject_process_name', swc_to_string), # str
    "Subject File Hash" : ('subject_file_hash', swc_to_string), # str
    "Subject Parent Process Name" : ('subject_parent_process_name', swc_to_string), # str
    "Subject Parent File Hash" : ('subject_parent_file_hash', swc_to_string), # str
    "Subject TrustSec ID" : ('subject_trustsec_id', swc_to_string), # str
    "Subject TrustSec Name" : ('subject_trustsec_name', swc_to_string), # str
    "Subject FIN Packets" : ('subject_fin_packets', swc_to_int), # int G * 10^6, M * 10^6, K 10^3
    "Subject RST Packets" : ('subject_rst_packets', swc_to_int), # int G * 10^6, M * 10^6, K 10^3
    "Subject SYN Packets" : ('subject_syn_packets', swc_to_int), # int G * 10^6, M * 10^6, K 10^3
    "Subject SYN/ACK Packets" : ('subject_syn_ack_packets', swc_to_int), # int G * 10^6, M * 10^6, K 10^3
    "Appliance" : ('appliance', swc_to_string), # str
    "Application" : ('application', swc_to_string), # str
    "Application (Flow Sensor)" : ('application_flow_sensor', swc_to_string), # str
    "Application (NBAR)" : ('application_nbar', swc_to_string), # str
    "Application (PacketShaper)" : ('application_packetshaper', swc_to_string), # str
    "Application (Palo Alto Networks)" : ('application_palo_alto_networks', swc_to_string), # str
    "Byte Rate" : ('byte_rate', swc_to_float), # float
    "Total Bytes" : ('total_bytes', swc_to_float), # float
    "Packet Rate" : ('packet_rate', swc_to_float), # float
    "Total Packets" : ('total_packets', swc_to_float), # float
    "Total Traffic (bps)" : ('total_traffic_bps', swc_to_float), # float
    "protocol" : ('protocol', swc_to_string), # str
    "Service" : ('service', swc_to_string), # str
    "TCP Connections" : ('tcp_connections', swc_to_string), # str
    "TCP Retransmissions" : ('tcp_retransmissions', swc_to_float), # float
    "TCP Retransmission Ratio" : ('tcp_retransmission_ratio', swc_to_float), # float
    "MPLS Label" : ('mpls_label', swc_to_string),
    "RTT Average" : ('rtt_average', swc_to_string),
    "RTT Maximum" : ('rtt_maximum', swc_to_string),
    "RTT Minimum" : ('rtt_minimum', swc_to_string),
    "SRT Average" : ('srt_average', swc_to_string),
    "SRT Maximum" : ('srt_maximum', swc_to_string),
    "SRT Minimum" : ('srt_minimum', swc_to_string),
    "VLAN ID" : ('vlan_id', swc_to_float), # float
    "Encryption TLS/SSL Version" : ('encryption_tls_ssl_version', swc_to_string),
    "Encryption Key Exchange" : ('encryption_key_exchange', swc_to_string),
    "Encryption Authentication Algorithm" : ('encryption_authentication_algorithm', swc_to_string),
    "Encryption Algorithm and Key Length" : ('encryption_algorithm_and_key_length', swc_to_string),
    "Encryption MAC" : ('encryption_mac', swc_to_string),
    "Peer ASN" : ('peer_asn', swc_to_string),
    "Peer ASN Assignment" : ('peer_asn_assignment', swc_to_string),
    "Peer Byte Ratio" : ('peer_byte_ratio', swc_to_string),
    "Peer IP Address" : ('peer_ip_address', swc_to_string),
    "Peer Hostname" : ('peer_hostname', swc_to_string),
    "Peer MAC Address" : ('peer_mac_address', swc_to_string),
    "Peer MAC Vendor" : ('peer_mac_vendor', swc_to_string),
    "Peer NAT" : ('peer_nat', swc_to_string),
    "Peer NAT Hostname" : ('peer_nat_hostname', swc_to_string),
    "Peer NAT Port" : ('peer_nat_port', swc_to_string),
    "Peer Orientation" : ('peer_orientation', swc_to_string),
    "Peer Port/Protocol" : ('peer_port_protocol', swc_to_string),
    "Peer Host Groups" : ('peer_host_groups', swc_to_string),
    "Peer Location" : ('peer_location', swc_to_string),
    "Peer User" : ('peer_user', swc_to_string),
    "Peer Bytes" : ('peer_bytes', swc_to_float), # float
    "Peer Byte Rate" : ('peer_byte_rate', swc_to_float), # float
    "Peer Interfaces" : ('peer_interfaces', swc_to_float), # float
    "Peer Packets" : ('peer_packets', swc_to_float), # float
    "Peer Packet Rate" : ('peer_packet_rate', swc_to_float), # float
    "Peer Payload" : ('peer_payload', swc_to_string),
    "Peer Process Account" : ('peer_process_account', swc_to_string),
    "Peer Process Name" : ('peer_process_name', swc_to_string),
    "Peer File Hash" : ('peer_file_hash', swc_to_string),
    "Peer Parent Process Name" : ('peer_parent_process_name', swc_to_string),
    "Peer Parent File Hash" : ('peer_parent_file_hash', swc_to_string),
    "Peer TrustSec ID" : ('peer_trustsec_id', swc_to_string),
    "Peer TrustSec Name" : ('peer_trustsec_name', swc_to_string),
    "Peer FIN Packets" : ('peer_fin_packets', swc_to_float), # float
    "Peer RST Packets" : ('peer_rst_packets', swc_to_float), # float
    "Peer SYN Packets" : ('peer_syn_packets', swc_to_float), # float
    "Peer SYN/ACK Packets" : ('peer_syn_ack_packets', swc_to_float), # float
    "Actions" : ('actions', swc_to_string),
}

MULTIPLIER = {
    'K': 10**3,
    'M': 10**6,
    'G': 10**9,
}