parse_log.py

# coding:utf-8
import argparse
import base64
import json
import os
from enum import Enum


args = None

summary = {
    "total": 0,
    "ignore": 0,
    "non-http": 0,
    "error": 0,
    "ok": 0,
    "abnormal-http": 0,
}


class STATE_STATUS(Enum):
    START = 0
    FIRST_LINE = 1
    HEADER = 2
    BODY = 3


class Tokenizer(object):
    def __init__(self, _payload):
        self.payload = _payload
        self.index = 0

    def read_char(self):
        t = self.payload[self.index]
        self.index += 1
        return t

    def read_char_num(self, num):
        t = self.payload[self.index:self.index + num]
        self.index += num
        return t

    def read_line(self):
        t = self.payload[self.index:].split(b"\r\n")[0]
        self.index += len(t) + 2
        return t

    def is_eof(self):
        return self.index >= len(self.payload)


def save_abnormal_http(l, filepath):
    with open(filepath, 'a') as f:
        f.write(l + "\n")


def save_http(sid, signature, payload):
    """
    1、payload中可能包含多个请求
    2、post请求后面如果还有请求，两个请求间没有分割符

    payload = b"GET /index.php/wap/goods/getGoodsListByConditions?category_id=1&brand_id=2&min_price=3&max_price=4&page=5&page_size=6&order=7%20and%20extractvalue(1,concat(1,(select%20md5(844)),0x7e))%20and%201=1&attr_array[][2]=8&spec_array[]=9 HTTP/1.1\r\nHost: 1.1.1.1:7789\r\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36\r\nAccept: */*\r\nWhouserscanner: xxxxx-SOC\r\nWhousescannner: xxxxx-SOC\r\nAccept-Encoding: gzip\r\n\r\nGET /cf-scripts/scripts/ajax/package/cfajax.js HTTP/1.1\r\nHost: 1.1.1.1:7789\r\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36\r\nAccept: */*\r\nWhouserscanner: xxxxx-SOC\r\nWhousescannner: xxxxx-SOC\r\nAccept-Encoding: gzip\r\n\r\nPOST /general/index/UploadFile.php?m=uploadPicture&uploadType=theme&userId= HTTP/1.1\r\nHost: 1.1.1.1:7789\r\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36\r\nContent-Length: 180\r\nAccept: */*\r\nContent-Type: multipart/form-data; boundary=cmceaavlecasqakgfysv\r\nWhouserscanner: xxxxx-SOC\r\nWhousescannner: xxxxx-SOC\r\nAccept-Encoding: gzip\r\n\r\n--cmceaavlecasqakgfysv\r\nContent-Disposition: form-data; name=\"Filedata\"; filename=\"test.php\"\r\nContent-Type: image/jpeg\r\n\r\n<?php\r\necho md5(130114931);\r\n?>\r\n --cmceaavlecasqakgfysv--GET /login.asp HTTP/1.1\r\nHost: 1.1.1.1:7789\r\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36\r\nAccept: */*\r\nWhouserscanner: xxxxx-SOC\r\nWhousescannner: xxxxx-SOC\r\nAccept-Encoding: gzip\r\n\r\nGET /module/api.php?mobile/webNasIPS HTTP/1.1\r\nHost: 1.1.1.1:7789\r\nUser-Agent: TNAS\r\nAccept: */*\r\nWhouserscanner: xxxxx-SOC\r\nWhousescannner: xxxxx-SOC\r\nAccept-Encoding: gzip\r\n\r\nGET /index.php?target=db_sql.php%253f/../../../../../../../../etc/passwd HTTP/1.1\r\nHost: 1.1.1.1:7789\r\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36\r\nAccept: */*\r\nWhouserscanner: xxxxx-SOC\r\nWhousescannner: xxxxx-SOC\r\nAccept-Encoding: gzip\r\n\r\nGET /logfile?d=crossdomain.xml HTTP/1.1\r\nHost: 1.1.1.1:7789\r\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36\r\nAccept: */*\r\nWhouserscanner: xxxxx-SOC\r\nWhousescannner: xxxxx-SOC\r\nAccept-Encoding: gzip\r\n\r\nPOST /index/Api/post_curl HTTP/1.1\r\nHost: 1.1.1.1:7789\r\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36\r\nContent-Length: 30\r\nAccept: */*\r\nContent-Type: application/x-www-form-urlencoded\r\nWhouserscanner: xxxxx-SOC\r\nWhousescannner: xxxxx-SOC\r\nAccept-Encoding: gzip\r\n\r\nurl=file:///etc/passwd&data[]=GET /index.php?m=user&c=Users&a=logout&referurl=https://example.com HTTP/1.1\r\nHost: 1.1.1.1:7789\r\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36\r\nAccept: */*\r\nWhouserscanner: xxxxx-SOC\r\nWhousescannner: xxxxx-SOC\r\nAccept-Encoding: gzip\r\n\r\n"
    """
    global args, summary

    if payload.split(b" ")[0] not in [b"GET", b"POST", b"PUT", b"DELETE", b"CONNECT"]:
        save_abnormal_http(line, args.error_file)
        summary["abnormal-http"] += 1
        return False

    current_status = STATE_STATUS.START

    http_requests = []
    request_text = b""
    request_method = b""
    content_length = 0
    t = Tokenizer(payload)
    while True:
        # x_line = .readline()
        if t.is_eof():
            break

        if current_status == STATE_STATUS.START:
            x_line = t.read_line()
            if x_line.split(b" ")[0] in [b"GET", b"POST", b"PUT", b"DELETE", b"CONNECT"]:
                request_text += x_line
                request_method = x_line.split(b" ")[0]
                current_status = STATE_STATUS.HEADER
            continue
        if current_status == STATE_STATUS.HEADER:
            x_line = t.read_line()
            if x_line != b"":
                request_text += b"\r\n" + x_line
                if x_line.startswith(b"Content-Length:"):
                    content_length = int(x_line.split(b" ")[1])
            else:
                if request_method in [b"POST", b"PUT"]:
                    current_status = STATE_STATUS.BODY
                else:
                    request_text += b"\r\n"
                    http_requests.append(request_text)

                    request_text = b""
                    request_method = b""
                    content_length = 0

                    current_status = STATE_STATUS.START
        if current_status == STATE_STATUS.BODY:
            request_text += b"\r\n\r\n" + t.read_char_num(content_length)
            http_requests.append(request_text)
            request_text = b""
            request_method = b""
            content_length = 0

            current_status = STATE_STATUS.START
            continue

    # print(http_requests)
    output_dir = args.output_dir
    if len(http_requests) > 1:
        count = 0
        for http_request in http_requests:
            fpath = "%s/%s_%d.http" % (output_dir, sid, count)
            with open(fpath, 'wb') as f:
                metadata = b"# format:http,sid:%d,signature:%b\n" % (sid, signature.encode("utf-8"))
                f.write(metadata)
                f.write(http_request)
            count += 1
    else:
        with open(output_dir + "/" + str(sid) + ".http", 'wb') as f:
            metadata = b"# format:http,sid:%d,signature:%b\n" % (sid, signature.encode("utf-8"))
            f.write(metadata)
            f.write(http_requests[0])
    return True


def test_save_http():
    p = b"GET /index.php/wap/goods/getGoodsListByConditions?category_id=1&brand_id=2&min_price=3&max_price=4&page=5&page_size=6&order=7%20and%20extractvalue(1,concat(1,(select%20md5(844)),0x7e))%20and%201=1&attr_array[][2]=8&spec_array[]=9 HTTP/1.1\r\nHost: 1.1.1.1:7789\r\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36\r\nAccept: */*\r\nWhouserscanner: xxxxx-SOC\r\nWhousescannner: xxxxx-SOC\r\nAccept-Encoding: gzip\r\n\r\nGET /cf-scripts/scripts/ajax/package/cfajax.js HTTP/1.1\r\nHost: 1.1.1.1:7789\r\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36\r\nAccept: */*\r\nWhouserscanner: xxxxx-SOC\r\nWhousescannner: xxxxx-SOC\r\nAccept-Encoding: gzip\r\n\r\nPOST /general/index/UploadFile.php?m=uploadPicture&uploadType=theme&userId= HTTP/1.1\r\nHost: 1.1.1.1:7789\r\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36\r\nContent-Length: 180\r\nAccept: */*\r\nContent-Type: multipart/form-data; boundary=cmceaavlecasqakgfysv\r\nWhouserscanner: xxxxx-SOC\r\nWhousescannner: xxxxx-SOC\r\nAccept-Encoding: gzip\r\n\r\n--cmceaavlecasqakgfysv\r\nContent-Disposition: form-data; name=\"Filedata\"; filename=\"test.php\"\r\nContent-Type: image/jpeg\r\n\r\n<?php\r\necho md5(130114931);\r\n?>\r\n --cmceaavlecasqakgfysv--GET /login.asp HTTP/1.1\r\nHost: 1.1.1.1:7789\r\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36\r\nAccept: */*\r\nWhouserscanner: xxxxx-SOC\r\nWhousescannner: xxxxx-SOC\r\nAccept-Encoding: gzip\r\n\r\nGET /module/api.php?mobile/webNasIPS HTTP/1.1\r\nHost: 1.1.1.1:7789\r\nUser-Agent: TNAS\r\nAccept: */*\r\nWhouserscanner: xxxxx-SOC\r\nWhousescannner: xxxxx-SOC\r\nAccept-Encoding: gzip\r\n\r\nGET /index.php?target=db_sql.php%253f/../../../../../../../../etc/passwd HTTP/1.1\r\nHost: 1.1.1.1:7789\r\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36\r\nAccept: */*\r\nWhouserscanner: xxxxx-SOC\r\nWhousescannner: xxxxx-SOC\r\nAccept-Encoding: gzip\r\n\r\nGET /logfile?d=crossdomain.xml HTTP/1.1\r\nHost: 1.1.1.1:7789\r\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36\r\nAccept: */*\r\nWhouserscanner: xxxxx-SOC\r\nWhousescannner: xxxxx-SOC\r\nAccept-Encoding: gzip\r\n\r\nPOST /index/Api/post_curl HTTP/1.1\r\nHost: 1.1.1.1:7789\r\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36\r\nContent-Length: 30\r\nAccept: */*\r\nContent-Type: application/x-www-form-urlencoded\r\nWhouserscanner: xxxxx-SOC\r\nWhousescannner: xxxxx-SOC\r\nAccept-Encoding: gzip\r\n\r\nurl=file:///etc/passwd&data[]=GET /index.php?m=user&c=Users&a=logout&referurl=https://example.com HTTP/1.1\r\nHost: 1.1.1.1:7789\r\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36\r\nAccept: */*\r\nWhouserscanner: xxxxx-SOC\r\nWhousescannner: xxxxx-SOC\r\nAccept-Encoding: gzip\r\n\r\n"
    save_http(1, "", p)


def parse_argument():
    global args
    parser = argparse.ArgumentParser(description='Process suricata log.')
    parser.add_argument('-i', '--input', type=str, default='suricata.log',
                        help='input file name')
    parser.add_argument('-o', '--output-dir', type=str, default='output',
                        help='output directory name')
    parser.add_argument('-e', '--error-file', default='error.log',
                        help='error file name')
    args = parser.parse_args()


if __name__ == '__main__':
    parse_argument()

    # create output dir
    try:
        os.mkdir(args.output_dir)
    except FileExistsError:
        pass

    sid_payload = {}

    with open(args.input, 'r', encoding="utf-8") as f:
        while True:
            line = f.readline()
            if not line:
                break
            try:
                summary["total"] += 1

                tmp = json.loads(line)
                sid = tmp['alert']['signature_id']
                signature = tmp['alert']['signature']
                payload = tmp['payload']
                if "app_proto" not in tmp or tmp['app_proto'] != "http":
                    summary["non-http"] += 1
                    continue
                if sid in sid_payload:
                    summary["ignore"] += 1
                    continue

                payload = base64.b64decode(payload)
                if save_http(sid, signature, payload) is True:
                    summary["ok"] += 1
                    sid_payload[sid] = True

            except Exception as e:
                # print(str(e), line)
                save_abnormal_http(line, args.error_file)
                summary["error"] += 1
                continue
    print(summary)