6d61726b760a · June 12, 2020 06:41
diff --git a/README.md b/README.md
diff --git a/get_trails.py b/get_trails.py
 import argparse
 import boto3
 import dateparser
 import datetime
 import logging
 import os
 import pytz
 import sys

 def parse_time_string(time_string):
    """thanks: https://github.com/flosell/trailscraper/blob/master/trailscraper/s3_download.py"""
    """Parse human readable strings (e.g. "now", "2017-01-01" and "one hour ago") into datetime"""
    return dateparser.parse(time_string, settings={'RETURN_AS_TIMEZONE_AWARE': True})


 def _s3_key_prefix(prefix, date, account_id, region):
    """thanks: https://github.com/flosell/trailscraper/blob/master/trailscraper/s3_download.py"""
    return f"{prefix}/AWSLogs/{account_id}/CloudTrail/{region}/{date.year}/{date.month:02d}/{date.day:02d}"


 def _s3_key_prefixes(prefix, account_ids, regions, from_date, to_date):
    """thanks: https://github.com/flosell/trailscraper/blob/master/trailscraper/s3_download.py"""
    delta = to_date.astimezone(pytz.utc) - from_date.astimezone(pytz.utc)
    days = [to_date - datetime.timedelta(days=delta_days)
            for delta_days in range(delta.days + 1)]
    return [_s3_key_prefix(prefix, day, account_id, region)
            for account_id in account_ids
            for day in days
            for region in regions]


 def _s3_download_recursive(client, bucket, prefix, target_dir):
    """thanks: https://github.com/flosell/trailscraper/blob/master/trailscraper/s3_download.py"""
    client = client

    def _download_file(object_info):
        key = object_info.get('Key')
        target = target_dir + os.sep + key
        if not os.path.exists(os.path.dirname(target)):
            os.makedirs(os.path.dirname(target))
        if not os.path.exists(target):
            logging.info(f"downloading {bucket}/{key}")
            client.download_file(bucket, key, target)
        else:
            logging.info(f"skipping, already exists: {bucket}/{key}")

    def _download_dir(dist):
        paginator = client.get_paginator('list_objects')
        for result in paginator.paginate(Bucket=bucket, Prefix=dist):
            if result.get('CommonPrefixes') is not None:
                for subdir in result.get('CommonPrefixes'):
                    _download_dir(subdir.get('Prefix'))
            if result.get('Contents') is not None:
                for content in result.get('Contents'):
                    _download_file(content)
    _download_dir(prefix)


 def main():
    scriptname = sys.argv[0]

    logging.basicConfig(
        level=logging.INFO,
        format="%(asctime)s [%(levelname)s] %(message)s",
        handlers=[
            logging.FileHandler(f"{sys.argv[0]}.log"),
            logging.StreamHandler()
        ]
    )

    parser = argparse.ArgumentParser(
        description='download cloudtrail logs from s3',
        epilog="example date strings:\n"+
               "    --from 'one day ago' --to 'now'\n"+
               "    --from 'two weeks ago' --to 'one week ago'\n"+
               "    --from '2019-10-05' --to '2019-10-19'\n"+
               "    --from 'today'",
        formatter_class=argparse.RawTextHelpFormatter
    )
    parser.add_argument("--profile",
                        help="the aws named profile to use",
                        action="store",
                        default="default",
                        required=True
                        )
    parser.add_argument("--bucket",
                        help="s3 bucket name",
                        action="store",
                        required=True
                        )
    parser.add_argument("--prefix",
                        help="s3 bucket name",
                        action="store",
                        default=None,
                        required=True
                        )
    parser.add_argument("--account",
                        help="aws account\ncan be specified multiple times",
                        action="append",
                        default=None,
                        required=True
                        )
    parser.add_argument("--region",
                        help="aws region\ncan be specified multiple times",
                        action="append",
                        default=None,
                        required=True
                        )
    parser.add_argument("--from",
                        dest="from_s",
                        help="start date\ndefault: 'one day ago'",
                        action="store",
                        default="one day ago"
                        )
    parser.add_argument("--to",
                        dest="to_s",
                        help="end date\ndefault: 'now'",
                        action="store",
                        default="now"
                        )
    parser.add_argument("--target_dir",
                        help="destination directory\ndefault: ./trails/",
                        action="store",
                        default="trails"
                        )

    args = parser.parse_args()
    input_args = vars(args)

    [logging.info(f"{arg}: {input_args[arg]}") for arg in input_args]

    session = boto3.session.Session(profile_name=args.profile)
    s3 = session.client("s3")
    
    from_date = parse_time_string(args.from_s)
    to_date = parse_time_string(args.to_s)
    
    logging.info(f"parsed start date: {from_date}")
    logging.info(f"parsed end date: {to_date}")
    
    prefixes = _s3_key_prefixes(
        prefix=args.prefix,
        account_ids=args.account,
        regions=args.region,
        from_date=from_date,
        to_date=to_date)
    
    logging.info(f"found {len(prefixes)} prefixes for download")
    
    [logging.debug(f"prefix: {prefix}") for prefix in prefixes]

    for prefix in prefixes:
        _s3_download_recursive(client=s3, bucket=args.bucket,
                               prefix=prefix, target_dir=args.target_dir)


 if __name__ == "__main__":
    main()
diff --git a/parse_trails.py b/parse_trails.py
 import argparse
 import glob
 import gzip
 import json
 import logging
 import os
 import pyjq
 import requests
 import sys

 def splunk(payload,identifier):
    """ send payload to splunk it to splunk hec """
    splunk_token = (os.getenv('SPLUNK_TOKEN'))
    if not splunk_token:
        raise ValueError(f'unable to find SPLUNK_TOKEN envvar')

    splunk_endpoint = (os.getenv('SPLUNK_ENDPOINT'))
    if not splunk_token:
        raise ValueError(f'unable to find SPLUNK_ENDPOINT envvar')

    splunkAuth = {'Authorization': 'Splunk ' + splunk_token}
    r = requests.post(splunk_endpoint, headers=splunkAuth, json=payload)
    logging.info(f"splunk: {identifier} [HTTP {r.status_code}]")

 def main():
    scriptname = sys.argv[0]

    logging.basicConfig(
        level=logging.INFO,
        format="%(asctime)s [%(levelname)s] %(message)s",
        handlers=[
            # logging.FileHandler(f"{sys.argv[0]}.log"),
            logging.StreamHandler()
        ]
    )

    parser = argparse.ArgumentParser(
        description='query cloudtrail logs in target directory using jq syntax',
        epilog="default action is to display a count of matching events per file\n\n"+
               "this utility expects cloudtrail logs to be gzipped (no need to unzip logs downloaded from s3)\n\n"+
               "if using --splunk the SPLUNK_TOKEN and SPLUNK_ENDPOINT envvar must be set. eg:\n"+
               "   export SPLUNK_TOKEN='XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX\n"
               "   export SPLUNK_ENDPOINT='https://hec-input.splunkcloud.com:443/services/collector/event\n\n",
        formatter_class=argparse.RawTextHelpFormatter
    )
    parser.add_argument("--query",
                        help="jq query to run against found trails",
                        action="store",
                        default="default"
                        )
    parser.add_argument("--target_dir",
                        help="directory to look for trails\ndefault: ./trails/",
                        action="store",
                        default="trails"
                        )
    parser.add_argument("--splunk",
                        help = "send result to splunk",
                        action = "store_true",
                        default = False
                        )
    parser.add_argument("--print",
                    help="print matching events to console",
                    action="store_true",
                    default=False
                    )

    args = parser.parse_args()
    input_args = vars(args)

    [logging.info(f"{arg}: {input_args[arg]}") for arg in input_args]

    files = glob.glob(args.target_dir + '/**/*.gz', recursive=True)

    logging.info(f"found {len(files)} files to query")

    for file in files:
        filename = os.path.basename(file)
        with gzip.open(file, 'rb') as f:
            data = json.load(f)
            result = pyjq.all(args.query, data)
            
            if len(result) >0:
                logging.info(f"found {len(result)} matching events in {filename}")
                for event in result:
                    if args.splunk:
                        payload = {
                            "index": "markv_testing",
                            "sourcetype": "cloudtrail:event",
                            "event": event
                        }
                        splunk(payload,event['eventID'])
                    if args.print:
                        print(f"{json.dumps(event)}\n")

 if __name__ == "__main__":
    main()
diff --git a/requirements.txt b/requirements.txt
 boto3==1.14.0
 dateparser==0.7.5
 pyjq==2.4.0requests==2.23.0
 requests==2.23.0
	import argparse
	import boto3
	import dateparser
	import datetime
	import logging
	import os
	import pytz
	import sys

	def parse_time_string(time_string):
	"""thanks: https://github.com/flosell/trailscraper/blob/master/trailscraper/s3_download.py"""
	"""Parse human readable strings (e.g. "now", "2017-01-01" and "one hour ago") into datetime"""
	return dateparser.parse(time_string, settings={'RETURN_AS_TIMEZONE_AWARE': True})


	def _s3_key_prefix(prefix, date, account_id, region):
	"""thanks: https://github.com/flosell/trailscraper/blob/master/trailscraper/s3_download.py"""
	return f"{prefix}/AWSLogs/{account_id}/CloudTrail/{region}/{date.year}/{date.month:02d}/{date.day:02d}"


	def _s3_key_prefixes(prefix, account_ids, regions, from_date, to_date):
	"""thanks: https://github.com/flosell/trailscraper/blob/master/trailscraper/s3_download.py"""
	delta = to_date.astimezone(pytz.utc) - from_date.astimezone(pytz.utc)
	days = [to_date - datetime.timedelta(days=delta_days)
	for delta_days in range(delta.days + 1)]
	return [_s3_key_prefix(prefix, day, account_id, region)
	for account_id in account_ids
	for day in days
	for region in regions]


	def _s3_download_recursive(client, bucket, prefix, target_dir):
	"""thanks: https://github.com/flosell/trailscraper/blob/master/trailscraper/s3_download.py"""
	client = client

	def _download_file(object_info):
	key = object_info.get('Key')
	target = target_dir + os.sep + key
	if not os.path.exists(os.path.dirname(target)):
	os.makedirs(os.path.dirname(target))
	if not os.path.exists(target):
	logging.info(f"downloading {bucket}/{key}")
	client.download_file(bucket, key, target)
	else:
	logging.info(f"skipping, already exists: {bucket}/{key}")

	def _download_dir(dist):
	paginator = client.get_paginator('list_objects')
	for result in paginator.paginate(Bucket=bucket, Prefix=dist):
	if result.get('CommonPrefixes') is not None:
	for subdir in result.get('CommonPrefixes'):
	_download_dir(subdir.get('Prefix'))
	if result.get('Contents') is not None:
	for content in result.get('Contents'):
	_download_file(content)
	_download_dir(prefix)


	def main():
	scriptname = sys.argv[0]

	logging.basicConfig(
	level=logging.INFO,
	format="%(asctime)s [%(levelname)s] %(message)s",
	handlers=[
	logging.FileHandler(f"{sys.argv[0]}.log"),
	logging.StreamHandler()
	]
	)

	parser = argparse.ArgumentParser(
	description='download cloudtrail logs from s3',
	epilog="example date strings:\n"+
	" --from 'one day ago' --to 'now'\n"+
	" --from 'two weeks ago' --to 'one week ago'\n"+
	" --from '2019-10-05' --to '2019-10-19'\n"+
	" --from 'today'",
	formatter_class=argparse.RawTextHelpFormatter
	)
	parser.add_argument("--profile",
	help="the aws named profile to use",
	action="store",
	default="default",
	required=True
	)
	parser.add_argument("--bucket",
	help="s3 bucket name",
	action="store",
	required=True
	)
	parser.add_argument("--prefix",
	help="s3 bucket name",
	action="store",
	default=None,
	required=True
	)
	parser.add_argument("--account",
	help="aws account\ncan be specified multiple times",
	action="append",
	default=None,
	required=True
	)
	parser.add_argument("--region",
	help="aws region\ncan be specified multiple times",
	action="append",
	default=None,
	required=True
	)
	parser.add_argument("--from",
	dest="from_s",
	help="start date\ndefault: 'one day ago'",
	action="store",
	default="one day ago"
	)
	parser.add_argument("--to",
	dest="to_s",
	help="end date\ndefault: 'now'",
	action="store",
	default="now"
	)
	parser.add_argument("--target_dir",
	help="destination directory\ndefault: ./trails/",
	action="store",
	default="trails"
	)

	args = parser.parse_args()
	input_args = vars(args)

	[logging.info(f"{arg}: {input_args[arg]}") for arg in input_args]

	session = boto3.session.Session(profile_name=args.profile)
	s3 = session.client("s3")

	from_date = parse_time_string(args.from_s)
	to_date = parse_time_string(args.to_s)

	logging.info(f"parsed start date: {from_date}")
	logging.info(f"parsed end date: {to_date}")

	prefixes = _s3_key_prefixes(
	prefix=args.prefix,
	account_ids=args.account,
	regions=args.region,
	from_date=from_date,
	to_date=to_date)

	logging.info(f"found {len(prefixes)} prefixes for download")

	[logging.debug(f"prefix: {prefix}") for prefix in prefixes]

	for prefix in prefixes:
	_s3_download_recursive(client=s3, bucket=args.bucket,
	prefix=prefix, target_dir=args.target_dir)


	if __name__ == "__main__":
	main()
	import argparse
	import glob
	import gzip
	import json
	import logging
	import os
	import pyjq
	import requests
	import sys

	def splunk(payload,identifier):
	""" send payload to splunk it to splunk hec """
	splunk_token = (os.getenv('SPLUNK_TOKEN'))
	if not splunk_token:
	raise ValueError(f'unable to find SPLUNK_TOKEN envvar')

	splunk_endpoint = (os.getenv('SPLUNK_ENDPOINT'))
	if not splunk_token:
	raise ValueError(f'unable to find SPLUNK_ENDPOINT envvar')

	splunkAuth = {'Authorization': 'Splunk ' + splunk_token}
	r = requests.post(splunk_endpoint, headers=splunkAuth, json=payload)
	logging.info(f"splunk: {identifier} [HTTP {r.status_code}]")

	def main():
	scriptname = sys.argv[0]

	logging.basicConfig(
	level=logging.INFO,
	format="%(asctime)s [%(levelname)s] %(message)s",
	handlers=[
	# logging.FileHandler(f"{sys.argv[0]}.log"),
	logging.StreamHandler()
	]
	)

	parser = argparse.ArgumentParser(
	description='query cloudtrail logs in target directory using jq syntax',
	epilog="default action is to display a count of matching events per file\n\n"+
	"this utility expects cloudtrail logs to be gzipped (no need to unzip logs downloaded from s3)\n\n"+
	"if using --splunk the SPLUNK_TOKEN and SPLUNK_ENDPOINT envvar must be set. eg:\n"+
	" export SPLUNK_TOKEN='XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX\n"
	" export SPLUNK_ENDPOINT='https://hec-input.splunkcloud.com:443/services/collector/event\n\n",
	formatter_class=argparse.RawTextHelpFormatter
	)
	parser.add_argument("--query",
	help="jq query to run against found trails",
	action="store",
	default="default"
	)
	parser.add_argument("--target_dir",
	help="directory to look for trails\ndefault: ./trails/",
	action="store",
	default="trails"
	)
	parser.add_argument("--splunk",
	help = "send result to splunk",
	action = "store_true",
	default = False
	)
	parser.add_argument("--print",
	help="print matching events to console",
	action="store_true",
	default=False
	)

	args = parser.parse_args()
	input_args = vars(args)

	[logging.info(f"{arg}: {input_args[arg]}") for arg in input_args]

	files = glob.glob(args.target_dir + '/*/.gz', recursive=True)

	logging.info(f"found {len(files)} files to query")

	for file in files:
	filename = os.path.basename(file)
	with gzip.open(file, 'rb') as f:
	data = json.load(f)
	result = pyjq.all(args.query, data)

	if len(result) >0:
	logging.info(f"found {len(result)} matching events in {filename}")
	for event in result:
	if args.splunk:
	payload = {
	"index": "markv_testing",
	"sourcetype": "cloudtrail:event",
	"event": event
	}
	splunk(payload,event['eventID'])
	if args.print:
	print(f"{json.dumps(event)}\n")

	if __name__ == "__main__":
	main()
	boto3==1.14.0
	dateparser==0.7.5
	pyjq==2.4.0requests==2.23.0
	requests==2.23.0