Pull down cloudwatch logs with boto

get_cloudwatch_logs.py

# IF YOU INCUR HUGE COSTS WITH THIS OR IT BREAKS DON'T BLAME ME License

# This is a throw-away script I wrote to pull the json events for all of the streams from a cloudwatch log
# For some reason, the naive way to do vpc network logging does logging to different streams in a cloudwatch
# log based on interface.
# Great for diagnosing lots of things, and generating verbose logs, but for the broad-stroke analysis I was doing,
# all I really wanted was the basic data. This would have been easier if I had logged to s3, but I did not see a
# way to do that in 2 clicks.

group_name = 'CHANGEME'
import boto3, json, time
client = boto3.client('logs')
all_streams = []

stream_batch = client.describe_log_streams(logGroupName=group_name)
all_streams += stream_batch['logStreams']
while 'nextToken' in stream_batch:
	stream_batch = client.describe_log_streams(logGroupName=group_name,nextToken=stream_batch['nextToken'])
	all_streams += stream_batch['logStreams']
	print(len(all_streams))

stream_names = [stream['logStreamName'] for stream in all_streams]
out_to = open(group_name + str(time.time()) + "cloud_logs.txt", 'w')
for stream in stream_names:
	logs_batch = client.get_log_events(logGroupName=group_name, logStreamName=stream)
	for event in logs_batch['events']:
		event.update({'group': group_name, 'stream':stream })
		out_to.write(json.dumps(event) + '\n')
	print(stream, ":", len(logs_batch['events']))
	while 'nextToken' in logs_batch:
		logs_batch = client.get_log_events(logGroupName=group_name, logStreamName=stream, nextToken=logs_batch['nextToken'])
		for event in logs_batch['events']:
			event.update({'group': group_name, 'stream':stream })
			out_to.write(json.dumps(event) + '\n')

Works perfectly, cheers

Note that you can use the filter_log_events method to search all streams.

Very helpful, thanks!

You will need to add

logs_batch = client.get_log_events(logGroupName=group_name, logStreamName=stream)
logs_batch = client.get_log_events(logGroupName=group_name, logStreamName=stream, startFromHead=True)
                                                                                                                                                         ----------------------

other wise the result wont be complete. check the docs https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/logs.html#CloudWatchLogs.Client.get_log_events