Delete empty log streams or logs streams that have only elements older than 30 days in it

aws_cleanup_cloudwatch.go

package main

// This script cleans up old LogGroups (empty and olde than 90 days and old
// LogStreams (last event timestamp is over 30 days old or if the logstream
// is empty and has been created over 30 days ago) from AWS Cloudwatch

import (
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
	"log"
	"sync"
	"time"
)

// CwProcessor holds the cloudwatch-related actions
type CwProcessor struct {
	svc    *cloudwatchlogs.CloudWatchLogs
	lsChan chan map[*string]*string
	lgChan chan *string
}

// NewCwProcessor creates a new instance of CwProcessor containing an already
// initialized cloudwatchlogs client
func NewCwProcessor() *CwProcessor {
	sess := session.Must(session.NewSessionWithOptions(session.Options{
		SharedConfigState: session.SharedConfigEnable,
	}))
	return &CwProcessor{svc: cloudwatchlogs.New(sess), lsChan: make(chan map[*string]*string), lgChan: make(chan *string)}
}

// ProcessLogStreams add the logstreams of the given loggroup to the lsChan
// channel if their last event timestamp is over 30 days old or if the logstream
// is empty and has been created over 30 days ago
func (p *CwProcessor) ProcessLogStreams(groupName *string) (int, error) {
	nbrLogStreams := 0
	err := p.svc.DescribeLogStreamsPages(&cloudwatchlogs.DescribeLogStreamsInput{LogGroupName: groupName},
		func(ls *cloudwatchlogs.DescribeLogStreamsOutput, lastPage bool) bool {
			nbrLogStreams = len(ls.LogStreams)
			for _, stream := range ls.LogStreams {
				lastTs := *stream.CreationTime / 1000
				if stream.LastEventTimestamp != nil {
					lastTs = *stream.LastEventTimestamp / 1000
				}
				if lastTs < time.Now().AddDate(0, 0, -30).Unix() {
					p.lsChan <- map[*string]*string{groupName: stream.LogStreamName}
				}
			}
			return !lastPage
		})
	return nbrLogStreams, err
}

// ProcessLogGroups does a cleanup of all LogGroups using the ProcessLogStreams
// function and add the Log Groups that have no logStream and are older than
// 90 days to the lgChan channel for deletion
func (p *CwProcessor) ProcessLogGroups() {
	err := p.svc.DescribeLogGroupsPages(&cloudwatchlogs.DescribeLogGroupsInput{},
		func(page *cloudwatchlogs.DescribeLogGroupsOutput, lastPage bool) bool {
			for _, lg := range page.LogGroups {
				nbrLs, err := p.ProcessLogStreams(lg.LogGroupName)
				if err != nil {
					log.Printf("[ERROR] Getting Log group %s streams: %v\n", *lg.Arn, err)
				} else {
					if nbrLs == 0 && (*lg.CreationTime/1000) < time.Now().AddDate(0, 0, -90).Unix() {
						p.lgChan <- lg.LogGroupName
					}
				}
			}

			return !lastPage
		})
	if err != nil {
		log.Fatalf("[ERROR] DescribeLogGroups returned: %v\n", err)
	}
	close(p.lsChan)
	close(p.lgChan)
}

// CleanupLogStreams delete the LogGroups that exist in the channel lsChan
func (p *CwProcessor) CleanupLogStreams() {
	for elt := range p.lsChan {
		for k, v := range elt {
			log.Printf("Deleting LogStream: %s from LogGroup: %s\n", *v, *k)
			if _, err := p.svc.DeleteLogStream(&cloudwatchlogs.DeleteLogStreamInput{LogGroupName: k, LogStreamName: v}); err != nil {
				log.Printf("[ERROR] Deleting LogGroup: %s, LogStream: %s --> %s\n", *k, *v, err.Error())
			}
		}
	}
}

// CleanupLogGroups delete the LogGroups that exist in the channel lgChan
func (p *CwProcessor) CleanupLogGroups() {
	for elt := range p.lgChan {
		log.Printf("Deleting LogGroup: %s\n", *elt)
		if _, err := p.svc.DeleteLogGroup(&cloudwatchlogs.DeleteLogGroupInput{LogGroupName: elt}); err != nil {
			log.Printf("[ERROR] Deleting LogGroup: %s --> %s\n", *elt, err.Error())
		}
	}
}

func main() {
	p := NewCwProcessor()
	wg := sync.WaitGroup{}
	wg.Add(2)
	go func() {
		p.CleanupLogStreams()
		wg.Done()
	}()
	go func() {
		p.CleanupLogGroups()
		wg.Done()
	}()

	p.ProcessLogGroups()
	wg.Wait()
}

Here's a CloudFormation template that provisions a Lambda function to each night delete all empty LogStreams for LogGroups with retention settings : https://github.com/gene1wood/delete-empty-cloudwatch-logstreams

This is nice as all you have to do is deploy the CloudFormation stack and it takes care of it from that point on.