How to install AWS CloudWatch Logs Agent to send logs from a docker container to AWS Cloudwatch, without using a linked container for logging.

aws.conf

[plugins]
cwlogs = cwlogs
[default]
region = ap-southeast-2

awslogs-agent.conf

[general]
# Path to the CloudWatch Logs agent's state file. The agent uses this file to maintain
# client side state across its executions.
state_file = /var/awslogs-agent-state

[/var/log/syslog.log]
datetime_format = %b %d %H:%M:%S
file = /var/log/syslog.log
buffer_duration = 5000
log_stream_name = {hostname}
initial_position = end_of_file
log_group_name = syslog

howto.md

Install CloudWatch Logs Agent

Install awscli-cwlogs python module. Amazon provides an interactive installer to do this, but it seems easier, and more repeatable to just install the module.

RUN pip install --trusted-host aws-cloudwatch.s3-website-us-east-1.amazonaws.com --extra-index-url=http://aws-cloudwatch.s3-website-us-east-1.amazonaws.com/ awscli-cwlogs==1.3.3

Configure CloudWatch Logs Agent

Add aws.conf file and awslogs.conf file to /etc/

Configure rsyslog

All applications should be setup to log to syslog, syslog will be configured to write all logs to a single file, the file will be rotated every 25mb.

Add rsyslog.conf file to /etc/.

Add script to remove old log files, remove-old-logs.sh, to /opt/bin/

Configure supervisor to start application, rsyslogd, and Cloudwatch Logs Agent

add start-awslogs.conf, and start-rsyslog.conf to /opt/supervisor/

Configure a supervisor conf file to start the application, then includ the conf files for starting rsyslog and CloudWatch Logs Agent. See web.conf for an example.

Configure Docker to run supervisor

add run_supervisord.sh to /opt/bin/

Set CMD in dockerfile

CMD ["/opt/bin/run_supervisord.sh", "web"]

remove-old-logs.sh

#!/bin/sh
mv -f /var/log/syslog.log /var/log/syslog.log.1

rsyslog.conf

$ModLoad imuxsock # provides support for local system logging

# Keep repeated messages
$RepeatedMsgReduction off

# Set the default permissions for all log files.
$FileOwner syslog
$FileGroup adm
$FileCreateMode 0640
$DirCreateMode 0755
$Umask 0022
$PrivDropToUser syslog
$PrivDropToGroup syslog

# Where to place spool and state files
$WorkDirectory /var/spool/rsyslog

#setup a format without hostname, and with time
$template NoHostnameFormat,"%timestamp% %syslogtag%%msg:::drop-last-lf%\n"

#setup a channel that limits file size
$outchannel log_rotation,/var/log/syslog.log,52428800,/opt/bin/remove-old-logs.sh

# Setup all logs to go to the channel
*.*    :omfile:$log_rotation;NoHostnameFormat

run_supervisord.sh

#!/bin/bash

# Runs supervisord, tailing any logfiles

set -eu

CONF_FILE=/opt/supervisor/$1.conf
TAIL_LOGS=${TAIL_LOGS:-""}


case "$TAIL_LOGS" in
    "true"|"True"|"yes")
    (umask 0 && truncate -s0 /var/log/syslog.log)
    tail -q --pid $$ -n0 -F /var/log/syslog.log &
    ;;
esac

exec /usr/bin/supervisord -c $CONF_FILE -n

start-awslogs.conf

[program:awslogs]
command=/usr/bin/env -i AWS_CONFIG_FILE=/etc/aws.conf /usr/local/bin/aws logs push --config-file /etc/awslogs.conf
numprocs=1
autostart=true
autorestart=true
stdout_logfile=/dev/null
stderr_logfile=/dev/null

start-rsyslog.conf

[program:rsyslog]
command=/usr/sbin/rsyslogd -n
numprocs=1
autostart=true
autorestart=true
stdout_logfile=syslog
stderr_logfile=syslog

web.conf

[supervisord]
nodaemon=true

[program:nginx]
command=/usr/sbin/nginx -c /opt/nginx/nginx.conf
autostart=True
autorestart=True
startsecs=2
stdout_logfile=syslog
stderr_logfile=syslog

[program:django_gunicorn]
environment=PYTHONPATH="/opt/apps/gaia/contrib",LOGGING_SYSLOG="True"
command=/opt/ve/gaia/bin/gunicorn wsgi -b 0.0.0.0:8010 -w 3 -k 'sync' --timeout=120
directory=/opt/apps/gaia/
user=gaia
autostart=True
autorestart=True
startsecs=3
stdout_logfile=syslog
stderr_logfile=syslog

[include]
files=/opt/supervisor/start-rsyslog.conf /opt/supervisor/start-awslogs.conf