Simple journald random logger service

docker-compose.yaml

---
version: "3.9"

services:
  random-logs-all-multiline:
    build:
      context: .
      dockerfile: Dockerfile.multiline
    labels:
      com.datadoghq.ad.logs: >-
        [{
          "source": "random",
          "service": "all-multiline",
          "log_processing_rules": [{
            "type": "multi_line",
            "name": "log_start_with_date",
            "pattern" : "\\d{4}-\\d{2}-\\d{2}\\s\\d{2}:\\d{2}:\\d{2},\\d{3}"
          }]
        }]
  random-logs-info-only:
    build:
      context: .
      dockerfile: Dockerfile.multiline
    labels:
      com.datadoghq.ad.logs: >-
        [{
          "source": "random",
          "service": "info-only",
          "log_processing_rules": [{
            "type": "multi_line",
            "name": "log_start_with_date",
            "pattern" : "\\d{4}-\\d{2}-\\d{2}\\s\\d{2}:\\d{2}:\\d{2},\\d{3}"
          },
          {
            "type": "include_at_match",
            "name": "include_only_warn",
            "pattern": "INFO"
          }]
        }]
  random-logs-warning-and-error-only:
    build:
      context: .
      dockerfile: Dockerfile.multiline
    labels:
      com.datadoghq.ad.logs: >-
        [{
          "source": "random",
          "service": "info-only",
          "log_processing_rules": [{
            "type": "multi_line",
            "name": "log_start_with_date",
            "pattern" : "\\d{4}-\\d{2}-\\d{2}\\s\\d{2}:\\d{2}:\\d{2},\\d{3}"
          },
          {
            "type": "include_at_match",
            "name": "include_only_warn",
            "pattern": "WARNING|ERROR"
          }]
        }]

Dockerfile

# syntax=docker/dockerfile:1

FROM python:3.11-alpine

COPY random-logger.py /usr/bin/

ENTRYPOINT [ "python" ]

CMD [ "/usr/bin/random-logger.py" ]

Dockerfile.multiline

# syntax=docker/dockerfile:1

FROM python:3.11-alpine

COPY random-multiline-logger.py /usr/bin/

ENTRYPOINT [ "python" ]

CMD [ "/usr/bin/random-multiline-logger.py" ]

random-logger-with-log-rorate.py

#!/usr/bin/env python3

import logging
import random
import threading
import time
import concurrent.futures

from logging.handlers import RotatingFileHandler

_charset = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"


class Info:
    def __init__(self):
        self.value = {'stop': False}
        self._lock = threading.Lock()

    def stop(self):
        with self._lock:
            self.value['stop'] = True

    def not_stopped(self):
        self._lock.acquire(timeout=20)
        try:
            answer = self.value['stop']
        finally:
            self._lock.release()

        return not answer


def worker(arg):
    while arg.not_stopped():
        rnd = ''.join(random.sample(_charset*9, 9))
        match random.choice(['debug', 'info', 'warn', 'error']):
            case 'debug':
                logging.debug(f'This is a debug log that shows a log that can be ignored. {rnd}',
                              extra={'identifier': f'{rnd}'})
            case 'info':
                logging.info(f'This is less important than debug log and is often used to provide context in the current task. {rnd}',
                             extra={'identifier': f'{rnd}'})
            case 'warn':
                logging.warning(f'A warning that should be ignored is usually at this level and should be actionable. {rnd}',
                                extra={'identifier': f'{rnd}'})
            case 'error':
                logging.error(f'An error is usually an exception that has been caught and not handled. {rnd}',
                              extra={'identifier': f'{rnd}'})

        time.sleep(random.uniform(0, 5))


def main():
    handler = RotatingFileHandler("bob.log", maxBytes=1000000,
                                  backupCount=5)
    logging.basicConfig(
        handlers=[handler],
        format='{"timestamp":"%(asctime)s","severity":"%(levelname)s","message":"%(message)s", "context":{"identifier":"%(identifier)s", "thread":"%(threadName)s"}}',
        level=logging.DEBUG)
    info = Info()

    with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
        futures = [
            executor.submit(worker, info) for i in range(1, 15)
        ]

        while True:
            try:
                time.sleep(5)
            except KeyboardInterrupt:
                info.stop()
                break

        executor.shutdown()


if __name__ == '__main__':
    main()

random-logger.py

#!/usr/bin/env python3

import logging
import random
from time import sleep

logging.basicConfig(format='%(asctime)s | %(levelname)s | %(message)s', level=logging.DEBUG)

while True:
    match random.choice(['debug', 'info', 'warn', 'error']):
        case 'debug':
            logging.debug('This is a debug log that shows a log that can be ignored."')
        case 'info':
            logging.info('This is less important than debug log and is often used to provide context in the current task.')
        case 'warn':
            logging.warning('A warning that should be ignored is usually at this level and should be actionable.')
        case 'error':
            logging.error('An error is usually an exception that has been caught and not handled.')

    sleep(random.uniform(0, 5))

random-logger.service

[Unit]
Description=Random logger

[Service]
ExecStart=/usr/bin/random-logger.py

[Install]
WantedBy=multi-user.target

random-multiline-logger-other.py

#!/usr/bin/env python3

import argparse
import logging
import random
from time import sleep

parser = argparse.ArgumentParser(description='Random Multiline logger.')
parser.add_argument('--log-file', dest='log_filename', default=None, help='Set logger to output to a logfile rather than stout')

args = parser.parse_args()

logging.basicConfig(filename=args.log_filename, format='%(message)s', level=logging.DEBUG)
mu = 4
sigma = 2
upper = mu + sigma

# For log level
alpha = 1
beta = 1.5
levels = ['debug', 'info', 'warn', 'error', 'critical']

while True:
    temp = random.gauss(mu, sigma)
    not_multiline = sigma <= temp <= upper

    temp = round(random.weibullvariate(alpha, beta))
    if temp >= len(levels):
        temp = len(levels) -1

    level = levels[temp]
    match level:
        case 'debug':
            if not_multiline:
                logging.debug('This is a debug log that shows a log that can be ignored."')
            else:
                logging.debug('bbx This is a \ndebug log that \nshows a log that \ncan be ignored."')
        case 'info':
            if not_multiline:
                logging.info('This is less important than debug log and is often used to provide context in the current task.')
            else:
                logging.info('ccc This is less \nimportant than debug log \nand is often used to provide context \nin the current task.')
        case 'warn':
            if not_multiline:
                logging.warning('A warning that should be ignored is usually at this level and should be actionable.')
            else:
                logging.warning('cycc A warning \nthat should be ignored is \nusually at this level and should be actionable.')
        case 'error':
            if not_multiline:
                logging.error('An error is usually an exception that has been caught and not handled.')
            else:
                logging.error('mcg An error is \nusually an exception that \nhas been caught and not handled.')
        case 'critical':
            if not_multiline:
                logging.critical('A critical error is usually an exception that has not been caught or handled.')
            else:
                logging.critical('bbx A critical error is \nusually an exception that \nhas not been caught or handled.')

    sleep(random.uniform(0, 5))

random-multiline-logger-other.sh

#!/usr/bin/env sh

# The script can be run as different user by:
# sudo runuser -u dd-agent -- ./random-multiline-logger-other.sh
# The following needs to be added to /etc/datadog-agent/datadog.yaml
# and random-multiline-logger.yaml needs to be copied to /etc/datadog-agent/conf.d/
# Also make sure that /var/log/test exists and is owned by `dd-agent`.
#
# logs_config:
#   container_collect_all: true
#   auto_multi_line_detection: true
#   auto_multi_line_default_match_threshold: 0.1
#   auto_multi_line_extra_patterns:
#     - (bbx|ccc|cycc|mcg)

python3 -u random-multiline-logger-other.py --log-file=/var/log/test/random-multiline-logger.log

random-multiline-logger.py

#!/usr/bin/env python3

import argparse
import logging
import random
from time import sleep

parser = argparse.ArgumentParser(description='Random Multiline logger.')
parser.add_argument('--log-file', dest='log_filename', default=None, help='Set logger to output to a logfile rather than stout')

args = parser.parse_args()

logging.basicConfig(filename=args.log_filename, format='%(asctime)s | %(levelname)s | %(message)s', level=logging.DEBUG)
mu = 4
sigma = 2
upper = mu + sigma

# For log level
alpha = 1
beta = 1.5
levels = ['debug', 'info', 'warn', 'error', 'critical']

while True:
    temp = random.gauss(mu, sigma)
    not_multiline = sigma <= temp <= upper

    temp = round(random.weibullvariate(alpha, beta))
    if temp >= len(levels):
        temp = len(levels) -1

    level = levels[temp]
    match level:
        case 'debug':
            if not_multiline:
                logging.debug('This is a debug log that shows a log that can be ignored."')
            else:
                logging.debug('This is a \ndebug log that \nshows a log that \ncan be ignored."')
        case 'info':
            if not_multiline:
                logging.info('This is less important than debug log and is often used to provide context in the current task.')
            else:
                logging.info('This is less \nimportant than debug log \nand is often used to provide context \nin the current task.')
        case 'warn':
            if not_multiline:
                logging.warning('A warning that should be ignored is usually at this level and should be actionable.')
            else:
                logging.warning('A warning \nthat should be ignored is \nusually at this level and should be actionable.')
        case 'error':
            if not_multiline:
                logging.error('An error is usually an exception that has been caught and not handled.')
            else:
                logging.error('An error is \nusually an exception that \nhas been caught and not handled.')
        case 'critical':
            if not_multiline:
                logging.critical('A critical error is usually an exception that has not been caught or handled.')
            else:
                logging.critical('A critical error is \nusually an exception that \nhas not been caught or handled.')

    sleep(random.uniform(0, 5))

random-multiline-logger.sh

#!/usr/bin/env sh

python3 -u random-multiline-logger.py --log-file=/var/log/test/random-multiline-logger.log

random-multiline-logger.yaml

---
logs:
  - type: file
    path: /var/log/test/random-multiline-logger.log
    service: random-multiline-logger
    source: python

• Copy random-logger.py to /usr/bin/
• Copy random-logger.service to /etc/systemd/system/
• Run systemctl daemon-reload
• To enable and start the new service run systemctl enable --now random-logger.service
• For multiple instance of this random logger service
  • just copy random-logger.service to /etc/systemd/system/ and name it random-logger-001.service, random-logger-002.service, etc.
  • To enable (after systemctl daemon-reload) it would just be systemctl enable --now random-logger-001.service, systemctl enable --now random-logger-002.service, etc.
  • You might want to cut down the number of journald logs you keep. You can do this by running journalctl --rotate --vacuum-time=1h. This will remove any logs older than an hour and will also make sure journald only keeps logs no older than one hour.