Simple demonstration of using Linux cgroups to set limits on the number of processes

bash-child-cgroups.sh

#!/bin/bash

# PREREQUISITES: On Ubuntu 18.04 need to make sure that the "libcgroup1" and "cgroup-tools" packages are installed

MAXNUMPROCS=11
MAXNUMCHILDPROCS=8
PARENTGROUP=sandwich

# Create the parent cgroup
##OPTIONAL: This part should be already there on more recent Linux distributions
# mkdir -p /sys/fs/cgroups/pids
# mount -t cgroup -o pids none /sys/fs/cgroup/pids

mkdir -p /sys/fs/cgroup/pids/parent

# Set the parent process limit
echo $$ > /sys/fs/cgroup/pids/$PARENTGROUP/cgroup.procs
echo $MAXNUMPROCS > /sys/fs/cgroup/pids/$PARENTGROUP/pids.max

# Create child group A
mkdir -p /sys/fs/cgroup/pids/$PARENTGROUP/avocado
# Set group A process limit
echo $$ > /sys/fs/cgroup/pids/$PARENTGROUP/avocado/cgroup.procs
echo $MAXNUMCHILDPROCS > /sys/fs/cgroup/pids/$PARENTGROUP/avocado/pids.max

# Create child group B
mkdir -p /sys/fs/cgroup/pids/$PARENTGROUP/bacon
# Set group B process limit
echo $$ > /sys/fs/cgroup/pids/$PARENTGROUP/bacon/cgroup.procs
echo $MAXNUMCHILDPROCS > /sys/fs/cgroup/pids/$PARENTGROUP/bacon/pids.max

# Switch back to parent cgroup
echo $$ > /sys/fs/cgroup/pids/$PARENTGROUP/cgroup.procs

# Run 8 processes in child group A
for i in {1..8}
do
	# NOTE: The below starts two separate processes
	#cgexec -g pids:$PARENTGROUP/avocado echo "Starting child $i in group A"
	echo "Starting child $i in group A"
	cgexec -g pids:$PARENTGROUP/avocado sleep 30 &
done

# Run 2 processes in child group B
for i in {1..2}
do
	#cgexec -g pids:$PARENTGROUP/avocado echo "Starting child $i in group B" && \
	echo "Starting child $i in group B"
	cgexec -g pids:$PARENTGROUP/bacon sleep 30 &
done

# Try to run extra process in group A (this should fail)
echo "Starting extra child in group A this should fail" 
cgexec -g pids:$PARENTGROUP/avocado sleep 30 &

python-sqs-listener-cgroups.py

#!/usr/bin/python3

import subprocess
import os
import boto3

# Create SQS client
sqs = boto3.client('sqs')

queue_url = 'https://sqs.ap-southeast-2.amazonaws.com/560616450834/testing-workers'

while True:
  # Receive message from SQS queue
  response = sqs.receive_message(
      QueueUrl=queue_url,
      AttributeNames=[
          'SentTimestamp'
      ],
      MaxNumberOfMessages=1,
      MessageAttributeNames=[
          'All'
      ],
      VisibilityTimeout=0,
      WaitTimeSeconds=10
  )

  if 'Messages' in response:
    #message = response['Messages'][0]
    #receipt_handle = message['ReceiptHandle']

    for message in response['Messages']:
      print("Spawning worker for message " + str(message))
      process = subprocess.Popen("cgexec -g pids:sandwich/bacon sleep 30", shell=True)
      #process.wait()
      #print(process.stdout)
      #print(process.stderr)
  else:
    print("No messages in queue")

print("All done")