Django logging filter to throttle repeated messages

log_settings.py

# Example logging configuration that will restrict console logging to 
# at most 2 repeated messages per 30 seconds. 
LOGGING = {
    'version': 1,
    'disable_existing_loggers': True,
    'formatters': {
        'simple': {
            'format': '%(asctime)s - %(name)s - %(levelname)s - %(filename)s:%(lineno)d - %(message)s'
        },
    },
    'filters': {
    	'time_throttled': {
    		'()': 'myproject.timethrottledfilter.TimeThrottledFilter',
    		'quantity': 2,
    		'interval': 30,
    		'ignore_lines': [
    			('myproject.middleware', 'process_exception'),
    			('django.request', 'handle_uncaught_exception'),
    		],
    	},
    },
    'handlers': {
        'console': {
            'level': 'DEBUG',
            'class': 'logging.StreamHandler',
            'stream': 'ext://sys.stdout',
            'formatter': 'simple',
            'filters': ['time_throttled'],
        },
    'loggers': {
        'myproject.app': {
            'handlers': ['console'],
            'level': 'WARNING',
            'propagate': True,
        },
        'django.request': {
            'handlers': ['console'],
            'level': 'ERROR',
            'propagate': False,
        },
    },
}

timethrottledfilter.py

import hashlib
import time

class TimeThrottledFilter(logging.Filter):
	"""
	A logging filter that only emits a certain number of similar exceptions per
	interval (in seconds).
	"""
	def __init__(self, quantity=10, interval=60, ignore_lines=None, throttle_users=False):
		"""
		Throttle logging of a message after encountering 'quantity' within
		'interval' seconds.

		'ignore_lines' contains a list of tuples that
		specify functions and specific lines that should not be used when
		determining if a message is a duplicate that should be throttled.
		The tuples may be of the form (logger name, func name) or
		(logger name, func name, line #).

		'throttle_users' indicates if all messages generated by a specific user/IP
		should lumped together and subject to throttling.
		"""
		self.quantity = quantity
		self.interval = interval
		self.ignore_lines = ignore_lines
		self.throttle_users = throttle_users

	def make_keys(self, record):
		"""
		Build a list of keys that describe the record.
		"""
		keys = []
		if record.exc_info:
			name = record.exc_info[0].__name__
			if name == 'Exception':
				name = repr(record.exc_info[1])

			keys.append('{name}'.format(
					name=name,
				)
			)

		# specific line of code
		skip_line_key = False
		if self.ignore_lines:
			for line in self.ignore_lines:
				if record.name == line[0] and record.funcName == line[1]:
					if len(line) == 2 or (len(line) == 3 and line[2] == record.lineno):
						skip_line_key = True
						break

		if not skip_line_key:
			keys.append('{name}:{func}:{lineno}'.format(
					name=record.name,
					func=record.funcName,
					lineno=record.lineno,
				)
			)

		if self.throttle_users and hasattr(record, 'request'):
			request = record.request
			# specific user or IP
			if request:
				user = request.user.username if hasattr(request, 'user') else request.META.get('REMOTE_ADDR', None)

			if user:
				keys.append('{user}'.format(
						user=user,
					)
				)

		time_bucket = int(time.time()//self.interval) & 0xffff

		return ['throttle-{0}:{1}'.format(time_bucket, hashlib.md5(k).hexdigest()) for k in keys]

	def filter(self, record):
		# check if the record has already been processed by this filter
		handled = getattr(record, '_TimeThrottledFilter_emit', None)
		if handled is not None:
			return handled

		keys = self.make_keys(record)

		from django.core.cache import cache
		# get_many followed by incr is not atomic, but that's okay
		vals = cache.get_many(keys)

		emit = True
		for k in keys:
			if not vals.has_key(k):
				cache.add(k, 1)
			else:
				val = cache.incr(k)
				if val > self.quantity:
					emit = False

		# cache to prevent multiple handlings
		record._TimeThrottledFilter_emit = emit
		return emit

Thanks, this is great! Exactly what I needed...