-
-
Save andraantariksa/9ba87af839f792da307d081a2aa2c865 to your computer and use it in GitHub Desktop.
StopwatchTestRunner
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import time | |
import statistics | |
import unittest | |
from unittest import TextTestRunner | |
from django.test.runner import DiscoverRunner | |
class StopwatchTestResult(unittest.TextTestResult): | |
""" | |
Times test runs and formats the result | |
""" | |
# Collection shared between all result instaces to calculate statistics | |
timings = {} | |
def __init__(self, *args, **kwargs): | |
super().__init__(*args, **kwargs) | |
self.start = 0 | |
self.stop = 0 | |
self.elapsed = 0 | |
def startTest(self, test): | |
self.start = time.time() | |
super().startTest(test) | |
def stopTest(self, test): | |
super().stopTest(test) | |
self.stop = time.time() | |
self.elapsed = self.stop - self.start | |
self.timings[test] = self.elapsed | |
def getDescription(self, test): | |
""" | |
Format test result with timing info | |
e.g. `test_add [0.1s]` | |
""" | |
description = super().getDescription(test) | |
return f'{description} [{self.elapsed:0.4f}s]' | |
@classmethod | |
def print_stats(cls): | |
""" | |
Calculate and print timings | |
These data are likely skewed, as is normal for reaction time data, | |
therefore mean and standard deviation are difficult to interpret. Thus, | |
the IQR is used to identify outliers. | |
""" | |
timings = StopwatchTestResult.timings.values() | |
count = len(timings) | |
mean = statistics.mean(timings) | |
stdev = statistics.stdev(timings) | |
slowest = max(timings) | |
q1, median, q3 = statistics.quantiles(timings) | |
fastest = min(timings) | |
total = sum(timings) | |
print() | |
print("Statistics") | |
print("==========") | |
print("") | |
print(f"count: {count:.0f}") | |
print(f" mean: {mean:.4f}s") | |
print(f" std: {stdev:.4f}s") | |
print(f" min: {fastest:.4f}s") | |
print(f" 25%: {q1:.4f}s") | |
print(f" 50%: {median:.4f}s") | |
print(f" 75%: {q3:.4f}s") | |
print(f" max: {slowest:.4f}s") | |
print(f"total: {total:.4f}s") | |
# https://en.wikipedia.org/wiki/Interquartile_range | |
iqr = q3 - q1 | |
fast = q1 - 1.5 * iqr | |
slow_threshold = q3 + 1.5 * iqr | |
slow_tests = [ | |
(test, elapsed) | |
for test, elapsed | |
in StopwatchTestResult.timings.items() | |
if elapsed >= slow_threshold | |
] | |
if not slow_tests: return | |
print() | |
print("Outliers") | |
print("========") | |
print("These were particularly slow:") | |
print() | |
for test, elapsed in slow_tests: | |
print(' ', test, f"[{elapsed:0.4f}s]") | |
class StopwatchTestRunner(DiscoverRunner): | |
def __init__(self, *args, **kwargs): | |
super().__init__(*args, **kwargs) | |
self._stats = kwargs['stats'] | |
@classmethod | |
def add_arguments(cls, parser): | |
DiscoverRunner.add_arguments(parser) | |
parser.add_argument( | |
"--stats", | |
action="store_true", | |
help="Print timing statistics", | |
) | |
def get_resultclass(self): | |
# super().get_resultclass() or | |
return StopwatchTestResult | |
def run_tests(self, test_labels, extra_tests=None, **kwargs): | |
super().run_tests(test_labels, extra_tests, **kwargs) | |
if self._stats: | |
StopwatchTestResult.print_stats() | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment