RetryTestExtension by mkobit - https://stackoverflow.com/a/46207476/14379

MyRetryableTest.kt

import org.junit.jupiter.api.Disabled
import org.junit.jupiter.api.DisplayName
import org.junit.jupiter.api.Test

internal class MyRetryableTest {

    // Just to help IntelliJ detect this test class:
    @Test
    @Disabled
    fun dummy() {}

    @DisplayName("Fail all retries")
    @Retry(invocationCount = 5, minSuccess = 3)
    internal fun failAllRetries(retryInfo: RetryInfo) {
        println(retryInfo)
        throw Exception("Failed at $retryInfo")
    }

    @DisplayName("Only fail once")
    @Retry(invocationCount = 5, minSuccess = 4)
    internal fun succeedOnRetry(retryInfo: RetryInfo) {
        if (retryInfo.invocation == 1) {
            throw Exception("Failed at ${retryInfo.invocation}")
        }
    }

    @DisplayName("Only requires single success and is first execution")
    @Retry(invocationCount = 5, minSuccess = 1)
    internal fun firstSuccess(retryInfo: RetryInfo) {
        println("Running: $retryInfo")
    }

    @DisplayName("Only requires single success and is last execution")
    @Retry(invocationCount = 5, minSuccess = 1)
    internal fun lastSuccess(retryInfo: RetryInfo) {
        if (retryInfo.invocation < 5) {
            throw Exception("Failed at ${retryInfo.invocation}")
        }
    }

    @DisplayName("All required all succeed")
    @Retry(invocationCount = 5, minSuccess = 5)
    internal fun allRequiredAllSucceed(retryInfo: RetryInfo) {
        println("Running: $retryInfo")
    }

    @DisplayName("Fail early and disable")
    @Retry(invocationCount = 5, minSuccess = 4)
    internal fun failEarly(retryInfo: RetryInfo) {
        throw Exception("Failed at ${retryInfo.invocation}")
    }
}

Retry.kt

import org.junit.jupiter.api.TestTemplate
import org.junit.jupiter.api.extension.ExtendWith

@TestTemplate
@Target(AnnotationTarget.FUNCTION)
@ExtendWith(RetryTestExtension::class)
annotation class Retry(val invocationCount: Int, val minSuccess: Int)

RetryInfo.kt

data class RetryInfo(val invocation: Int, val maxInvocations: Int)

RetryingTestExecutionExtension.kt

import org.junit.jupiter.api.extension.ConditionEvaluationResult
import org.junit.jupiter.api.extension.ExecutionCondition
import org.junit.jupiter.api.extension.ExtensionContext
import org.junit.jupiter.api.extension.ParameterContext
import org.junit.jupiter.api.extension.ParameterResolver
import org.junit.jupiter.api.extension.TestExecutionExceptionHandler
import org.opentest4j.TestAbortedException

internal class RetryingTestExecutionExtension(
        private val invocation: Int,
        private val maxInvocations: Int,
        private val minSuccess: Int
) : ExecutionCondition, ParameterResolver, TestExecutionExceptionHandler {
    override fun evaluateExecutionCondition(
            context: ExtensionContext
    ): ConditionEvaluationResult {
        val failureCount = getFailures(context).size
        // Shift -1 because this happens before test
        val successCount = (invocation - 1) - failureCount
        when {
            (maxInvocations - failureCount) < minSuccess -> // Case when we cannot hit our minimum success
                return ConditionEvaluationResult.disabled("Cannot hit minimum success rate of $minSuccess/$maxInvocations - $failureCount failures already")
            successCount < minSuccess -> // Case when we haven't hit success threshold yet
                return ConditionEvaluationResult.enabled("Have not ran $minSuccess/$maxInvocations successful executions")
            else -> return ConditionEvaluationResult.disabled("$minSuccess/$maxInvocations successful runs have already ran. Skipping run $invocation")
        }
    }

    override fun supportsParameter(
            parameterContext: ParameterContext,
            extensionContext: ExtensionContext
    ): Boolean = parameterContext.parameter.type == RetryInfo::class.java

    override fun resolveParameter(
            parameterContext: ParameterContext,
            extensionContext: ExtensionContext
    ): Any = RetryInfo(invocation, maxInvocations)

    override fun handleTestExecutionException(
            context: ExtensionContext,
            throwable: Throwable
    ) {

        val testFailure = RetryingTestFailure(invocation, throwable)
        val failures: MutableList<RetryingTestFailure> = getFailures(context)
        failures.add(testFailure)
        val failureCount = failures.size
        val successCount = invocation - failureCount
        if ((maxInvocations - failureCount) < minSuccess) {
            throw testFailure
        } else if (successCount < minSuccess) {
            // Case when we have still have retries left
            throw TestAbortedException("Aborting test #$invocation/$maxInvocations- still have retries left",
                    testFailure)
        }
    }

    private fun getFailures(context: ExtensionContext): MutableList<RetryingTestFailure> {
        val namespace = ExtensionContext.Namespace.create(
                RetryingTestExecutionExtension::class.java)
        val store = context.parent.get().getStore(namespace)
        @Suppress("UNCHECKED_CAST")
        return store.getOrComputeIfAbsent(context.requiredTestMethod.name, { mutableListOf<RetryingTestFailure>() }, MutableList::class.java) as MutableList<RetryingTestFailure>
    }
}

RetryingTestFailure.kt

import java.lang.Exception

internal class RetryingTestFailure(invocation: Int, cause: Throwable) : Exception("Failed test execution at invocation #$invocation", cause)

RetryTemplateContext.kt

import org.junit.jupiter.api.extension.Extension
import org.junit.jupiter.api.extension.TestTemplateInvocationContext

class RetryTemplateContext(
        private val invocation: Int,
        private val maxInvocations: Int,
        private val minSuccess: Int
) : TestTemplateInvocationContext {
    override fun getDisplayName(invocationIndex: Int): String {
        return "Invocation number $invocationIndex (requires $minSuccess success)"
    }

    override fun getAdditionalExtensions(): MutableList<Extension> {
        return mutableListOf(
                RetryingTestExecutionExtension(invocation, maxInvocations, minSuccess)
        )
    }
}

RetryTestExtension.kt

import org.junit.jupiter.api.extension.ExtensionContext
import org.junit.jupiter.api.extension.ExtensionContextException
import org.junit.jupiter.api.extension.TestTemplateInvocationContext
import org.junit.jupiter.api.extension.TestTemplateInvocationContextProvider
import org.junit.platform.commons.support.AnnotationSupport
import java.util.stream.IntStream
import java.util.stream.Stream

class RetryTestExtension : TestTemplateInvocationContextProvider {
    override fun supportsTestTemplate(context: ExtensionContext): Boolean {
        return context.testMethod.map { it.isAnnotationPresent(Retry::class.java) }.orElse(false)
    }

    override fun provideTestTemplateInvocationContexts(context: ExtensionContext): Stream<TestTemplateInvocationContext> {
        val annotation = AnnotationSupport.findAnnotation(
                context.testMethod.orElseThrow { ExtensionContextException("Must be annotated on method") },
                Retry::class.java
        ).orElseThrow { ExtensionContextException("${Retry::class.java} not found on method") }

        checkValidRetry(annotation)

        return IntStream.rangeClosed(1, annotation.invocationCount)
                .mapToObj { RetryTemplateContext(it, annotation.invocationCount, annotation.minSuccess) }
    }

    private fun checkValidRetry(annotation: Retry) {
        if (annotation.invocationCount < 1) {
            throw ExtensionContextException("${annotation.invocationCount} must be greater than or equal to 1")
        }
        if (annotation.minSuccess < 1 || annotation.minSuccess > annotation.invocationCount) {
            throw ExtensionContextException("Invalid ${annotation.minSuccess}")
        }
    }
}