diff --git a/.idea/runConfigurations/android_app.xml b/.idea/runConfigurations/android_app.xml
index 367f2db76..523e7dfb8 100644
--- a/.idea/runConfigurations/android_app.xml
+++ b/.idea/runConfigurations/android_app.xml
@@ -5,9 +5,9 @@
-
+
-
+
\ No newline at end of file
diff --git a/.idea/runConfigurations/android_library.xml b/.idea/runConfigurations/android_library.xml
index 466470bdd..d95e2935a 100644
--- a/.idea/runConfigurations/android_library.xml
+++ b/.idea/runConfigurations/android_library.xml
@@ -1,15 +1,11 @@
-
-
-
-
-
-
-
+
-
+
+
+
-
+
diff --git a/configuration/src/main/kotlin/com/malinskiy/marathon/config/Configuration.kt b/configuration/src/main/kotlin/com/malinskiy/marathon/config/Configuration.kt
index 4e619a185..5dc0f7939 100644
--- a/configuration/src/main/kotlin/com/malinskiy/marathon/config/Configuration.kt
+++ b/configuration/src/main/kotlin/com/malinskiy/marathon/config/Configuration.kt
@@ -2,6 +2,7 @@ package com.malinskiy.marathon.config
import com.fasterxml.jackson.databind.annotation.JsonDeserialize
import com.malinskiy.marathon.config.strategy.BatchingStrategyConfiguration
+import com.malinskiy.marathon.config.strategy.ExecutionStrategyConfiguration
import com.malinskiy.marathon.config.strategy.FlakinessStrategyConfiguration
import com.malinskiy.marathon.config.strategy.PoolingStrategyConfiguration
import com.malinskiy.marathon.config.strategy.RetryStrategyConfiguration
@@ -18,7 +19,7 @@ private const val DEFAULT_DEVICE_INITIALIZATION_TIMEOUT_MILLIS = 180_000L
data class Configuration private constructor(
val name: String,
val outputDir: File,
-
+
val outputConfiguration: OutputConfiguration,
val analyticsConfiguration: AnalyticsConfiguration,
@@ -31,8 +32,8 @@ data class Configuration private constructor(
val filteringConfiguration: FilteringConfiguration,
val ignoreFailures: Boolean,
+ val executionStrategy: ExecutionStrategyConfiguration,
val isCodeCoverageEnabled: Boolean,
- val strictMode: Boolean,
val uncompletedTestRetryQuota: Int,
val testClassRegexes: Collection,
@@ -65,7 +66,7 @@ data class Configuration private constructor(
"filtering" to filteringConfiguration.toString(),
"ignoreFailures" to ignoreFailures.toString(),
"isCodeCoverageEnabled" to isCodeCoverageEnabled.toString(),
- "strictMode" to strictMode.toString(),
+ "executionStrategy" to executionStrategy.toString(),
"testClassRegexes" to testClassRegexes.toString(),
"includeSerialRegexes" to includeSerialRegexes.toString(),
"excludeSerialRegexes" to excludeSerialRegexes.toString(),
@@ -96,7 +97,7 @@ data class Configuration private constructor(
if (filteringConfiguration != other.filteringConfiguration) return false
if (ignoreFailures != other.ignoreFailures) return false
if (isCodeCoverageEnabled != other.isCodeCoverageEnabled) return false
- if (strictMode != other.strictMode) return false
+ if (executionStrategy != other.executionStrategy) return false
if (uncompletedTestRetryQuota != other.uncompletedTestRetryQuota) return false
//For testing we need to compare configuration instances. Unfortunately Regex equality is broken so need to map it to String
if (testClassRegexes.map { it.pattern } != other.testClassRegexes.map { it.pattern }) return false
@@ -127,7 +128,7 @@ data class Configuration private constructor(
result = 31 * result + filteringConfiguration.hashCode()
result = 31 * result + ignoreFailures.hashCode()
result = 31 * result + isCodeCoverageEnabled.hashCode()
- result = 31 * result + strictMode.hashCode()
+ result = 31 * result + executionStrategy.hashCode()
result = 31 * result + uncompletedTestRetryQuota
result = 31 * result + testClassRegexes.hashCode()
result = 31 * result + includeSerialRegexes.hashCode()
@@ -141,38 +142,39 @@ data class Configuration private constructor(
result = 31 * result + deviceInitializationTimeoutMillis.hashCode()
return result
}
+
data class Builder(
- val name: String,
- val outputDir: File,
- var analyticsConfiguration: AnalyticsConfiguration = AnalyticsConfiguration.DisabledAnalytics,
- var poolingStrategy: PoolingStrategyConfiguration = PoolingStrategyConfiguration.OmniPoolingStrategyConfiguration,
- var shardingStrategy: ShardingStrategyConfiguration = ShardingStrategyConfiguration.ParallelShardingStrategyConfiguration,
- var sortingStrategy: SortingStrategyConfiguration = SortingStrategyConfiguration.NoSortingStrategyConfiguration,
- var batchingStrategy: BatchingStrategyConfiguration = BatchingStrategyConfiguration.IsolateBatchingStrategyConfiguration,
- var flakinessStrategy: FlakinessStrategyConfiguration = FlakinessStrategyConfiguration.IgnoreFlakinessStrategyConfiguration,
- var retryStrategy: RetryStrategyConfiguration = RetryStrategyConfiguration.NoRetryStrategyConfiguration,
- var filteringConfiguration: FilteringConfiguration = FilteringConfiguration(emptyList(), emptyList()),
-
- var ignoreFailures: Boolean = false,
- var isCodeCoverageEnabled: Boolean = false,
- var strictMode: Boolean = false,
- var uncompletedTestRetryQuota: Int = Integer.MAX_VALUE,
-
- var testClassRegexes: Collection = listOf(Regex("^((?!Abstract).)*Test[s]*$")),
- var includeSerialRegexes: Collection = emptyList(),
- var excludeSerialRegexes: Collection = emptyList(),
-
- var testBatchTimeoutMillis: Long = DEFAULT_BATCH_EXECUTION_TIMEOUT_MILLIS,
- var testOutputTimeoutMillis: Long = DEFAULT_OUTPUT_TIMEOUT_MILLIS,
- var debug: Boolean = true,
-
- var screenRecordingPolicy: ScreenRecordingPolicy = ScreenRecordingPolicy.ON_FAILURE,
-
- var analyticsTracking: Boolean = false,
- var deviceInitializationTimeoutMillis: Long = DEFAULT_DEVICE_INITIALIZATION_TIMEOUT_MILLIS,
-
- var outputConfiguration: OutputConfiguration = OutputConfiguration(),
- var vendorConfiguration: VendorConfiguration = VendorConfiguration.EmptyVendorConfiguration(),
+ val name: String,
+ val outputDir: File,
+ var analyticsConfiguration: AnalyticsConfiguration = AnalyticsConfiguration.DisabledAnalytics,
+ var poolingStrategy: PoolingStrategyConfiguration = PoolingStrategyConfiguration.OmniPoolingStrategyConfiguration,
+ var shardingStrategy: ShardingStrategyConfiguration = ShardingStrategyConfiguration.ParallelShardingStrategyConfiguration,
+ var sortingStrategy: SortingStrategyConfiguration = SortingStrategyConfiguration.NoSortingStrategyConfiguration,
+ var batchingStrategy: BatchingStrategyConfiguration = BatchingStrategyConfiguration.IsolateBatchingStrategyConfiguration,
+ var flakinessStrategy: FlakinessStrategyConfiguration = FlakinessStrategyConfiguration.IgnoreFlakinessStrategyConfiguration,
+ var retryStrategy: RetryStrategyConfiguration = RetryStrategyConfiguration.NoRetryStrategyConfiguration,
+ var filteringConfiguration: FilteringConfiguration = FilteringConfiguration(emptyList(), emptyList()),
+
+ var ignoreFailures: Boolean = false,
+ var isCodeCoverageEnabled: Boolean = false,
+ var executionStrategy: ExecutionStrategyConfiguration = ExecutionStrategyConfiguration(),
+ var uncompletedTestRetryQuota: Int = Integer.MAX_VALUE,
+
+ var testClassRegexes: Collection = listOf(Regex("^((?!Abstract).)*Test[s]*$")),
+ var includeSerialRegexes: Collection = emptyList(),
+ var excludeSerialRegexes: Collection = emptyList(),
+
+ var testBatchTimeoutMillis: Long = DEFAULT_BATCH_EXECUTION_TIMEOUT_MILLIS,
+ var testOutputTimeoutMillis: Long = DEFAULT_OUTPUT_TIMEOUT_MILLIS,
+ var debug: Boolean = true,
+
+ var screenRecordingPolicy: ScreenRecordingPolicy = ScreenRecordingPolicy.ON_FAILURE,
+
+ var analyticsTracking: Boolean = false,
+ var deviceInitializationTimeoutMillis: Long = DEFAULT_DEVICE_INITIALIZATION_TIMEOUT_MILLIS,
+
+ var outputConfiguration: OutputConfiguration = OutputConfiguration(),
+ var vendorConfiguration: VendorConfiguration = VendorConfiguration.EmptyVendorConfiguration(),
) {
fun build(): Configuration {
return Configuration(
@@ -189,7 +191,7 @@ data class Configuration private constructor(
filteringConfiguration = filteringConfiguration,
ignoreFailures = ignoreFailures,
isCodeCoverageEnabled = isCodeCoverageEnabled,
- strictMode = strictMode,
+ executionStrategy = executionStrategy,
uncompletedTestRetryQuota = uncompletedTestRetryQuota,
testClassRegexes = testClassRegexes,
includeSerialRegexes = includeSerialRegexes,
diff --git a/configuration/src/main/kotlin/com/malinskiy/marathon/config/LogicalConfigurationValidator.kt b/configuration/src/main/kotlin/com/malinskiy/marathon/config/LogicalConfigurationValidator.kt
index 559b062e8..f734f7c6e 100644
--- a/configuration/src/main/kotlin/com/malinskiy/marathon/config/LogicalConfigurationValidator.kt
+++ b/configuration/src/main/kotlin/com/malinskiy/marathon/config/LogicalConfigurationValidator.kt
@@ -1,7 +1,9 @@
package com.malinskiy.marathon.config
import com.malinskiy.marathon.config.exceptions.ConfigurationException
+import com.malinskiy.marathon.config.strategy.ExecutionMode
import com.malinskiy.marathon.config.strategy.FlakinessStrategyConfiguration
+import com.malinskiy.marathon.config.strategy.RetryStrategyConfiguration
import com.malinskiy.marathon.config.strategy.ShardingStrategyConfiguration
import com.malinskiy.marathon.config.vendor.VendorConfiguration
@@ -28,5 +30,27 @@ class LogicalConfigurationValidator : ConfigurationValidator {
else -> Unit
}
+
+ when(configuration.executionStrategy.mode) {
+ ExecutionMode.ANY_SUCCESS -> {
+ if (configuration.shardingStrategy !is ShardingStrategyConfiguration.ParallelShardingStrategyConfiguration) {
+ throw ConfigurationException(
+ "Configuration is invalid: can't use complex sharding and any success execution strategy at the same time. Consult documentation for the any success execution logic"
+ )
+ }
+ }
+ ExecutionMode.ALL_SUCCESS -> {
+ if (configuration.flakinessStrategy !is FlakinessStrategyConfiguration.IgnoreFlakinessStrategyConfiguration) {
+ throw ConfigurationException(
+ "Configuration is invalid: can't use complex flakiness strategy and all success execution strategy at the same time. Consult documentation for the all success execution logic"
+ )
+ }
+ if (configuration.retryStrategy !is RetryStrategyConfiguration.NoRetryStrategyConfiguration) {
+ throw ConfigurationException(
+ "Configuration is invalid: can't use complex retry strategy and all success execution strategy at the same time. Consult documentation for the all success execution logic"
+ )
+ }
+ }
+ }
}
}
diff --git a/configuration/src/main/kotlin/com/malinskiy/marathon/config/strategy/ExecutionStrategyConfiguration.kt b/configuration/src/main/kotlin/com/malinskiy/marathon/config/strategy/ExecutionStrategyConfiguration.kt
new file mode 100644
index 000000000..baf7f3b3b
--- /dev/null
+++ b/configuration/src/main/kotlin/com/malinskiy/marathon/config/strategy/ExecutionStrategyConfiguration.kt
@@ -0,0 +1,31 @@
+package com.malinskiy.marathon.config.strategy
+
+import com.fasterxml.jackson.annotation.JsonProperty
+
+/**
+ * @property fast fail-fast or success-fast depending on the value of [mode]. This doesn't affect the result
+ * of the run, it only saves compute time
+ */
+data class ExecutionStrategyConfiguration(
+ @JsonProperty("mode") val mode: ExecutionMode = ExecutionMode.ANY_SUCCESS,
+ @JsonProperty("fast") val fast: Boolean = true,
+)
+
+/**
+ * @property ANY_SUCCESS test passes if any of executions is passing
+ * this mode works only if there is no complex sharding strategy applied
+ *
+ * Why: it doesn't make sense when user asks for N number of tests
+ * to run explicitly, and we pass on the first one. Sharding used to verify probability of passing with an explicit boundary for precision
+ * @property ALL_SUCCESS test passes if and only if all the executions are passing
+ * this mode works only if there are no retries, i.e. no complex flakiness strategy, no retry strategy
+ *
+ * Why: when adding retries to tests with retry+flakiness strategies users want to trade-off cost for reliability, i.e. add more retries
+ * and pass if one of them passes, so retries only make sense for the [ANY_SUCCESS] mode. When we use [ALL_SUCCESS] mode it means user
+ * wants to verify each test with a number of tries (they are not retries per se) and pass only if all of them succeed. This is the case
+ * when fixing a flaky test or adding a new test, and we want to have a signal that the test is fixed/not flaky.
+ */
+enum class ExecutionMode {
+ @JsonProperty("ANY_SUCCESS") ANY_SUCCESS,
+ @JsonProperty("ALL_SUCCESS") ALL_SUCCESS,
+}
diff --git a/configuration/src/test/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactoryTest.kt b/configuration/src/test/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactoryTest.kt
index bec6847c7..28e52d838 100644
--- a/configuration/src/test/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactoryTest.kt
+++ b/configuration/src/test/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactoryTest.kt
@@ -163,7 +163,6 @@ class ConfigurationFactoryTest {
configuration.excludeSerialRegexes.joinToString(separator = "") { it.pattern } shouldBeEqualTo """emulator-5002""".toRegex().pattern
configuration.ignoreFailures shouldBeEqualTo false
configuration.isCodeCoverageEnabled shouldBeEqualTo false
- configuration.strictMode shouldBeEqualTo true
configuration.testBatchTimeoutMillis shouldBeEqualTo 20_000
configuration.testOutputTimeoutMillis shouldBeEqualTo 30_000
configuration.debug shouldBeEqualTo true
diff --git a/configuration/src/test/resources/fixture/config/sample_1.yaml b/configuration/src/test/resources/fixture/config/sample_1.yaml
index 100d84b45..754869033 100644
--- a/configuration/src/test/resources/fixture/config/sample_1.yaml
+++ b/configuration/src/test/resources/fixture/config/sample_1.yaml
@@ -68,7 +68,9 @@ excludeSerialRegexes:
- "emulator-5002"
ignoreFailures: false
isCodeCoverageEnabled: false
-strictMode: true
+executionStrategy:
+ mode: ANY_SUCCESS
+ fast: true
testBatchTimeoutMillis: 20000
testOutputTimeoutMillis: 30000
debug: true
diff --git a/configuration/src/test/resources/fixture/config/sample_1_rp.yaml b/configuration/src/test/resources/fixture/config/sample_1_rp.yaml
index 185b37872..7a1465277 100644
--- a/configuration/src/test/resources/fixture/config/sample_1_rp.yaml
+++ b/configuration/src/test/resources/fixture/config/sample_1_rp.yaml
@@ -66,7 +66,9 @@ excludeSerialRegexes:
- "emulator-5002"
ignoreFailures: false
isCodeCoverageEnabled: false
-strictMode: true
+executionStrategy:
+ mode: ANY_SUCCESS
+ fast: true
testBatchTimeoutMillis: 20000
testOutputTimeoutMillis: 30000
debug: true
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/Marathon.kt b/core/src/main/kotlin/com/malinskiy/marathon/Marathon.kt
index f31d1cb3f..102e25211 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/Marathon.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/Marathon.kt
@@ -19,7 +19,6 @@ import com.malinskiy.marathon.execution.TestParser
import com.malinskiy.marathon.execution.TestShard
import com.malinskiy.marathon.execution.bundle.TestBundleIdentifier
import com.malinskiy.marathon.execution.command.parse.MarathonTestParseCommand
-import com.malinskiy.marathon.execution.progress.ProgressReporter
import com.malinskiy.marathon.execution.withRetry
import com.malinskiy.marathon.extension.toFlakinessStrategy
import com.malinskiy.marathon.extension.toShardingStrategy
@@ -34,7 +33,6 @@ import com.malinskiy.marathon.usageanalytics.UsageAnalytics
import com.malinskiy.marathon.usageanalytics.tracker.Event
import kotlinx.coroutines.NonCancellable
import kotlinx.coroutines.async
-import kotlinx.coroutines.delay
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withContext
import kotlinx.coroutines.withTimeoutOrNull
@@ -52,7 +50,6 @@ class Marathon(
private val logConfigurator: MarathonLogConfigurator,
private val tracker: TrackerInternal,
private val analytics: Analytics,
- private val progressReporter: ProgressReporter,
private val track: Track,
private val timer: Timer,
private val marathonTestParseCommand: MarathonTestParseCommand
@@ -142,7 +139,6 @@ class Marathon(
analytics,
configuration,
shard,
- progressReporter,
track,
timer,
testBundleIdentifier,
@@ -155,12 +151,13 @@ class Marathon(
}
configuration.outputDir.mkdirs()
- if (parsedFilteredTests.isNotEmpty()) {
+ val result = if (parsedFilteredTests.isNotEmpty()) {
scheduler.execute()
+ } else {
+ true
}
onFinish(analytics, deviceProvider)
- val result = progressReporter.aggregateResult()
stopKoin()
return result
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/device/Device.kt b/core/src/main/kotlin/com/malinskiy/marathon/device/Device.kt
index d06267498..92d3fe326 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/device/Device.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/device/Device.kt
@@ -2,7 +2,6 @@ package com.malinskiy.marathon.device
import com.malinskiy.marathon.config.Configuration
import com.malinskiy.marathon.execution.TestBatchResults
-import com.malinskiy.marathon.execution.progress.ProgressReporter
import com.malinskiy.marathon.test.TestBatch
import kotlinx.coroutines.CompletableDeferred
import mu.KLogger
@@ -45,7 +44,6 @@ interface Device {
devicePoolId: DevicePoolId,
testBatch: TestBatch,
deferred: CompletableDeferred,
- progressReporter: ProgressReporter
)
/**
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/di/Modules.kt b/core/src/main/kotlin/com/malinskiy/marathon/di/Modules.kt
index 3628c430f..f59596aa6 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/di/Modules.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/di/Modules.kt
@@ -7,7 +7,6 @@ import com.malinskiy.marathon.analytics.external.AnalyticsFactory
import com.malinskiy.marathon.analytics.internal.pub.Track
import com.malinskiy.marathon.config.Configuration
import com.malinskiy.marathon.execution.command.parse.MarathonTestParseCommand
-import com.malinskiy.marathon.execution.progress.ProgressReporter
import com.malinskiy.marathon.io.FileManager
import com.malinskiy.marathon.json.FileSerializer
import com.malinskiy.marathon.time.SystemTimer
@@ -37,12 +36,11 @@ val coreModule = module {
}
single { Clock.systemDefaultZone() }
single { SystemTimer(get()) }
- single { ProgressReporter(get()) }
single {
val configuration = get()
MarathonTestParseCommand(configuration.outputDir)
}
- single { Marathon(get(), get(), get(), get(), get(), get(), get(), get(), get(), get(), get()) }
+ single { Marathon(get(), get(), get(), get(), get(), get(), get(), get(), get(), get()) }
}
fun marathonStartKoin(configuration: Configuration, modules: List): KoinApplication {
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/DevicePoolActor.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/DevicePoolActor.kt
index 92c223193..715d5ce35 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/DevicePoolActor.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/DevicePoolActor.kt
@@ -12,7 +12,7 @@ import com.malinskiy.marathon.device.toDeviceInfo
import com.malinskiy.marathon.execution.bundle.TestBundleIdentifier
import com.malinskiy.marathon.execution.device.DeviceActor
import com.malinskiy.marathon.execution.device.DeviceEvent
-import com.malinskiy.marathon.execution.progress.ProgressReporter
+import com.malinskiy.marathon.execution.progress.PoolProgressAccumulator
import com.malinskiy.marathon.execution.queue.QueueActor
import com.malinskiy.marathon.execution.queue.QueueMessage
import com.malinskiy.marathon.log.MarathonLogging
@@ -26,10 +26,9 @@ import kotlin.coroutines.CoroutineContext
class DevicePoolActor(
private val poolId: DevicePoolId,
private val configuration: Configuration,
+ private val poolProgressAccumulator: PoolProgressAccumulator,
analytics: Analytics,
shard: TestShard,
- private val progressReporter: ProgressReporter,
- track: Track,
timer: Timer,
parent: Job,
context: CoroutineContext,
@@ -64,10 +63,9 @@ class DevicePoolActor(
analytics,
this,
poolId,
- progressReporter,
- track,
timer,
testBundleIdentifier,
+ poolProgressAccumulator,
poolJob,
context
)
@@ -153,7 +151,7 @@ class DevicePoolActor(
}
logger.debug { "add device ${device.serialNumber}" }
- val actor = DeviceActor(poolId, this, configuration, device, progressReporter, poolJob, coroutineContext)
+ val actor = DeviceActor(poolId, this, configuration, device, poolJob, coroutineContext)
devices[device.serialNumber] = actor
actor.safeSend(DeviceEvent.Initialize)
}
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/Scheduler.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/Scheduler.kt
index 0031819cf..4e4cf89d8 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/Scheduler.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/Scheduler.kt
@@ -12,7 +12,7 @@ import com.malinskiy.marathon.execution.DevicePoolMessage.FromScheduler
import com.malinskiy.marathon.execution.DevicePoolMessage.FromScheduler.AddDevice
import com.malinskiy.marathon.execution.DevicePoolMessage.FromScheduler.RemoveDevice
import com.malinskiy.marathon.execution.bundle.TestBundleIdentifier
-import com.malinskiy.marathon.execution.progress.ProgressReporter
+import com.malinskiy.marathon.execution.progress.PoolProgressAccumulator
import com.malinskiy.marathon.extension.toPoolingStrategy
import com.malinskiy.marathon.log.MarathonLogging
import com.malinskiy.marathon.time.Timer
@@ -32,13 +32,11 @@ import kotlin.coroutines.CoroutineContext
* 1) Subscribe on DeviceProvider
* 2) Create device pools using PoolingStrategy
*/
-
class Scheduler(
private val deviceProvider: DeviceProvider,
private val analytics: Analytics,
private val configuration: Configuration,
private val shard: TestShard,
- private val progressReporter: ProgressReporter,
private val track: Track,
private val timer: Timer,
private val testBundleIdentifier: TestBundleIdentifier?,
@@ -46,12 +44,16 @@ class Scheduler(
) : CoroutineScope {
private val job = Job()
- private val pools = ConcurrentHashMap>()
+ private val pools = ConcurrentHashMap()
+ private val results = ConcurrentHashMap()
private val poolingStrategy = configuration.poolingStrategy.toPoolingStrategy()
private val logger = MarathonLogging.logger("Scheduler")
- suspend fun execute() {
+ /**
+ * @return true if scheduled tests passed successfully, false otherwise
+ */
+ suspend fun execute() : Boolean {
subscribeOnDevices(job)
try {
withTimeout(deviceProvider.deviceInitializationTimeoutMillis) {
@@ -66,6 +68,8 @@ class Scheduler(
for (child in job.children) {
child.join()
}
+
+ return results.all { it.value.aggregateResult() }
}
private fun subscribeOnDevices(job: Job): Job {
@@ -109,9 +113,13 @@ class Scheduler(
val poolId = poolingStrategy.associate(device)
logger.debug { "device ${device.serialNumber} associated with poolId ${poolId.name}" }
+ val accumulator = results.computeIfAbsent(poolId) { id ->
+ PoolProgressAccumulator(id, shard, configuration, track)
+ }
+
pools.computeIfAbsent(poolId) { id ->
logger.debug { "pool actor ${id.name} is being created" }
- DevicePoolActor(id, configuration, analytics, shard, progressReporter, track, timer, parent, context, testBundleIdentifier)
+ DevicePoolActor(id, configuration, accumulator, analytics, shard, timer, parent, context, testBundleIdentifier)
}
pools[poolId]?.send(AddDevice(device)) ?: logger.debug {
"not sending the AddDevice event " +
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/device/DeviceActor.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/device/DeviceActor.kt
index eec934399..f90415551 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/device/DeviceActor.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/device/DeviceActor.kt
@@ -10,7 +10,6 @@ import com.malinskiy.marathon.exceptions.TestBatchExecutionException
import com.malinskiy.marathon.execution.DevicePoolMessage
import com.malinskiy.marathon.execution.DevicePoolMessage.FromDevice.IsReady
import com.malinskiy.marathon.execution.TestBatchResults
-import com.malinskiy.marathon.execution.progress.ProgressReporter
import com.malinskiy.marathon.execution.withRetry
import com.malinskiy.marathon.log.MarathonLogging
import com.malinskiy.marathon.test.TestBatch
@@ -29,7 +28,6 @@ class DeviceActor(
private val pool: SendChannel,
private val configuration: Configuration,
val device: Device,
- private val progressReporter: ProgressReporter,
parent: Job,
context: CoroutineContext
) :
@@ -184,7 +182,7 @@ class DeviceActor(
logger.debug { "executeBatch ${device.serialNumber}" }
job = async {
try {
- device.execute(configuration, devicePoolId, batch, result, progressReporter)
+ device.execute(configuration, devicePoolId, batch, result)
state.transition(DeviceEvent.Complete)
} catch (e: CancellationException) {
logger.warn(e) { "Device execution has been cancelled" }
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulator.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulator.kt
new file mode 100644
index 000000000..0516c8e3c
--- /dev/null
+++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulator.kt
@@ -0,0 +1,431 @@
+package com.malinskiy.marathon.execution.progress
+
+import com.malinskiy.marathon.actor.StateMachine
+import com.malinskiy.marathon.analytics.internal.pub.Track
+import com.malinskiy.marathon.config.Configuration
+import com.malinskiy.marathon.config.strategy.ExecutionMode
+import com.malinskiy.marathon.device.DeviceInfo
+import com.malinskiy.marathon.device.DevicePoolId
+import com.malinskiy.marathon.execution.TestResult
+import com.malinskiy.marathon.execution.TestShard
+import com.malinskiy.marathon.execution.TestStatus
+import com.malinskiy.marathon.execution.queue.TestAction
+import com.malinskiy.marathon.execution.queue.TestEvent
+import com.malinskiy.marathon.execution.queue.TestState
+import com.malinskiy.marathon.log.MarathonLogging
+import com.malinskiy.marathon.test.Test
+import com.malinskiy.marathon.test.toTestName
+import kotlin.math.roundToInt
+
+class PoolProgressAccumulator(
+ private val poolId: DevicePoolId,
+ shard: TestShard,
+ private val configuration: Configuration,
+ private val track: Track
+) {
+ private val tests: HashMap> = HashMap()
+ private val logger = MarathonLogging.logger {}
+ private val executionStrategy = configuration.executionStrategy
+
+ private fun createState(initialCount: Int) = StateMachine.create {
+ initialState(TestState.Added(initialCount))
+ state {
+ on {
+ transitionTo(TestState.Added(total, running + 1))
+ }
+ on {
+ when (executionStrategy.mode) {
+ ExecutionMode.ANY_SUCCESS -> {
+ if (executionStrategy.fast || total <= 1) {
+ transitionTo(TestState.Passed(total = total, done = 1, running = running - 1), TestAction.Complete)
+ } else {
+ transitionTo(TestState.Passing(total = total, done = 1, running = running - 1))
+ }
+ }
+
+ ExecutionMode.ALL_SUCCESS -> {
+ if (total <= 1) {
+ transitionTo(TestState.Passed(total = total, done = 1, running = running - 1), TestAction.Complete)
+ } else {
+ transitionTo(TestState.Passing(total = total, done = 1, running = running - 1))
+ }
+ }
+ }
+ }
+ on {
+ when (executionStrategy.mode) {
+ ExecutionMode.ANY_SUCCESS -> {
+ if (total <= 1) {
+ transitionTo(TestState.Failed(total = total, done = 1, running = running - 1), TestAction.Complete)
+ } else {
+ transitionTo(TestState.Failing(total = total, done = 1, running = running - 1))
+ }
+ }
+
+ ExecutionMode.ALL_SUCCESS -> {
+ if (executionStrategy.fast || total <= 1) {
+ transitionTo(TestState.Failed(total = total, done = 1, running = running - 1), TestAction.Complete)
+ } else {
+ transitionTo(TestState.Failing(total = total, done = 1, running = running - 1))
+ }
+ }
+ }
+ }
+ on {
+ if (it.final) {
+ transitionTo(TestState.Failed(total = total, done = 0, running = running - 1), TestAction.Complete)
+ } else {
+ transitionTo(TestState.Added(total = total, running = running - 1))
+ }
+ }
+ on {
+ transitionTo(TestState.Added(total = total + 1, running = running))
+ }
+ on {
+ transitionTo(TestState.Added(total = total - it.count, running = running))
+ }
+ }
+ state {
+ on {
+ transitionTo(TestState.Passing(total = total, running = running + 1, done = done))
+ }
+ on {
+ when (executionStrategy.mode) {
+ ExecutionMode.ANY_SUCCESS -> {
+ if (executionStrategy.fast || done + 1 >= total) {
+ transitionTo(TestState.Passed(total = total, running = running - 1, done = done + 1), TestAction.Complete)
+ } else {
+ transitionTo(TestState.Passing(total = total, running = running - 1, done = done + 1))
+ }
+ }
+
+ ExecutionMode.ALL_SUCCESS -> {
+ if (total <= done + 1) {
+ transitionTo(TestState.Passed(total = total, running = running - 1, done = done + 1), TestAction.Complete)
+ } else {
+ transitionTo(TestState.Passing(total = total, running = running - 1, done = done + 1))
+ }
+ }
+ }
+ }
+ on {
+ when (executionStrategy.mode) {
+ ExecutionMode.ANY_SUCCESS -> {
+ if (executionStrategy.fast || done + 1 >= total) {
+ transitionTo(TestState.Passed(total = total, done = done + 1, running = running - 1), TestAction.Complete)
+ } else {
+ transitionTo(TestState.Passing(total = total, done = done + 1, running = running - 1))
+ }
+ }
+
+ ExecutionMode.ALL_SUCCESS -> {
+ if (executionStrategy.fast || done + 1 >= total) {
+ transitionTo(TestState.Failed(total = total, done = done + 1, running = running - 1), TestAction.Complete)
+ } else {
+ transitionTo(TestState.Failing(total = total, done = done + 1, running = running - 1))
+ }
+ }
+ }
+ }
+ on {
+ if (it.final) {
+ when (executionStrategy.mode) {
+ ExecutionMode.ANY_SUCCESS -> {
+ transitionTo(TestState.Passed(total = total, done = done, running = running - 1), TestAction.Complete)
+ }
+
+ ExecutionMode.ALL_SUCCESS -> {
+ transitionTo(TestState.Failed(total = total, done = done, running = running - 1), TestAction.Complete)
+ }
+ }
+ } else {
+ transitionTo(TestState.Passing(total = total, done = done, running = running - 1))
+ }
+ }
+ on {
+ transitionTo(TestState.Passing(total = total - it.count, running = running, done = done))
+ }
+ on {
+ transitionTo(TestState.Passing(total = total + 1, running = running, done = done))
+ }
+ }
+ state {
+ on {
+ transitionTo(TestState.Failing(total = total, running = running + 1, done = done))
+ }
+ on {
+ when (executionStrategy.mode) {
+ ExecutionMode.ANY_SUCCESS -> {
+ if (executionStrategy.fast || done + 1 >= total) {
+ transitionTo(TestState.Passed(total = total, done = done + 1, running = running - 1), TestAction.Complete)
+ } else {
+ transitionTo(TestState.Passing(total = total, done = done + 1, running = running - 1))
+ }
+ }
+
+ ExecutionMode.ALL_SUCCESS -> {
+ if (executionStrategy.fast || done + 1 >= total) {
+ transitionTo(TestState.Failed(total = total, done = done + 1, running = running - 1), TestAction.Complete)
+ } else {
+ transitionTo(TestState.Failing(total = total, done = done + 1, running = running - 1))
+ }
+ }
+ }
+ }
+ on {
+ when (executionStrategy.mode) {
+ ExecutionMode.ANY_SUCCESS -> {
+ if (done + 1 >= total) {
+ transitionTo(TestState.Failed(total = total, done = done + 1, running = running - 1), TestAction.Complete)
+ } else {
+ transitionTo(TestState.Failing(total = total, done = done + 1, running = running - 1))
+ }
+ }
+
+ ExecutionMode.ALL_SUCCESS -> {
+ if (executionStrategy.fast || done + 1 >= total) {
+ transitionTo(TestState.Failed(total = total, done = done + 1, running = running - 1), TestAction.Complete)
+ } else {
+ transitionTo(TestState.Failing(total = total, done = done + 1, running = running - 1))
+ }
+ }
+ }
+ }
+ on {
+ if (it.final) {
+ transitionTo(TestState.Failed(total = total, done = done, running = running - 1), TestAction.Complete)
+ } else {
+ transitionTo(TestState.Failing(total = total, done = done, running = running - 1))
+ }
+ }
+ on {
+ transitionTo(TestState.Failing(total = total - it.count, done = done, running = running))
+ }
+ on {
+ transitionTo(TestState.Failing(total = total + 1, done = done, running = running))
+ }
+ }
+ state {
+ on {
+ transitionTo(TestState.Failed(total = total, running = running + 1, done = done))
+ }
+ on {
+ transitionTo(TestState.Failed(total = total, running = running - 1, done = done + 1))
+ }
+ on {
+ transitionTo(TestState.Failed(total = total, running = running - 1, done = done + 1))
+ }
+ on {
+ transitionTo(TestState.Failed(total = total, running = running - 1, done = done))
+ }
+ on {
+ transitionTo(TestState.Failed(total = total + 1, running = running, done = done))
+ }
+ on {
+ transitionTo(TestState.Failed(total = total - it.count, running = running, done = done))
+ }
+ }
+ state {
+ on {
+ transitionTo(TestState.Passed(total = total, running = running + 1, done = done))
+ }
+ on {
+ transitionTo(TestState.Passed(total = total, running = running - 1, done = done + 1))
+ }
+ on {
+ transitionTo(TestState.Passed(total = total, running = running - 1, done = done + 1))
+ }
+ on {
+ transitionTo(TestState.Passed(total = total, running = running - 1, done = done))
+ }
+ on {
+ transitionTo(TestState.Passed(total = total + 1, running = running, done = done))
+ }
+ on {
+ transitionTo(TestState.Passed(total = total - it.count, running = running, done = done))
+ }
+ }
+ onTransition {
+ if (it as? StateMachine.Transition.Valid !is StateMachine.Transition.Valid) {
+ logger.error { "from ${it.fromState} event ${it.event}" }
+ }
+ trackTestTransition(poolId, it)
+ }
+ }
+
+ init {
+ val allTests = shard.tests + shard.flakyTests
+ allTests.groupBy { it }.map {
+ val count = it.value.size
+ it.key.toTestName() to createState(count)
+ }.also {
+ tests.putAll(it)
+ }
+ }
+
+ fun testStarted(device: DeviceInfo, test: Test) {
+ transition(test, TestEvent.Started)
+ println("${toPercent(progress())} | [${poolId.name}]-[${device.serialNumber}] ${test.toTestName()} started")
+ }
+
+ /**
+ * @param final used for incomplete tests to signal no more retries left, hence a decision on the status has to be made
+ */
+ fun testEnded(device: DeviceInfo, testResult: TestResult, final: Boolean = false): TestAction? {
+ return when (testResult.status) {
+ TestStatus.FAILURE -> {
+ println("${toPercent(progress())} | [${poolId.name}]-[${device.serialNumber}] ${testResult.test.toTestName()} failed")
+ transition(testResult.test, TestEvent.Failed(device, testResult)).sideffect()
+ }
+
+ TestStatus.PASSED -> {
+ println("${toPercent(progress())} | [${poolId.name}]-[${device.serialNumber}] ${testResult.test.toTestName()} passed")
+ transition(testResult.test, TestEvent.Passed(device, testResult)).sideffect()
+ }
+
+ TestStatus.IGNORED, TestStatus.ASSUMPTION_FAILURE -> {
+ println("${toPercent(progress())} | [${poolId.name}]-[${device.serialNumber}] ${testResult.test.toTestName()} ignored")
+ transition(testResult.test, TestEvent.Passed(device, testResult)).sideffect()
+ }
+
+ TestStatus.INCOMPLETE -> {
+ println("${toPercent(progress())} | [${poolId.name}]-[${device.serialNumber}] ${testResult.test.toTestName()} incomplete")
+ transition(testResult.test, TestEvent.Incomplete(device, testResult, final)).sideffect()
+ }
+ }
+ }
+
+
+ /**
+ * Should always be called before testEnded, otherwise the FSM might transition into a terminal state prematurely
+ */
+ fun retryTest(test: Test): TestAction? {
+ return transition(test, TestEvent.AddRetry).sideffect()
+ }
+
+ fun removeTest(test: Test, diff: Int): TestAction? {
+ return transition(test, TestEvent.RemoveAttempts(diff)).sideffect()
+ }
+
+ private fun trackTestTransition(poolId: DevicePoolId, transition: StateMachine.Transition) {
+ val validTransition = transition as? StateMachine.Transition.Valid
+ val final = if (validTransition is StateMachine.Transition.Valid) {
+ when (validTransition.sideEffect) {
+ is TestAction.Complete -> true
+ else -> false
+ }
+ } else false
+
+ val (testResult: TestResult?, device: DeviceInfo?) = extractEventAndDevice(transition)
+ if (testResult == null || device == null) return
+
+ track.test(poolId, device, testResult, final)
+ }
+
+ private fun extractEventAndDevice(transition: StateMachine.Transition): Pair {
+ val event = transition.event
+ val testResult: TestResult? = when (event) {
+ is TestEvent.Passed -> event.testResult
+ is TestEvent.Failed -> event.testResult
+ is TestEvent.Incomplete -> {
+ if (event.final) {
+ event.testResult.copy(status = TestStatus.FAILURE)
+ } else {
+ event.testResult
+ }
+ }
+ else -> null
+ }
+ val device: DeviceInfo? = when (event) {
+ is TestEvent.Passed -> event.device
+ is TestEvent.Failed -> event.device
+ is TestEvent.Incomplete -> event.device
+ else -> null
+ }
+ return Pair(testResult, device)
+ }
+
+ fun aggregateResult(): Boolean {
+ synchronized(tests) {
+ return tests.map {
+ when (it.value.state) {
+ is TestState.Added -> {
+ logger.error { "Expected to run ${it.key} but no events received" }
+ false
+ }
+
+ is TestState.Failed -> false
+ is TestState.Failing -> {
+ logger.error { "Expected to run ${it.key} more, but no terminal events received" }
+ false
+ }
+
+ is TestState.Passed -> true
+ is TestState.Passing -> {
+ logger.error { "Expected to run ${it.key} more, but no terminal events received" }
+ //The test is passing but the execution mode might require all the
+ //runs to pass before considering this an actual pass
+ executionStrategy.mode == ExecutionMode.ANY_SUCCESS
+ }
+ }
+ }.reduce { a, b -> a && b }
+ }
+ }
+
+ fun progress(): Float {
+ var done = 0
+ var total = 0
+
+ tests.values.forEach {
+ val state = it.state
+ when (state) {
+ is TestState.Added -> {
+ total += state.total
+ }
+
+ is TestState.Failed -> {
+ total += state.total
+ done += state.done
+ }
+
+ is TestState.Failing -> {
+ total += state.total
+ done += state.done
+ }
+
+ is TestState.Passed -> {
+ total += state.total
+ done += state.done
+ }
+
+ is TestState.Passing -> {
+ total += state.total
+ done += state.done
+ }
+ }
+ }
+ return done.toFloat() / total.toFloat()
+ }
+
+ private fun transition(test: Test, transition: TestEvent): StateMachine.Transition? {
+ return tests[test.toTestName()]?.transition(transition)?.also { logger.warn { "No FSM registered for test ${test.toTestName()}" } }
+ }
+
+ private fun toPercent(float: Float): String {
+ val percent = (float * HUNDRED_PERCENT_IN_FLOAT).roundToInt()
+ val format = "%02d%%"
+ return String.format(format, percent)
+ }
+
+ companion object {
+ const val HUNDRED_PERCENT_IN_FLOAT: Float = 100.0f
+ }
+}
+
+private fun StateMachine.Transition?.sideffect(): SIDE_EFFECT? {
+ return when (this) {
+ is StateMachine.Transition.Invalid -> null
+ is StateMachine.Transition.Valid -> this.sideEffect
+ null -> null
+ }
+}
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/ProgressReporter.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/ProgressReporter.kt
index 93df275bf..19be2590d 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/ProgressReporter.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/ProgressReporter.kt
@@ -1,82 +1,25 @@
package com.malinskiy.marathon.execution.progress
-import com.malinskiy.marathon.config.Configuration
import com.malinskiy.marathon.device.DeviceInfo
import com.malinskiy.marathon.device.DevicePoolId
-import com.malinskiy.marathon.execution.progress.tracker.PoolProgressTracker
import com.malinskiy.marathon.test.Test
+import com.malinskiy.marathon.test.TestBatch
import com.malinskiy.marathon.test.toTestName
-import java.util.concurrent.ConcurrentHashMap
-import kotlin.math.roundToInt
-const val HUNDRED_PERCENT_IN_FLOAT: Float = 100.0f
-
-class ProgressReporter(private val configuration: Configuration) {
- private val reporters = ConcurrentHashMap()
-
- private inline fun execute(poolId: DevicePoolId, f: (PoolProgressTracker) -> T): T {
- val reporter = reporters[poolId] ?: PoolProgressTracker(configuration)
- val result = f(reporter)
- reporters[poolId] = reporter
- return result
- }
-
- private fun toPercent(float: Float): String {
- val percent = (float * HUNDRED_PERCENT_IN_FLOAT).roundToInt()
- val format = "%02d%%"
- return String.format(format, percent)
- }
-
- fun testStarted(poolId: DevicePoolId, device: DeviceInfo, test: Test) {
- execute(poolId) { it.testStarted(test) }
- println("${toPercent(progress(poolId))} | [${poolId.name}]-[${device.serialNumber}] ${test.toTestName()} started")
- }
-
- fun testFailed(poolId: DevicePoolId, device: DeviceInfo, test: Test) {
- execute(poolId) { it.testFailed(test) }
- println("${toPercent(progress(poolId))} | [${poolId.name}]-[${device.serialNumber}] ${test.toTestName()} failed")
- }
-
- fun testPassed(poolId: DevicePoolId, device: DeviceInfo, test: Test) {
- execute(poolId) { it.testPassed(test) }
- println("${toPercent(progress(poolId))} | [${poolId.name}]-[${device.serialNumber}] ${test.toTestName()} ended")
- }
-
- fun testIgnored(poolId: DevicePoolId, device: DeviceInfo, test: Test) {
- execute(poolId) { it.testIgnored(test) }
- println("${toPercent(progress(poolId))} | [${poolId.name}]-[${device.serialNumber}] ${test.toTestName()} ignored")
- }
-
- fun aggregateResult(): Boolean {
- return reporters.isNotEmpty() && reporters.values.all {
- it.aggregateResult()
- }
- }
-
- fun testCountExpectation(poolId: DevicePoolId, size: Int) {
- execute(poolId) { it.testCountExpectation(size) }
- }
-
- fun removeTests(poolId: DevicePoolId, count: Int) {
- execute(poolId) { it.removeTests(count) }
- }
-
- fun addTestDiscoveredDuringRuntime(poolId: DevicePoolId, test: Test) {
- execute(poolId) { it.addTestDiscoveredDuringRuntime(test) }
+class ProgressReporter(private val batch: TestBatch, private val poolId: DevicePoolId, private val device: DeviceInfo) {
+ fun testStarted(test: Test) {
+ println("${batch.id} | [${poolId.name}]-[${device.serialNumber}] ${test.toTestName()} started")
}
- fun addRetries(poolId: DevicePoolId, count: Int) {
- execute(poolId) { it.addTestRetries(count) }
+ fun testFailed(test: Test) {
+ println("${batch.id} | [${poolId.name}]-[${device.serialNumber}] ${test.toTestName()} failed")
}
- fun progress(): Float {
- val size = reporters.size
- return reporters.values.sumOf {
- it.progress().toDouble()
- }.toFloat() / size
+ fun testPassed(test: Test) {
+ println("${batch.id} | [${poolId.name}]-[${device.serialNumber}] ${test.toTestName()} passed")
}
- fun progress(poolId: DevicePoolId): Float {
- return execute(poolId) { it.progress() }
+ fun testIgnored(test: Test) {
+ println("${batch.id} | [${poolId.name}]-[${device.serialNumber}] ${test.toTestName()} ignored")
}
}
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/tracker/PoolProgressTracker.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/tracker/PoolProgressTracker.kt
deleted file mode 100644
index e0db4b2f1..000000000
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/tracker/PoolProgressTracker.kt
+++ /dev/null
@@ -1,149 +0,0 @@
-package com.malinskiy.marathon.execution.progress.tracker
-
-import com.malinskiy.marathon.actor.StateMachine
-import com.malinskiy.marathon.config.Configuration
-import com.malinskiy.marathon.log.MarathonLogging
-import com.malinskiy.marathon.test.Test
-import java.util.concurrent.atomic.AtomicInteger
-
-class PoolProgressTracker(private val configuration: Configuration) {
-
- private val tests = mutableMapOf>()
- private val runtimeDiscoveredTests = mutableSetOf()
- private val logger = MarathonLogging.logger {}
-
- private fun createState() = StateMachine.create {
- initialState(ProgressTestState.Started)
- state {
- on {
- transitionTo(ProgressTestState.Failed)
- }
- on {
- transitionTo(ProgressTestState.Passed)
- }
- on {
- transitionTo(ProgressTestState.Ignored)
- }
- }
- state {
- on {
- if (configuration.strictMode) {
- transitionTo(ProgressTestState.Failed)
- } else {
- dontTransition()
- }
- }
- on {
- dontTransition()
- }
- }
- state {
- on {
- if (configuration.strictMode) {
- dontTransition()
- } else {
- transitionTo(ProgressTestState.Passed)
- }
- }
-
- }
- state {
- on {
- transitionTo(ProgressTestState.Passed)
- }
- }
- }
-
- private fun updateStatus(test: Test, newStatus: ProgressEvent) {
- synchronized(tests) {
- tests[test]?.transition(newStatus)
- }
- }
-
- private val expectedTestCount = AtomicInteger(0)
- private val completed = AtomicInteger(0)
- private val failed = AtomicInteger(0)
- private val ignored = AtomicInteger(0)
- private val retries = AtomicInteger(0)
-
- fun testStarted(test: Test) {
- synchronized(tests) {
- tests.computeIfAbsent(test) { _ -> createState() }
- }
- }
-
- fun testFailed(test: Test) {
- failed.updateAndGet {
- it + 1
- }
- updateStatus(test, ProgressEvent.Failed)
- }
-
- fun testPassed(test: Test) {
- completed.updateAndGet {
- it + 1
- }
- updateStatus(test, ProgressEvent.Passed)
- }
-
- fun testIgnored(test: Test) {
- ignored.updateAndGet {
- it + 1
- }
- updateStatus(test, ProgressEvent.Ignored)
- }
-
- fun aggregateResult(): Boolean {
- synchronized(tests) {
- val expected = expectedTestCount.get()
- val actual = tests.size
- return if (actual == expected) {
- tests.all {
- when (it.value.state) {
- is ProgressTestState.Passed -> true
- is ProgressTestState.Ignored -> true
- else -> false
- }
- }
- } else {
- logger.error { "Expected to run $expected tests but received results for only $actual" }
- false
- }
- }
- }
-
- fun testCountExpectation(size: Int) {
- expectedTestCount.set(size)
- }
-
- fun removeTests(count: Int) {
- expectedTestCount.updateAndGet {
- it - count
- }
- }
-
- fun progress(): Float {
- return (completed.toFloat() + failed.toFloat() + ignored.toFloat()) / (expectedTestCount.toFloat() + retries.toFloat())
- }
-
- /**
- * This is for parameterized test discovery that can happen at runtime
- * Unfortunately a runtime discovered tests retries it will go through discovery process again, so we have to collect these
- */
- fun addTestDiscoveredDuringRuntime(test: Test) {
- synchronized(runtimeDiscoveredTests) {
- val before = runtimeDiscoveredTests.size
- runtimeDiscoveredTests.add(test)
- val after = runtimeDiscoveredTests.size
- expectedTestCount.updateAndGet {
- it + (after - before)
- }
- }
- }
-
- fun addTestRetries(count: Int) {
- retries.updateAndGet {
- it + count
- }
- }
-}
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/tracker/ProgressEvent.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/tracker/ProgressEvent.kt
deleted file mode 100644
index a41c5ff3f..000000000
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/tracker/ProgressEvent.kt
+++ /dev/null
@@ -1,7 +0,0 @@
-package com.malinskiy.marathon.execution.progress.tracker
-
-sealed class ProgressEvent {
- object Passed : ProgressEvent()
- object Failed : ProgressEvent()
- object Ignored : ProgressEvent()
-}
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/tracker/ProgressTestState.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/tracker/ProgressTestState.kt
deleted file mode 100644
index f8619c366..000000000
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/progress/tracker/ProgressTestState.kt
+++ /dev/null
@@ -1,8 +0,0 @@
-package com.malinskiy.marathon.execution.progress.tracker
-
-sealed class ProgressTestState {
- object Started : ProgressTestState()
- object Passed : ProgressTestState()
- object Failed : ProgressTestState()
- object Ignored : ProgressTestState()
-}
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/QueueActor.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/QueueActor.kt
index a96cf1e50..f5fe2517a 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/QueueActor.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/QueueActor.kt
@@ -2,18 +2,16 @@ package com.malinskiy.marathon.execution.queue
import com.malinskiy.marathon.actor.Actor
import com.malinskiy.marathon.analytics.external.Analytics
-import com.malinskiy.marathon.analytics.internal.pub.Track
import com.malinskiy.marathon.config.Configuration
import com.malinskiy.marathon.device.DeviceInfo
import com.malinskiy.marathon.device.DevicePoolId
-import com.malinskiy.marathon.execution.DevicePoolMessage
import com.malinskiy.marathon.execution.DevicePoolMessage.FromQueue
import com.malinskiy.marathon.execution.TestBatchResults
import com.malinskiy.marathon.execution.TestResult
import com.malinskiy.marathon.execution.TestShard
import com.malinskiy.marathon.execution.TestStatus
import com.malinskiy.marathon.execution.bundle.TestBundleIdentifier
-import com.malinskiy.marathon.execution.progress.ProgressReporter
+import com.malinskiy.marathon.execution.progress.PoolProgressAccumulator
import com.malinskiy.marathon.extension.toBatchingStrategy
import com.malinskiy.marathon.extension.toRetryStrategy
import com.malinskiy.marathon.extension.toSortingStrategy
@@ -28,7 +26,6 @@ import kotlinx.coroutines.channels.SendChannel
import java.util.PriorityQueue
import java.util.Queue
import kotlin.coroutines.CoroutineContext
-import kotlin.math.max
class QueueActor(
private val configuration: Configuration,
@@ -36,10 +33,9 @@ class QueueActor(
private val analytics: Analytics,
private val pool: SendChannel,
private val poolId: DevicePoolId,
- private val progressReporter: ProgressReporter,
- private val track: Track,
private val timer: Timer,
private val testBundleIdentifier: TestBundleIdentifier?,
+ private val poolProgressAccumulator: PoolProgressAccumulator,
poolJob: Job,
coroutineContext: CoroutineContext
) :
@@ -56,11 +52,8 @@ class QueueActor(
private val activeBatches = mutableMapOf()
private val uncompletedTestsRetryCount = mutableMapOf()
- private val testResultReporter = TestResultReporter(poolId, analytics, testShard, configuration, track)
-
init {
queue.addAll(testShard.tests + testShard.flakyTests)
- progressReporter.testCountExpectation(poolId, queue.size)
}
override suspend fun receive(msg: QueueMessage) {
@@ -68,15 +61,19 @@ class QueueActor(
is QueueMessage.RequestBatch -> {
onRequestBatch(msg.device)
}
+
is QueueMessage.IsEmpty -> {
msg.deferred.complete(queue.isEmpty() && activeBatches.isEmpty())
}
+
is QueueMessage.Terminate -> {
onTerminate()
}
+
is QueueMessage.Completed -> {
onBatchCompleted(msg.device, msg.results)
}
+
is QueueMessage.ReturnBatch -> {
onReturnBatch(msg.device, msg.batch)
}
@@ -112,22 +109,20 @@ class QueueActor(
uncompletedTestsRetryCount[it.test] = (uncompletedTestsRetryCount[it.test] ?: 0) + 1
}
- if (uncompletedRetryQuotaExceeded.isNotEmpty()) {
- logger.debug { "uncompletedRetryQuotaExceeded for ${uncompletedRetryQuotaExceeded.joinToString(separator = ", ") { it.test.toTestName() }}" }
- val uncompletedToFailed = uncompletedRetryQuotaExceeded.map {
- it.copy(status = TestStatus.FAILURE)
- }
- for (test in uncompletedToFailed) {
- testResultReporter.testIncomplete(device, test, final = true)
- }
+ for (test in uncompletedRetryQuotaExceeded) {
+ logger.debug { "uncompletedTestRetryQuota exceeded for ${test.test.toTestName()}}" }
+ val testAction = poolProgressAccumulator.testEnded(device, test, final = true)
+ processTestAction(testAction, test)
}
if (uncompleted.isNotEmpty()) {
- for (test in uncompleted) {
- testResultReporter.testIncomplete(device, test, final = false)
+ for (testResult in uncompleted) {
+ val testAction = poolProgressAccumulator.testEnded(device, testResult, final = false)
+ when (testAction) {
+ TestAction.Complete -> processTestAction(testAction, testResult)
+ null -> rerunTest(testResult.test)
+ }
}
- returnTests(uncompleted.map { it.test })
- progressReporter.addRetries(poolId, uncompleted.size)
}
}
@@ -154,32 +149,35 @@ class QueueActor(
}
}
- private fun returnTests(tests: Collection) {
- queue.addAll(tests)
+ private fun rerunTest(test: Test) {
+ queue.add(test)
}
private fun onTerminate() {
close()
}
- private fun handleFinishedTests(finished: Collection, device: DeviceInfo) {
- finished.filter { testShard.flakyTests.contains(it.test) }.let {
- it.forEach {
+ private fun processTestAction(testAction: TestAction?, testResult: TestResult) {
+ when (testAction) {
+ TestAction.Complete -> {
+ //Test has reached final state. No need to run any of the other retries
+ //This doesn't do anything with retries currently in progress
val oldSize = queue.size
- queue.removeAll(listOf(it.test))
- /**
- * Important edge case:
- * 1. Multiple runs of test X scheduled via flaky tests
- * 2. One run of test X finishes and removes other non started flaky retries
- * 3. Another parallel run of test X finishes and should be counted towards reducing the expected tests
- */
- val diff = max(oldSize - queue.size, 1)
- testResultReporter.removeTest(it.test, diff)
- progressReporter.removeTests(poolId, diff)
+ queue.removeAll(setOf(testResult.test))
+ val diff = oldSize - queue.size
+ if (diff >= 0) {
+ poolProgressAccumulator.removeTest(testResult.test, diff)
+ }
}
+
+ null -> Unit
}
+ }
+
+ private fun handleFinishedTests(finished: Collection, device: DeviceInfo) {
finished.forEach {
- testResultReporter.testFinished(device, it)
+ val testAction = poolProgressAccumulator.testEnded(device, it)
+ processTestAction(testAction, it)
}
}
@@ -190,10 +188,11 @@ class QueueActor(
logger.debug { "handle failed tests ${device.serialNumber}" }
val retryList = retryStrategy.process(poolId, failed, testShard)
- progressReporter.addRetries(poolId, retryList.size)
- queue.addAll(retryList.map { it.test })
retryList.forEach {
- testResultReporter.retryTest(device, it)
+ poolProgressAccumulator.retryTest(it.test)
+ val testAction = poolProgressAccumulator.testEnded(device, it)
+ processTestAction(testAction, it)
+ rerunTest(it.test)
}
val (_, noRetries) = failed.partition { testResult ->
@@ -201,7 +200,8 @@ class QueueActor(
}
noRetries.forEach {
- testResultReporter.testFailed(device, it)
+ val testAction = poolProgressAccumulator.testEnded(device, it)
+ processTestAction(testAction, it)
}
}
@@ -218,12 +218,11 @@ class QueueActor(
return
}
if (queueIsEmpty && activeBatches.isEmpty()) {
- pool.send(DevicePoolMessage.FromQueue.Terminated)
+ pool.send(FromQueue.Terminated)
onTerminate()
} else if (queueIsEmpty) {
logger.debug {
- "queue is empty but there are active batches present for " +
- "${activeBatches.keys.joinToString { it }}"
+ "queue is empty but there are active batches present for " + activeBatches.keys.joinToString { it }
}
}
}
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestAction.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestAction.kt
index f864fdce4..ed2255771 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestAction.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestAction.kt
@@ -1,8 +1,9 @@
package com.malinskiy.marathon.execution.queue
-import com.malinskiy.marathon.device.DeviceInfo
-import com.malinskiy.marathon.execution.TestResult
-
sealed class TestAction {
- data class SaveReport(val deviceInfo: DeviceInfo, val testResult: TestResult) : TestAction()
+ /**
+ * Indicates that test reached terminal state of no return according to the current execution strategy logic
+ * Test outputs can be produced after this action and some logic about retries left can be executed
+ */
+ object Complete : TestAction()
}
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestEvent.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestEvent.kt
index e9a69a0ef..520147d76 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestEvent.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestEvent.kt
@@ -4,6 +4,8 @@ import com.malinskiy.marathon.device.DeviceInfo
import com.malinskiy.marathon.execution.TestResult
sealed class TestEvent {
+ object Started : TestEvent()
+
data class Failed(
val device: DeviceInfo,
val testResult: TestResult
@@ -13,13 +15,10 @@ sealed class TestEvent {
val device: DeviceInfo,
val testResult: TestResult
) : TestEvent()
+
+ data class RemoveAttempts(val count: Int) : TestEvent()
- data class Remove(val diff: Int) : TestEvent()
-
- data class Retry(
- val device: DeviceInfo,
- val testResult: TestResult
- ) : TestEvent()
+ object AddRetry : TestEvent()
data class Incomplete(
val device: DeviceInfo,
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestResultReporter.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestResultReporter.kt
deleted file mode 100644
index b1c09949c..000000000
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestResultReporter.kt
+++ /dev/null
@@ -1,168 +0,0 @@
-package com.malinskiy.marathon.execution.queue
-
-import com.malinskiy.marathon.actor.StateMachine
-import com.malinskiy.marathon.analytics.external.Analytics
-import com.malinskiy.marathon.analytics.internal.pub.Track
-import com.malinskiy.marathon.config.Configuration
-import com.malinskiy.marathon.device.DeviceInfo
-import com.malinskiy.marathon.device.DevicePoolId
-import com.malinskiy.marathon.execution.TestResult
-import com.malinskiy.marathon.execution.TestShard
-import com.malinskiy.marathon.log.MarathonLogging
-import com.malinskiy.marathon.test.Test
-import com.malinskiy.marathon.test.toTestName
-
-class TestResultReporter(
- private val poolId: DevicePoolId,
- private val analytics: Analytics,
- shard: TestShard,
- private val configuration: Configuration,
- private val track: Track
-) {
-
- private val tests: HashMap> = HashMap()
-
- private val logger = MarathonLogging.logger("TestResultReporter")
-
- private fun createState(initialCount: Int) = StateMachine.create {
- initialState(TestState.Added(initialCount))
- state {
- on {
- if (!configuration.strictMode || count <= 1) {
- transitionTo(TestState.Passed(it.device, it.testResult), TestAction.SaveReport(it.device, it.testResult))
- } else {
- transitionTo(TestState.Executed(it.device, it.testResult, count - 1))
- }
- }
- on {
- if (configuration.strictMode || count <= 1) {
- transitionTo(TestState.Failed(it.device, it.testResult), TestAction.SaveReport(it.device, it.testResult))
- } else {
- transitionTo(TestState.Executed(it.device, it.testResult, count - 1))
- }
- }
- on {
- if (it.final) {
- transitionTo(TestState.Failed(it.device, it.testResult), TestAction.SaveReport(it.device, it.testResult))
- } else {
- transitionTo(TestState.Executed(it.device, it.testResult, count))
- }
- }
- on {
- dontTransition()
- }
- on {
- transitionTo(this.copy(count = this.count - it.diff))
- }
- }
- state {
- on {
- if (configuration.strictMode || count <= 1) {
- transitionTo(TestState.Failed(it.device, it.testResult), TestAction.SaveReport(it.device, it.testResult))
- } else {
- transitionTo(TestState.Executed(it.device, it.testResult, count - 1))
- }
- }
- on {
- if (it.final) {
- transitionTo(TestState.Failed(it.device, it.testResult), TestAction.SaveReport(it.device, it.testResult))
- } else {
- transitionTo(TestState.Executed(it.device, it.testResult, count))
- }
- }
- on {
- transitionTo(this.copy(count = this.count - it.diff))
- }
- on {
- if (!configuration.strictMode || count <= 1) {
- transitionTo(TestState.Passed(it.device, it.testResult), TestAction.SaveReport(it.device, it.testResult))
- } else {
- transitionTo(TestState.Executed(it.device, it.testResult, count - 1))
- }
- }
- on {
- transitionTo(this.copy(count = this.count + 1))
- }
- }
- state {
- }
- state {
- on {
- dontTransition()
- }
- on {
- dontTransition()
- }
- }
- onTransition {
- if (it as? StateMachine.Transition.Valid !is StateMachine.Transition.Valid) {
- logger.error { "from ${it.fromState} event ${it.event}" }
- }
- trackTestTransition(poolId, it)
- }
- }
-
- init {
- val allTests = shard.tests + shard.flakyTests
- allTests.groupBy { it }.map {
- val count = it.value.size
- it.key.toTestName() to createState(count)
- }.also {
- tests.putAll(it)
- }
- }
-
- fun testFinished(device: DeviceInfo, testResult: TestResult) {
- tests[testResult.test.toTestName()]?.transition(TestEvent.Passed(device, testResult))
- }
-
- fun testFailed(device: DeviceInfo, testResult: TestResult) {
- tests[testResult.test.toTestName()]?.transition(TestEvent.Failed(device, testResult))
- }
-
- fun testIncomplete(device: DeviceInfo, testResult: TestResult, final: Boolean) {
- tests[testResult.test.toTestName()]?.transition(TestEvent.Incomplete(device, testResult, final))
- }
-
- fun retryTest(device: DeviceInfo, testResult: TestResult) {
- tests[testResult.test.toTestName()]?.transition(TestEvent.Retry(device, testResult))
- }
-
- fun removeTest(test: Test, diff: Int) {
- tests[test.toTestName()]?.transition(TestEvent.Remove(diff))
- }
-
- private fun trackTestTransition(poolId: DevicePoolId, transition: StateMachine.Transition) {
- val validTransition = transition as? StateMachine.Transition.Valid
- val final = if (validTransition is StateMachine.Transition.Valid) {
- when (validTransition.sideEffect) {
- is TestAction.SaveReport -> true
- else -> false
- }
- } else false
-
- val (testResult: TestResult?, device: DeviceInfo?) = extractEventAndDevice(transition)
- if (testResult == null || device == null) return
-
- track.test(poolId, device, testResult, final)
- }
-
- private fun extractEventAndDevice(transition: StateMachine.Transition): Pair {
- val event = transition.event
- val testResult: TestResult? = when (event) {
- is TestEvent.Passed -> event.testResult
- is TestEvent.Failed -> event.testResult
- is TestEvent.Retry -> event.testResult
- is TestEvent.Incomplete -> event.testResult
- else -> null
- }
- val device: DeviceInfo? = when (event) {
- is TestEvent.Passed -> event.device
- is TestEvent.Failed -> event.device
- is TestEvent.Retry -> event.device
- is TestEvent.Incomplete -> event.device
- else -> null
- }
- return Pair(testResult, device)
- }
-}
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestState.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestState.kt
index b70c83690..98cad7590 100644
--- a/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestState.kt
+++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/TestState.kt
@@ -1,24 +1,36 @@
package com.malinskiy.marathon.execution.queue
-import com.malinskiy.marathon.device.DeviceInfo
-import com.malinskiy.marathon.execution.TestResult
+/**
+ * TestState accumulates all the information about a single test case for one pool
+ */
sealed class TestState {
- data class Added(val count: Int) : TestState()
+ data class Added(
+ val total: Int,
+ val running: Int = 0,
+ ) : TestState()
+
+ data class Passing(
+ val total: Int,
+ val running: Int,
+ val done: Int,
+ ) : TestState()
- data class Executed(
- val device: DeviceInfo,
- val testResult: TestResult,
- val count: Int
+ data class Failing(
+ val total: Int,
+ val running: Int,
+ val done: Int,
) : TestState()
data class Failed(
- val device: DeviceInfo,
- val testResult: TestResult
+ val total: Int,
+ val running: Int,
+ val done: Int,
) : TestState()
data class Passed(
- val device: DeviceInfo,
- val testResult: TestResult
+ val total: Int,
+ val running: Int,
+ val done: Int,
) : TestState()
}
diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/test_state.md b/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/test_state.md
new file mode 100644
index 000000000..d1c820188
--- /dev/null
+++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/queue/test_state.md
@@ -0,0 +1,41 @@
+```mermaid
+stateDiagram-v2
+ [*] --> Started
+
+ Started --> Failed : Failed
+ Started --> Passed : Passed
+ Started --> Ignored : Ignored
+
+ Passed --> Failed : Failed
+
+ Failed --> Passed : Passed
+
+ Ignored --> Passed : Passed
+
+ Failed --> [*]
+ Passed --> [*]
+ Ignored --> [*]
+```
+
+```mermaid
+stateDiagram-v2
+ [*] --> Added
+
+ Added --> Passed : Passed
+ Added --> Passing : Passed
+ Added --> Failed : Failed
+ Added --> Passing : Failed
+ Added --> Failed : Incomplete
+ Added --> Passing : Incomplete
+
+
+ Passed --> Failed : Failed
+
+ Failed --> Passed : Passed
+
+ Ignored --> Passed : Passed
+
+ Failed --> [*]
+ Passed --> [*]
+ Ignored --> [*]
+```
diff --git a/core/src/test/kotlin/com/malinskiy/marathon/TestGenerator.kt b/core/src/test/kotlin/com/malinskiy/marathon/TestGenerator.kt
index 2d44122ca..4df9d690f 100644
--- a/core/src/test/kotlin/com/malinskiy/marathon/TestGenerator.kt
+++ b/core/src/test/kotlin/com/malinskiy/marathon/TestGenerator.kt
@@ -7,8 +7,8 @@ fun generateTest(
pkg: String = "pkg",
clazz: String = "clazz",
method: String = "method",
- annotations: List = emptyList()
-) = Test(pkg, clazz, method, annotations)
+ metaProperties: List = emptyList()
+) = Test(pkg, clazz, method, metaProperties)
fun generateTests(
count: Int,
@@ -20,4 +20,4 @@ fun generateTests(
return (0 until count).map {
Test("$pkg$it", "$clazz$it", "$method$it", annotations)
}
-}
\ No newline at end of file
+}
diff --git a/core/src/test/kotlin/com/malinskiy/marathon/device/DeviceStub.kt b/core/src/test/kotlin/com/malinskiy/marathon/device/DeviceStub.kt
index 47e9fbf2f..85c4b44e8 100644
--- a/core/src/test/kotlin/com/malinskiy/marathon/device/DeviceStub.kt
+++ b/core/src/test/kotlin/com/malinskiy/marathon/device/DeviceStub.kt
@@ -2,11 +2,9 @@ package com.malinskiy.marathon.device
import com.malinskiy.marathon.config.Configuration
import com.malinskiy.marathon.execution.TestBatchResults
-import com.malinskiy.marathon.execution.progress.ProgressReporter
import com.malinskiy.marathon.log.MarathonLogging
import com.malinskiy.marathon.test.TestBatch
import kotlinx.coroutines.CompletableDeferred
-import mu.KLogger
class DeviceStub(
override var operatingSystem: OperatingSystem = OperatingSystem("25"),
@@ -24,8 +22,7 @@ class DeviceStub(
configuration: Configuration,
devicePoolId: DevicePoolId,
testBatch: TestBatch,
- deferred: CompletableDeferred,
- progressReporter: ProgressReporter
+ deferred: CompletableDeferred
) {
}
diff --git a/core/src/test/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulatorTest.kt b/core/src/test/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulatorTest.kt
new file mode 100644
index 000000000..9d38c8810
--- /dev/null
+++ b/core/src/test/kotlin/com/malinskiy/marathon/execution/progress/PoolProgressAccumulatorTest.kt
@@ -0,0 +1,809 @@
+package com.malinskiy.marathon.execution.progress
+
+import com.malinskiy.marathon.analytics.external.Analytics
+import com.malinskiy.marathon.analytics.internal.pub.Track
+import com.malinskiy.marathon.config.Configuration
+import com.malinskiy.marathon.config.strategy.ExecutionMode
+import com.malinskiy.marathon.config.strategy.ExecutionStrategyConfiguration
+import com.malinskiy.marathon.config.vendor.VendorConfiguration
+import com.malinskiy.marathon.device.DevicePoolId
+import com.malinskiy.marathon.execution.TestResult
+import com.malinskiy.marathon.execution.TestShard
+import com.malinskiy.marathon.execution.TestStatus
+import com.malinskiy.marathon.generateTest
+import com.malinskiy.marathon.report.getDevice
+import com.nhaarman.mockitokotlin2.mock
+import com.nhaarman.mockitokotlin2.reset
+import org.amshove.kluent.shouldBeEqualTo
+import org.junit.jupiter.api.BeforeEach
+import org.junit.jupiter.api.Test
+import java.io.File
+
+class PoolProgressAccumulatorTest {
+ private val track = mock