Merge pull request MarathonLabs#758 from MarathonLabs/feature/executi…

…on-strategy

feat(core): execution strategy

configuration/src/main/kotlin/com/malinskiy/marathon/config/Configuration.kt

@@ -2,6 +2,7 @@ package com.malinskiy.marathon.config
 
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize
 import com.malinskiy.marathon.config.strategy.BatchingStrategyConfiguration
+import com.malinskiy.marathon.config.strategy.ExecutionStrategyConfiguration
 import com.malinskiy.marathon.config.strategy.FlakinessStrategyConfiguration
 import com.malinskiy.marathon.config.strategy.PoolingStrategyConfiguration
 import com.malinskiy.marathon.config.strategy.RetryStrategyConfiguration
@@ -18,7 +19,7 @@ private const val DEFAULT_DEVICE_INITIALIZATION_TIMEOUT_MILLIS = 180_000L
 data class Configuration private constructor(
  val name: String,
  val outputDir: File,
- 
+
  val outputConfiguration: OutputConfiguration,
 
  val analyticsConfiguration: AnalyticsConfiguration,
@@ -31,8 +32,8 @@ data class Configuration private constructor(
  val filteringConfiguration: FilteringConfiguration,
 
  val ignoreFailures: Boolean,
+ val executionStrategy: ExecutionStrategyConfiguration,
  val isCodeCoverageEnabled: Boolean,
- val strictMode: Boolean,
  val uncompletedTestRetryQuota: Int,
 
  val testClassRegexes: Collection<Regex>,
@@ -65,7 +66,7 @@ data class Configuration private constructor(
  "filtering" to filteringConfiguration.toString(),
  "ignoreFailures" to ignoreFailures.toString(),
  "isCodeCoverageEnabled" to isCodeCoverageEnabled.toString(),
- "strictMode" to strictMode.toString(),
+ "executionStrategy" to executionStrategy.toString(),
  "testClassRegexes" to testClassRegexes.toString(),
  "includeSerialRegexes" to includeSerialRegexes.toString(),
  "excludeSerialRegexes" to excludeSerialRegexes.toString(),
@@ -96,7 +97,7 @@ data class Configuration private constructor(
  if (filteringConfiguration != other.filteringConfiguration) return false
  if (ignoreFailures != other.ignoreFailures) return false
  if (isCodeCoverageEnabled != other.isCodeCoverageEnabled) return false
- if (strictMode != other.strictMode) return false
+ if (executionStrategy != other.executionStrategy) return false
  if (uncompletedTestRetryQuota != other.uncompletedTestRetryQuota) return false
  //For testing we need to compare configuration instances. Unfortunately Regex equality is broken so need to map it to String
  if (testClassRegexes.map { it.pattern } != other.testClassRegexes.map { it.pattern }) return false
@@ -127,7 +128,7 @@ data class Configuration private constructor(
  result = 31 * result + filteringConfiguration.hashCode()
  result = 31 * result + ignoreFailures.hashCode()
  result = 31 * result + isCodeCoverageEnabled.hashCode()
- result = 31 * result + strictMode.hashCode()
+ result = 31 * result + executionStrategy.hashCode()
  result = 31 * result + uncompletedTestRetryQuota
  result = 31 * result + testClassRegexes.hashCode()
  result = 31 * result + includeSerialRegexes.hashCode()
@@ -141,38 +142,39 @@ data class Configuration private constructor(
  result = 31 * result + deviceInitializationTimeoutMillis.hashCode()
  return result
  }
+
  data class Builder(
- val name: String,
- val outputDir: File,
- var analyticsConfiguration: AnalyticsConfiguration = AnalyticsConfiguration.DisabledAnalytics,
- var poolingStrategy: PoolingStrategyConfiguration = PoolingStrategyConfiguration.OmniPoolingStrategyConfiguration,
- var shardingStrategy: ShardingStrategyConfiguration = ShardingStrategyConfiguration.ParallelShardingStrategyConfiguration,
- var sortingStrategy: SortingStrategyConfiguration = SortingStrategyConfiguration.NoSortingStrategyConfiguration,
- var batchingStrategy: BatchingStrategyConfiguration = BatchingStrategyConfiguration.IsolateBatchingStrategyConfiguration,
- var flakinessStrategy: FlakinessStrategyConfiguration = FlakinessStrategyConfiguration.IgnoreFlakinessStrategyConfiguration,
- var retryStrategy: RetryStrategyConfiguration = RetryStrategyConfiguration.NoRetryStrategyConfiguration,
- var filteringConfiguration: FilteringConfiguration = FilteringConfiguration(emptyList(), emptyList()),
-
- var ignoreFailures: Boolean = false,
- var isCodeCoverageEnabled: Boolean = false,
- var strictMode: Boolean = false,
- var uncompletedTestRetryQuota: Int = Integer.MAX_VALUE,
-
- var testClassRegexes: Collection<Regex> = listOf(Regex("^((?!Abstract).)*Test[s]*$")),
- var includeSerialRegexes: Collection<Regex> = emptyList(),
- var excludeSerialRegexes: Collection<Regex> = emptyList(),
-
- var testBatchTimeoutMillis: Long = DEFAULT_BATCH_EXECUTION_TIMEOUT_MILLIS,
- var testOutputTimeoutMillis: Long = DEFAULT_OUTPUT_TIMEOUT_MILLIS,
- var debug: Boolean = true,
-
- var screenRecordingPolicy: ScreenRecordingPolicy = ScreenRecordingPolicy.ON_FAILURE,
-
- var analyticsTracking: Boolean = false,
- var deviceInitializationTimeoutMillis: Long = DEFAULT_DEVICE_INITIALIZATION_TIMEOUT_MILLIS,
-
- var outputConfiguration: OutputConfiguration = OutputConfiguration(),
- var vendorConfiguration: VendorConfiguration = VendorConfiguration.EmptyVendorConfiguration(),
+  val name: String,
+  val outputDir: File,
+  var analyticsConfiguration: AnalyticsConfiguration = AnalyticsConfiguration.DisabledAnalytics,
+  var poolingStrategy: PoolingStrategyConfiguration = PoolingStrategyConfiguration.OmniPoolingStrategyConfiguration,
+  var shardingStrategy: ShardingStrategyConfiguration = ShardingStrategyConfiguration.ParallelShardingStrategyConfiguration,
+  var sortingStrategy: SortingStrategyConfiguration = SortingStrategyConfiguration.NoSortingStrategyConfiguration,
+  var batchingStrategy: BatchingStrategyConfiguration = BatchingStrategyConfiguration.IsolateBatchingStrategyConfiguration,
+  var flakinessStrategy: FlakinessStrategyConfiguration = FlakinessStrategyConfiguration.IgnoreFlakinessStrategyConfiguration,
+  var retryStrategy: RetryStrategyConfiguration = RetryStrategyConfiguration.NoRetryStrategyConfiguration,
+  var filteringConfiguration: FilteringConfiguration = FilteringConfiguration(emptyList(), emptyList()),
+
+  var ignoreFailures: Boolean = false,
+  var isCodeCoverageEnabled: Boolean = false,
+  var executionStrategy: ExecutionStrategyConfiguration = ExecutionStrategyConfiguration(),
+  var uncompletedTestRetryQuota: Int = Integer.MAX_VALUE,
+
+  var testClassRegexes: Collection<Regex> = listOf(Regex("^((?!Abstract).)*Test[s]*$")),
+  var includeSerialRegexes: Collection<Regex> = emptyList(),
+  var excludeSerialRegexes: Collection<Regex> = emptyList(),
+
+  var testBatchTimeoutMillis: Long = DEFAULT_BATCH_EXECUTION_TIMEOUT_MILLIS,
+  var testOutputTimeoutMillis: Long = DEFAULT_OUTPUT_TIMEOUT_MILLIS,
+  var debug: Boolean = true,
+
+  var screenRecordingPolicy: ScreenRecordingPolicy = ScreenRecordingPolicy.ON_FAILURE,
+
+  var analyticsTracking: Boolean = false,
+  var deviceInitializationTimeoutMillis: Long = DEFAULT_DEVICE_INITIALIZATION_TIMEOUT_MILLIS,
+
+  var outputConfiguration: OutputConfiguration = OutputConfiguration(),
+  var vendorConfiguration: VendorConfiguration = VendorConfiguration.EmptyVendorConfiguration(),
  ) {
  fun build(): Configuration {
  return Configuration(
@@ -189,7 +191,7 @@ data class Configuration private constructor(
  filteringConfiguration = filteringConfiguration,
  ignoreFailures = ignoreFailures,
  isCodeCoverageEnabled = isCodeCoverageEnabled,
- strictMode = strictMode,
+ executionStrategy = executionStrategy,
  uncompletedTestRetryQuota = uncompletedTestRetryQuota,
  testClassRegexes = testClassRegexes,
  includeSerialRegexes = includeSerialRegexes,

configuration/src/main/kotlin/com/malinskiy/marathon/config/LogicalConfigurationValidator.kt

@@ -1,7 +1,9 @@
 package com.malinskiy.marathon.config
 
 import com.malinskiy.marathon.config.exceptions.ConfigurationException
+import com.malinskiy.marathon.config.strategy.ExecutionMode
 import com.malinskiy.marathon.config.strategy.FlakinessStrategyConfiguration
+import com.malinskiy.marathon.config.strategy.RetryStrategyConfiguration
 import com.malinskiy.marathon.config.strategy.ShardingStrategyConfiguration
 import com.malinskiy.marathon.config.vendor.VendorConfiguration
 
@@ -28,5 +30,27 @@ class LogicalConfigurationValidator : ConfigurationValidator {
 
  else -> Unit
  }
+
+ when(configuration.executionStrategy.mode) {
+ ExecutionMode.ANY_SUCCESS -> {
+ if (configuration.shardingStrategy !is ShardingStrategyConfiguration.ParallelShardingStrategyConfiguration) {
+ throw ConfigurationException(
+ "Configuration is invalid: can't use complex sharding and any success execution strategy at the same time. Consult documentation for the any success execution logic"
+ )
+ }
+ }
+ ExecutionMode.ALL_SUCCESS -> {
+ if (configuration.flakinessStrategy !is FlakinessStrategyConfiguration.IgnoreFlakinessStrategyConfiguration) {
+ throw ConfigurationException(
+ "Configuration is invalid: can't use complex flakiness strategy and all success execution strategy at the same time. Consult documentation for the all success execution logic"
+ )
+ }
+ if (configuration.retryStrategy !is RetryStrategyConfiguration.NoRetryStrategyConfiguration) {
+ throw ConfigurationException(
+ "Configuration is invalid: can't use complex retry strategy and all success execution strategy at the same time. Consult documentation for the all success execution logic"
+ )
+ }
+ }
+ }
  }
 }

configuration/src/main/kotlin/com/malinskiy/marathon/config/strategy/ExecutionStrategyConfiguration.kt

@@ -0,0 +1,31 @@
+package com.malinskiy.marathon.config.strategy
+
+import com.fasterxml.jackson.annotation.JsonProperty
+
+/**
+ * @property fast fail-fast or success-fast depending on the value of [mode]. This doesn't affect the result
+ * of the run, it only saves compute time
+ */
+data class ExecutionStrategyConfiguration(
+ @JsonProperty("mode") val mode: ExecutionMode = ExecutionMode.ANY_SUCCESS,
+ @JsonProperty("fast") val fast: Boolean = true,
+)
+
+/**
+ * @property ANY_SUCCESS test passes if any of executions is passing
+ * this mode works only if there is no complex sharding strategy applied
+ * 
+ * Why: it doesn't make sense when user asks for N number of tests
+ * to run explicitly, and we pass on the first one. Sharding used to verify probability of passing with an explicit boundary for precision
+ * @property ALL_SUCCESS test passes if and only if all the executions are passing
+ * this mode works only if there are no retries, i.e. no complex flakiness strategy, no retry strategy
+ * 
+ * Why: when adding retries to tests with retry+flakiness strategies users want to trade-off cost for reliability, i.e. add more retries
+ * and pass if one of them passes, so retries only make sense for the [ANY_SUCCESS] mode. When we use [ALL_SUCCESS] mode it means user
+ * wants to verify each test with a number of tries (they are not retries per se) and pass only if all of them succeed. This is the case
+ * when fixing a flaky test or adding a new test, and we want to have a signal that the test is fixed/not flaky.
+ */
+enum class ExecutionMode {
+ @JsonProperty("ANY_SUCCESS") ANY_SUCCESS,
+ @JsonProperty("ALL_SUCCESS") ALL_SUCCESS,
+}

configuration/src/test/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactoryTest.kt

@@ -163,7 +163,6 @@ class ConfigurationFactoryTest {
  configuration.excludeSerialRegexes.joinToString(separator = "") { it.pattern } shouldBeEqualTo """emulator-5002""".toRegex().pattern
  configuration.ignoreFailures shouldBeEqualTo false
  configuration.isCodeCoverageEnabled shouldBeEqualTo false
- configuration.strictMode shouldBeEqualTo true
  configuration.testBatchTimeoutMillis shouldBeEqualTo 20_000
  configuration.testOutputTimeoutMillis shouldBeEqualTo 30_000
  configuration.debug shouldBeEqualTo true

configuration/src/test/resources/fixture/config/sample_1.yaml

@@ -68,7 +68,9 @@ excludeSerialRegexes:
  - "emulator-5002"
 ignoreFailures: false
 isCodeCoverageEnabled: false
-strictMode: true
+executionStrategy:
+ mode: ANY_SUCCESS
+ fast: true
 testBatchTimeoutMillis: 20000
 testOutputTimeoutMillis: 30000
 debug: true

configuration/src/test/resources/fixture/config/sample_1_rp.yaml

@@ -66,7 +66,9 @@ excludeSerialRegexes:
  - "emulator-5002"
 ignoreFailures: false
 isCodeCoverageEnabled: false
-strictMode: true
+executionStrategy:
+ mode: ANY_SUCCESS
+ fast: true
 testBatchTimeoutMillis: 20000
 testOutputTimeoutMillis: 30000
 debug: true

core/src/main/kotlin/com/malinskiy/marathon/Marathon.kt

@@ -19,7 +19,6 @@ import com.malinskiy.marathon.execution.TestParser
 import com.malinskiy.marathon.execution.TestShard
 import com.malinskiy.marathon.execution.bundle.TestBundleIdentifier
 import com.malinskiy.marathon.execution.command.parse.MarathonTestParseCommand
-import com.malinskiy.marathon.execution.progress.ProgressReporter
 import com.malinskiy.marathon.execution.withRetry
 import com.malinskiy.marathon.extension.toFlakinessStrategy
 import com.malinskiy.marathon.extension.toShardingStrategy
@@ -34,7 +33,6 @@ import com.malinskiy.marathon.usageanalytics.UsageAnalytics
 import com.malinskiy.marathon.usageanalytics.tracker.Event
 import kotlinx.coroutines.NonCancellable
 import kotlinx.coroutines.async
-import kotlinx.coroutines.delay
 import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.withContext
 import kotlinx.coroutines.withTimeoutOrNull
@@ -52,7 +50,6 @@ class Marathon(
  private val logConfigurator: MarathonLogConfigurator,
  private val tracker: TrackerInternal,
  private val analytics: Analytics,
- private val progressReporter: ProgressReporter,
  private val track: Track,
  private val timer: Timer,
  private val marathonTestParseCommand: MarathonTestParseCommand
@@ -142,7 +139,6 @@ class Marathon(
  analytics,
  configuration,
  shard,
- progressReporter,
  track,
  timer,
  testBundleIdentifier,
@@ -155,12 +151,13 @@ class Marathon(
  }
  configuration.outputDir.mkdirs()
 
- if (parsedFilteredTests.isNotEmpty()) {
+ val result = if (parsedFilteredTests.isNotEmpty()) {
  scheduler.execute()
+ } else {
+ true
  }
 
  onFinish(analytics, deviceProvider)
- val result = progressReporter.aggregateResult()
 
  stopKoin()
  return result

core/src/main/kotlin/com/malinskiy/marathon/device/Device.kt

@@ -2,7 +2,6 @@ package com.malinskiy.marathon.device
 
 import com.malinskiy.marathon.config.Configuration
 import com.malinskiy.marathon.execution.TestBatchResults
-import com.malinskiy.marathon.execution.progress.ProgressReporter
 import com.malinskiy.marathon.test.TestBatch
 import kotlinx.coroutines.CompletableDeferred
 import mu.KLogger
@@ -45,7 +44,6 @@ interface Device {
  devicePoolId: DevicePoolId,
  testBatch: TestBatch,
  deferred: CompletableDeferred<TestBatchResults>,
- progressReporter: ProgressReporter
  )
 
  /**

core/src/main/kotlin/com/malinskiy/marathon/di/Modules.kt

@@ -7,7 +7,6 @@ import com.malinskiy.marathon.analytics.external.AnalyticsFactory
 import com.malinskiy.marathon.analytics.internal.pub.Track
 import com.malinskiy.marathon.config.Configuration
 import com.malinskiy.marathon.execution.command.parse.MarathonTestParseCommand
-import com.malinskiy.marathon.execution.progress.ProgressReporter
 import com.malinskiy.marathon.io.FileManager
 import com.malinskiy.marathon.json.FileSerializer
 import com.malinskiy.marathon.time.SystemTimer
@@ -37,12 +36,11 @@ val coreModule = module {
  }
  single<Clock> { Clock.systemDefaultZone() }
  single<Timer> { SystemTimer(get()) }
- single { ProgressReporter(get()) }
  single {
  val configuration = get<Configuration>()
  MarathonTestParseCommand(configuration.outputDir)
  }
- single { Marathon(get(), get(), get(), get(), get(), get(), get(), get(), get(), get(), get()) }
+ single { Marathon(get(), get(), get(), get(), get(), get(), get(), get(), get(), get()) }
 }
 
 fun marathonStartKoin(configuration: Configuration, modules: List<Module>): KoinApplication {