Optimize dependency resolution code path and introduce experiment to …

…limit parallelism (#140)

Fixes #140

build.gradle

@@ -143,6 +143,9 @@ grazel {
             sha = "154cdfa4f6f552a9873e2b4448f7a80415cb3427c4c771a50c6a8a8b434ffd0a"
         }
     }
+    experiments {
+        limitDependencyResolutionParallelism.set(true)
+    }
 }
 
 idea {

grazel-gradle-plugin/src/main/kotlin/com/grab/grazel/GrazelExtension.kt

@@ -18,6 +18,7 @@ package com.grab.grazel
 
 import com.grab.grazel.extension.AndroidExtension
 import com.grab.grazel.extension.DependenciesExtension
+import com.grab.grazel.extension.ExperimentsExtension
 import com.grab.grazel.extension.HybridExtension
 import com.grab.grazel.extension.RulesExtension
 import groovy.lang.Closure
@@ -55,6 +56,8 @@ open class GrazelExtension(
 
     val hybrid = HybridExtension(rootProject.objects)
 
+    val experiments = ExperimentsExtension(rootProject.objects)
+
     /**
      * Android specific configuration used to configure parameters for android_binary or other android related
      * rules
@@ -190,4 +193,35 @@ open class GrazelExtension(
         closure.delegate = hybrid
         closure.call()
     }
+
+    /**
+     * Extension to configure experiments
+     *
+     * ```
+     * experiments {
+     *
+     * }
+     * ```
+     * @see ExperimentsExtension
+     * @param block Configuration block with [ExperimentsExtension] as the receiver
+     */
+    fun experiments(block: ExperimentsExtension.() -> Unit) {
+        block(experiments)
+    }
+
+    /**
+     * Extension to configure experiments
+     *
+     * ```
+     * experiments {
+     *
+     * }
+     * ```
+     * @see ExperimentsExtension
+     * @param closure Closure block with [ExperimentsExtension] as the delegate
+     */
+    fun experiments(closure: Closure<*>) {
+        closure.delegate = experiments
+        closure.call()
+    }
 }

grazel-gradle-plugin/src/main/kotlin/com/grab/grazel/bazel/exec/Bazel.kt

@@ -62,7 +62,11 @@ internal fun ExecOperations.bazelCommand(
         add("--noshow_progress")
         add("--color=yes")
     }
-    logger.quiet("${"Running".ansiGreen} ${commands.joinToString(separator = " ").ansiYellow}")
+    logger.quiet(
+        "${"Running".ansiGreen} ${
+            commands.toList().dropLast(2).joinToString(separator = " ").ansiYellow
+        }"
+    )
     return exec {
         commandLine(*commands.toTypedArray())
         standardOutput = outputStream ?: LogOutputStream(logger, LogLevel.QUIET)

grazel-gradle-plugin/src/main/kotlin/com/grab/grazel/extension/ExperimentsExtension.kt

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2024 Grabtaxi Holdings PTE LTD (GRAB)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.grab.grazel.extension
+
+import org.gradle.api.model.ObjectFactory
+import org.gradle.api.provider.Property
+import org.gradle.kotlin.dsl.property
+
+/**
+ * Additional experiments configuration
+ */
+data class ExperimentsExtension(private val objects: ObjectFactory) {
+
+    /**
+     * Limits the no of concurrent Gradle dependency resolution requests by establishing inter task dependencies
+     * mirroring project dependency graph such that successors are always resolved first before predecessor
+     * project is resolved. This is useful for large project with large dependency which can be memory intensive
+     * to compute.
+     *
+     * Enabling this does not actually control the no of parallel requests, for that
+     * please use `--max-workers` property from Gradle.
+     */
+    val limitDependencyResolutionParallelism: Property<Boolean> = objects
+        .property<Boolean>()
+        .convention(false)
+}

grazel-gradle-plugin/src/main/kotlin/com/grab/grazel/gradle/variant/AndroidVariants.kt

@@ -93,14 +93,26 @@ abstract class AndroidNonVariant<T>(
 
     override val name
         get() = backingVariant.name + when (variantType) {
-            AndroidTest -> AndroidTest.name
-            Test -> Test.name
+            AndroidTest, Test -> variantType.testSuffix
             Lint -> Lint.name
             else -> ""
         }
 
     override val baseName get() = backingVariant.name.capitalize()
 
+    override val extendsFrom: Set<String> by lazy {
+        buildList {
+            add(DEFAULT_VARIANT)
+            if (variantType.isTest) {
+                add(backingVariant.name)
+                add(TEST_VARIANT)
+                if (variantType.isAndroidTest) {
+                    add(ANDROID_TEST_VARIANT)
+                }
+            }
+        }.toSet()
+    }
+
     override val variantConfigurations: Set<Configuration>
         get() = super.variantConfigurations
             .asSequence()
@@ -159,13 +171,7 @@ class AndroidBuildType(
     backingVariant = backingVariant,
     variantType = variantType,
     toIgnoreKeywords = flavors
-) {
-    override val extendsFrom: Set<String> = buildList {
-        add(DEFAULT_VARIANT)
-        if (variantType.isTest) add(backingVariant.name)
-        if (variantType == Test) add(TEST_VARIANT)
-    }.toSet()
-}
+)
 
 /**
  * A [Variant] implementation to denote a [ProductFlavor] with [toIgnoreKeywords] set to buildTypes
@@ -187,13 +193,7 @@ class AndroidFlavor(
     backingVariant = backingVariant,
     variantType = variantType,
     toIgnoreKeywords = buildTypes
-) {
-    override val extendsFrom: Set<String> = buildList {
-        add(DEFAULT_VARIANT)
-        if (variantType.isTest) add(backingVariant.name)
-        if (variantType == Test) add(TEST_VARIANT)
-    }.toSet()
-}
+)
 
 data class DefaultVariantData(
     val project: Project,
@@ -225,6 +225,7 @@ class AndroidDefaultVariant(
     override val backingVariant: DefaultVariantData get() = defaultVariantData
     override val project: Project get() = defaultVariantData.project
     override val variantType: VariantType get() = defaultVariantData.variantType
+
     override val extendsFrom: Set<String> = buildSet {
         if (variantType.isTest) add(DEFAULT_VARIANT)
         if (variantType.isAndroidTest) add(TEST_VARIANT)

grazel-gradle-plugin/src/main/kotlin/com/grab/grazel/gradle/variant/ConfigurationParsingVariant.kt

@@ -79,7 +79,7 @@ interface ConfigurationParsingVariant<T> : Variant<T> {
                     AndroidBuild -> configName.startsWith("kapt${namePattern.capitalize()}")
                     AndroidTest -> configName.startsWith("kaptAndroidTest${basePattern.capitalize()}")
                     Test -> configName.startsWith("kaptTest${basePattern.capitalize()}")
-                    VariantType.Lint -> false
+                    Lint -> false
                     JvmBuild -> error("Invalid variant type ${JvmBuild.name} for Android variant")
                 }
             }.addTo(this)

grazel-gradle-plugin/src/main/kotlin/com/grab/grazel/gradle/variant/Variant.kt

@@ -31,6 +31,7 @@ import org.gradle.api.artifacts.Configuration
  */
 interface Variant<T> {
     val name: String
+
     val backingVariant: T
 
     val project: Project
@@ -95,12 +96,15 @@ fun BaseVariant.toVariantType(): VariantType = when (this) {
 
 val Variant<*>.isBase get() = name == DEFAULT_VARIANT
 
+val Variant<*>.id get() = name + variantType.toString()
+
 /**
  * Bridge function to map [ConfigurationScope] to [VariantType]
  * Not required once fully migrated to [Variant] APIs
  *
  * @return whether this [VariantType] corresponds to [ConfigurationScope]
  */
+@Deprecated(message = "Deprecated, new code should use Variant API directly")
 fun VariantType.isConfigScope(
     project: Project,
     configurationScope: ConfigurationScope

grazel-gradle-plugin/src/main/kotlin/com/grab/grazel/gradle/variant/VariantBuilder.kt

@@ -70,7 +70,7 @@ constructor(
                     ),
                     AndroidDefaultVariant(
                         project = project,
-                        variantType = VariantType.Lint,
+                        variantType = Lint,
                         ignoreKeywords = flavorsBuildTypes
                     )
                 )
@@ -96,7 +96,7 @@ constructor(
                     }
                 (parsedAndroidVariants + defaultVariants)
                     .asSequence()
-                    .distinctBy { it.name + it.variantType }
+                    .distinctBy { it.id }
                     .sortedBy { it.name.length }
                     .toSet()
             } else if (project.isJvm) {
@@ -118,86 +118,84 @@ constructor(
 
     override fun onVariants(project: Project, action: (Variant<*>) -> Unit) {
         project.afterEvaluate {
+            val variantCache = ConcurrentHashMap<String, Variant<*>>()
             if (project.isAndroid) {
-                val flavors = variantDataSource.getFlavors(project)
-                val flavorNames = flavors.map { it.name }.toSet()
-                val buildTypes = variantDataSource.getBuildTypes(project)
-                val buildTypeNames = buildTypes.map { it.name }.toSet()
-                val flavorsBuildTypes = (flavorNames + buildTypeNames).toSet()
-                action(
+                val allFlavors = variantDataSource.getFlavors(project)
+                val allFlavorNames = allFlavors.map { it.name }.toSet()
+                val allBuildTypes = variantDataSource.getBuildTypes(project)
+                val allBuildTypeNames = allBuildTypes.map { it.name }.toSet()
+                val allFlavorBuildTypes = (allFlavorNames + allBuildTypeNames).toSet()
+
+                fun variantAction(variant: Variant<*>) {
+                    if (!variantCache.containsKey(variant.id)) {
+                        action(variant)
+                        variantCache[variant.id] = variant
+                    }
+                }
+
+                variantAction(
                     AndroidDefaultVariant(
                         project = project,
                         variantType = AndroidBuild,
-                        ignoreKeywords = flavorsBuildTypes
+                        ignoreKeywords = allFlavorBuildTypes
                     )
                 )
-                action(
+                variantAction(
                     AndroidDefaultVariant(
                         project = project,
                         variantType = Test,
-                        ignoreKeywords = flavorsBuildTypes
+                        ignoreKeywords = allFlavorBuildTypes
                     )
                 )
-                action(
+                variantAction(
                     AndroidDefaultVariant(
                         project = project,
                         variantType = AndroidTest,
-                        ignoreKeywords = flavorsBuildTypes
+                        ignoreKeywords = allFlavorBuildTypes
                     )
                 )
-                action(
+                variantAction(
                     AndroidDefaultVariant(
                         project = project,
                         variantType = Lint,
-                        ignoreKeywords = flavorsBuildTypes
+                        ignoreKeywords = allFlavorBuildTypes
                     )
                 )
 
                 variantDataSource.migratableVariants(project) { variant ->
                     action(AndroidVariant(project, variant))
-                }
-
-
-                if (flavors.isNotEmpty()) {
-                    // Special case, if this module does not have flavors declared then variants
-                    // will be just buildTypes. Since we already would have passed buildType variants
-                    // above we don't need to pass it again here.
-                    buildTypes
-                        .asSequence()
-                        .flatMap { buildType ->
-                            VariantType.values()
-                                .filter { it != JvmBuild && it != Lint }
-                                .map { variantType ->
+                    if (allFlavors.isNotEmpty()) {
+                        VariantType.values()
+                            .asSequence()
+                            .filter { it != JvmBuild && it != Lint }
+                            .forEach { variantType ->
+                                variantAction(
                                     AndroidBuildType(
                                         project = project,
-                                        backingVariant = buildType,
+                                        backingVariant = variant.buildType,
                                         variantType = variantType,
-                                        flavors = flavorNames
+                                        flavors = allFlavorNames
                                     )
-                                }
-                        }.distinctBy { it.name + it.variantType }
-                        .forEach(action)
-
-                    VariantType
-                        .values()
-                        .asSequence()
-                        .filter { it != JvmBuild && it != Lint }
-                        .flatMap { variantType ->
-                            flavors.map { flavor ->
-                                AndroidFlavor(
-                                    project,
-                                    flavor,
-                                    variantType,
-                                    buildTypeNames
                                 )
+                                variant.productFlavors.forEach { flavor ->
+                                    variantAction(
+                                        AndroidFlavor(
+                                            project = project,
+                                            backingVariant = flavor,
+                                            variantType = variantType,
+                                            buildTypes = allBuildTypeNames
+                                        )
+                                    )
+                                }
                             }
-                        }.forEach(action)
+                    }
                 }
             } else if (project.isJvm) {
                 action(JvmVariant(project = project, variantType = JvmBuild))
                 action(JvmVariant(project = project, variantType = Test))
                 action(JvmVariant(project = project, variantType = Lint))
             }
         }
+        variantCache.clear()
     }
 }

grazel-gradle-plugin/src/main/kotlin/com/grab/grazel/tasks/internal/ComputeWorkspaceDependenciesTask.kt

@@ -26,6 +26,7 @@ import org.gradle.api.Project
 import org.gradle.api.file.RegularFile
 import org.gradle.api.file.RegularFileProperty
 import org.gradle.api.provider.ListProperty
+import org.gradle.api.provider.Property
 import org.gradle.api.tasks.CacheableTask
 import org.gradle.api.tasks.InputFiles
 import org.gradle.api.tasks.OutputFile
@@ -60,6 +61,7 @@ abstract class ComputeWorkspaceDependenciesTask : DefaultTask() {
         internal fun register(
             rootProject: Project,
             variantBuilderProvider: Lazy<VariantBuilder>,
+            limitDependencyResolutionParallelism: Property<Boolean>,
         ): TaskProvider<ComputeWorkspaceDependenciesTask> {
             val computeTask = rootProject.tasks
                 .register<ComputeWorkspaceDependenciesTask>(TASK_NAME) {
@@ -70,6 +72,7 @@ abstract class ComputeWorkspaceDependenciesTask : DefaultTask() {
             ResolveVariantDependenciesTask.register(
                 rootProject,
                 variantBuilderProvider,
+                limitDependencyResolutionParallelism,
             ) { taskProvider ->
                 computeTask.configure {
                     compileDependenciesJsons.add(taskProvider.flatMap { it.resolvedDependencies })