feat(helper-cli): Allow generating an analyzer result from package list

The command defines a data model called `PackageList` which is read as input using any of the known file extensions. It outputs an analyzer result with a single project containing the provided dependencies as a flat list. When a project does not use a package manager, currently the only way of scanning it with ORT is to craft SPDX files and running the analyzer against them. This command tries to provide a minimalistic alternative to fulfill the need to scan any project which does not use any package manager (supported by ORT). The package metadata holds attributes which are usually relevant for license clearance, namely the excluded state and the type of linkage. So, it can be useful to do license clearance for arbitrary sets of packages. This is for now limited to detected license clearance, as providing other attributes like e.g. the declared licenses is not (yet) implemented. Signed-off-by: Frank Viernau <[email protected]>
oss-review-toolkit · Sep 12, 2023 · 65992e0 · 65992e0
1 parent 72a5644
commit 65992e0
Show file tree

Hide file tree

Showing 5 changed files with 388 additions and 0 deletions.
diff --git a/helper-cli/src/funTest/assets/create-analyzer-result-from-pkg-list-expected-output.yml b/helper-cli/src/funTest/assets/create-analyzer-result-from-pkg-list-expected-output.yml
@@ -0,0 +1,113 @@
+---
+repository:
+  vcs:
+    type: "Git"
+    url: "https://github.com/example/project.git"
+    revision: "2222222222222222222222222222222222222222"
+    path: "vcs-path/project"
+  vcs_processed:
+    type: "Git"
+    url: "https://github.com/example/project.git"
+    revision: "2222222222222222222222222222222222222222"
+    path: "vcs-path/project"
+  config:
+    excludes:
+      scopes:
+      - pattern: "excluded"
+        reason: "DEV_DEPENDENCY_OF"
+analyzer:
+  start_time: "1970-01-01T00:00:00Z"
+  end_time: "1970-01-01T00:00:00Z"
+  environment:
+    ort_version: "1.1.0-SNAPSHOT"
+    java_version: "17.0.8.1"
+    os: "Linux"
+    processors: 8
+    max_memory: 536870912
+    variables: {}
+    tool_versions: {}
+  config:
+    allow_dynamic_versions: false
+    skip_excluded: false
+  result:
+    projects:
+    - id: "Unmanaged::Example project name:"
+      definition_file_path: ""
+      declared_licenses: []
+      declared_licenses_processed: {}
+      vcs:
+        type: "Git"
+        url: "https://github.com/example/project.git"
+        revision: "2222222222222222222222222222222222222222"
+        path: "vcs-path/project"
+      vcs_processed:
+        type: "Git"
+        url: "https://github.com/example/project.git"
+        revision: "2222222222222222222222222222222222222222"
+        path: "vcs-path/project"
+      homepage_url: ""
+      scopes:
+      - name: "excluded"
+        dependencies:
+        - id: "NPM::example-dependency-one:1.0.0"
+      - name: "main"
+        dependencies:
+        - id: "NPM::example-dependency-two:2.0.0"
+          linkage: "STATIC"
+    packages:
+    - id: "NPM::example-dependency-one:1.0.0"
+      purl: ""
+      declared_licenses: []
+      declared_licenses_processed: {}
+      description: ""
+      homepage_url: ""
+      binary_artifact:
+        url: ""
+        hash:
+          value: ""
+          algorithm: ""
+      source_artifact:
+        url: "https://example.org/example-dependency-one.zip"
+        hash:
+          value: ""
+          algorithm: ""
+      vcs:
+        type: "Git"
+        url: "https://github.com/example/depedency-one.git"
+        revision: "0000000000000000000000000000000000000000"
+        path: "vcs-path/dependency-one"
+      vcs_processed:
+        type: "Git"
+        url: "https://github.com/example/depedency-one.git"
+        revision: "0000000000000000000000000000000000000000"
+        path: "vcs-path/dependency-one"
+    - id: "NPM::example-dependency-two:2.0.0"
+      purl: ""
+      declared_licenses: []
+      declared_licenses_processed: {}
+      description: ""
+      homepage_url: ""
+      binary_artifact:
+        url: ""
+        hash:
+          value: ""
+          algorithm: ""
+      source_artifact:
+        url: "https://example.org/example-dependency-two.zip"
+        hash:
+          value: ""
+          algorithm: ""
+      vcs:
+        type: "Git"
+        url: "https://github.com/example/depedency-1.git"
+        revision: "1111111111111111111111111111111111111111"
+        path: "vcs-path/dependency-two"
+      vcs_processed:
+        type: "Git"
+        url: "https://github.com/example/depedency-1.git"
+        revision: "1111111111111111111111111111111111111111"
+        path: "vcs-path/dependency-two"
+scanner: null
+advisor: null
+evaluator: null
+resolved_configuration: {}
diff --git a/helper-cli/src/funTest/assets/package-list.yml b/helper-cli/src/funTest/assets/package-list.yml
@@ -0,0 +1,28 @@
+---
+projectName: "Example project name"
+projectVcs:
+  type: "Git"
+  url: "https://github.com/example/project.git"
+  revision: "2222222222222222222222222222222222222222"
+  path: "vcs-path/project"
+dependencies:
+  - id: "NPM::example-dependency-one:1.0.0"
+    vcs:
+      type: "Git"
+      url: "https://github.com/example/depedency-one.git"
+      revision: "0000000000000000000000000000000000000000"
+      path: "vcs-path/dependency-one"
+    sourceArtifact:
+      url: "https://example.org/example-dependency-one.zip"
+    isExcluded: true
+    isDynamicallyLinked: true
+  - id: "NPM::example-dependency-two:2.0.0"
+    vcs:
+      type: "Git"
+      url: "https://github.com/example/depedency-1.git"
+      revision: "1111111111111111111111111111111111111111"
+      path: "vcs-path/dependency-two"
+    sourceArtifact:
+      url: "https://example.org/example-dependency-two.zip"
+    isExcluded: false
+    isDynamicallyLinked: false
diff --git a/helper-cli/src/funTest/kotlin/commands/CreateAnalyzerResultFromPackageListCommandFunTest.kt b/helper-cli/src/funTest/kotlin/commands/CreateAnalyzerResultFromPackageListCommandFunTest.kt
@@ -0,0 +1,64 @@
+/*
+ * Copyright (C) 2023 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+package org.ossreviewtoolkit.helper.commands
+
+import com.github.ajalt.clikt.core.ProgramResult
+
+import io.kotest.core.spec.style.WordSpec
+import io.kotest.matchers.shouldBe
+
+import org.ossreviewtoolkit.helper.HelperMain
+import org.ossreviewtoolkit.model.OrtResult
+import org.ossreviewtoolkit.model.readValue
+import org.ossreviewtoolkit.model.toYaml
+import org.ossreviewtoolkit.utils.ort.Environment
+import org.ossreviewtoolkit.utils.ort.createOrtTempDir
+import org.ossreviewtoolkit.utils.test.getAssetFile
+
+class CreateAnalyzerResultFromPackageListCommandFunTest : WordSpec({
+    "The command" should {
+        "generate the expected analyzer result file" {
+            val inputFile = getAssetFile("package-list.yml")
+            val outputFile = createOrtTempDir().resolve("analyzer-result.yml")
+            val expectedOutputFile = getAssetFile("create-analyzer-result-from-pkg-list-expected-output.yml")
+
+            runMain(
+                "create-analyzer-result-from-package-list",
+                "--package-list-file",
+                inputFile.absolutePath,
+                "--ort-file",
+                outputFile.absolutePath
+            )
+
+            outputFile.readText() shouldBe expectedOutputFile.readValue<OrtResult>().patchEnvironment().toYaml()
+        }
+    }
+})
+
+private fun runMain(vararg args: String) {
+    @Suppress("SwallowedException")
+    try {
+        HelperMain().parse(args.asList())
+    } catch (e: ProgramResult) {
+        // Ignore exceptions that just propagate the program result.
+    }
+}
+
+private fun OrtResult.patchEnvironment(): OrtResult = copy(analyzer = analyzer?.copy(environment = Environment()))
diff --git a/helper-cli/src/main/kotlin/HelperMain.kt b/helper-cli/src/main/kotlin/HelperMain.kt
@@ -79,6 +79,7 @@ internal class HelperMain : CliktCommand(
         subcommands(
             ConvertOrtFileCommand(),
             CreateAnalyzerResultCommand(),
+            CreateAnalyzerResultFromPackageListCommand(),
             DevCommand(),
             ExtractRepositoryConfigurationCommand(),
             GenerateTimeoutErrorResolutionsCommand(),

diff --git a/helper-cli/src/main/kotlin/commands/CreateAnalyzerResultFromPackageListCommand.kt b/helper-cli/src/main/kotlin/commands/CreateAnalyzerResultFromPackageListCommand.kt
@@ -0,0 +1,182 @@
+/*
+ * Copyright (C) 2023 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+package org.ossreviewtoolkit.helper.commands
+
+import com.fasterxml.jackson.databind.PropertyNamingStrategies
+import com.fasterxml.jackson.module.kotlin.readValue
+
+import com.github.ajalt.clikt.core.CliktCommand
+import com.github.ajalt.clikt.parameters.options.convert
+import com.github.ajalt.clikt.parameters.options.option
+import com.github.ajalt.clikt.parameters.options.required
+import com.github.ajalt.clikt.parameters.types.file
+
+import org.ossreviewtoolkit.helper.utils.writeOrtResult
+import org.ossreviewtoolkit.model.AnalyzerResult
+import org.ossreviewtoolkit.model.AnalyzerRun
+import org.ossreviewtoolkit.model.Hash
+import org.ossreviewtoolkit.model.Identifier
+import org.ossreviewtoolkit.model.OrtResult
+import org.ossreviewtoolkit.model.Package
+import org.ossreviewtoolkit.model.PackageLinkage
+import org.ossreviewtoolkit.model.PackageReference
+import org.ossreviewtoolkit.model.Project
+import org.ossreviewtoolkit.model.RemoteArtifact
+import org.ossreviewtoolkit.model.Repository
+import org.ossreviewtoolkit.model.Scope
+import org.ossreviewtoolkit.model.VcsInfo
+import org.ossreviewtoolkit.model.VcsType
+import org.ossreviewtoolkit.model.config.Excludes
+import org.ossreviewtoolkit.model.config.RepositoryConfiguration
+import org.ossreviewtoolkit.model.config.ScopeExclude
+import org.ossreviewtoolkit.model.config.ScopeExcludeReason
+import org.ossreviewtoolkit.model.mapper
+import org.ossreviewtoolkit.model.orEmpty
+import org.ossreviewtoolkit.utils.common.expandTilde
+import org.ossreviewtoolkit.utils.ort.Environment
+
+internal class CreateAnalyzerResultFromPackageListCommand : CliktCommand(
+    "A command which turns a package list file into an analyzer result."
+) {
+    private val packageListFile by option(
+        "--package-list-file", "-i",
+        help = "The package list file to read the packages metadata and the project metadata from."
+    ).convert { it.expandTilde() }
+        .file(mustExist = false, canBeFile = true, canBeDir = false, mustBeWritable = false, mustBeReadable = false)
+        .convert { it.absoluteFile.normalize() }
+        .required()
+
+    private val ortFile by option(
+        "--ort-file", "-o",
+        help = "The ORT file to write the generated analyzer result to."
+    ).convert { it.expandTilde() }
+        .file(mustExist = false, canBeFile = true, canBeDir = false, mustBeWritable = false, mustBeReadable = false)
+        .convert { it.absoluteFile.normalize() }
+        .required()
+
+    override fun run() {
+        val packageList = packageListFile.mapper().copy().apply {
+            // Use camel case already now (even if in all places snake case is used), because there is a plan
+            // to migrate from snake case to camel case in context of
+            // https://github.com/oss-review-toolkit/ort/issues/3904.
+            propertyNamingStrategy = PropertyNamingStrategies.LOWER_CAMEL_CASE
+        }.readValue<PackageList>(packageListFile)
+
+        val projectName = packageList.projectName?.takeUnless { it.isBlank() } ?: DEFAULT_PROJECT_NAME
+        val projectVcs = packageList.projectVcs.toVcsInfo()
+
+        val project = Project.EMPTY.copy(
+            id = Identifier("$PROJECT_TYPE::$projectName:"),
+            vcs = projectVcs,
+            vcsProcessed = projectVcs.normalize(),
+            scopeDependencies = setOfNotNull(
+                packageList.dependencies.filterNot { it.isExcluded }.toScope(MAIN_SCOPE_NAME),
+                packageList.dependencies.filter { it.isExcluded }.toScope(EXCLUDED_SCOPE_NAME)
+            )
+        )
+
+        val ortResult = OrtResult(
+            analyzer = AnalyzerRun.EMPTY.copy(
+                result = AnalyzerResult(
+                    projects = setOf(project),
+                    packages = packageList.dependencies.mapTo(mutableSetOf()) { it.toPackage() }
+                ),
+                environment = Environment()
+            ),
+            repository = Repository(
+                vcs = projectVcs.normalize(),
+                config = RepositoryConfiguration(
+                    excludes = Excludes(
+                        scopes = listOf(
+                            ScopeExclude(EXCLUDED_SCOPE_NAME, ScopeExcludeReason.DEV_DEPENDENCY_OF)
+                        )
+                    )
+                )
+            )
+        )
+
+        writeOrtResult(ortResult, ortFile)
+    }
+}
+
+private const val DEFAULT_PROJECT_NAME = "unknown"
+private const val EXCLUDED_SCOPE_NAME = "excluded"
+private const val MAIN_SCOPE_NAME = "main"
+private const val PROJECT_TYPE = "Unmanaged" // This refers to the package manager (plugin) named "Unmanaged".
+
+private data class PackageList(
+    val projectName: String? = null,
+    val projectVcs: Vcs? = null,
+    val dependencies: List<Dependency> = emptyList()
+)
+
+private data class Dependency(
+    val id: Identifier,
+    val vcs: Vcs? = null,
+    val sourceArtifact: SourceArtifact? = null,
+    val isExcluded: Boolean = false,
+    val isDynamicallyLinked: Boolean = false
+)
+
+private data class SourceArtifact(
+    val url: String,
+    val hash: Hash? = null
+)
+
+private data class Vcs(
+    val type: String? = null,
+    val url: String? = null,
+    val revision: String? = null,
+    val path: String? = null
+)
+
+private fun Vcs?.toVcsInfo(): VcsInfo =
+    if (this == null) {
+        VcsInfo.EMPTY
+    } else {
+        VcsInfo(
+            type = type?.let { VcsType.forName(it) } ?: VcsType.UNKNOWN,
+            url = url.orEmpty(),
+            revision = revision.orEmpty(),
+            path = path.orEmpty()
+        )
+    }
+
+private fun Collection<Dependency>.toScope(name: String): Scope =
+    Scope(
+        name = name,
+        dependencies = mapTo(mutableSetOf()) { dependency ->
+            PackageReference(
+                id = dependency.id,
+                linkage = PackageLinkage.STATIC.takeUnless { dependency.isDynamicallyLinked } ?: PackageLinkage.DYNAMIC
+            )
+        }
+    )
+
+private fun Dependency.toPackage(): Package {
+    val vcsInfo = vcs.toVcsInfo()
+
+    return Package.EMPTY.copy(
+        id = id,
+        sourceArtifact = sourceArtifact?.let { RemoteArtifact(url = it.url, it.hash ?: Hash.NONE) }.orEmpty(),
+        vcs = vcsInfo,
+        vcsProcessed = vcsInfo.normalize()
+    )
+}