model: Introduce a new declared copyrights field to data model

In German law, the author and the copyright holder can be two separate legal entities and therefore also need to be tracked separately. Introduce a new field for declared copyrights that is now the primary source for copyright holder information. Authors are still only used as copyright holders if the `addAuthorsToCopyrights` option is enabled. For now, all package manager implementations set empty declared copyrights. Filling the declared copyrights is left as a future exercise (also see [1]). Currently, the only way to add declared copyrights is via curations. This change resolves #4519. [1]: #5504 (comment) Signed-off-by: Rainer Bieniek <[email protected]> Signed-off-by: Sebastian Schuberth <[email protected]>
oss-review-toolkit · Sep 21, 2022 · f2fd86d · f2fd86d
1 parent 62a4fcc
commit f2fd86d
Show file tree

Hide file tree

Showing 11 changed files with 89 additions and 10 deletions.
diff --git a/model/src/main/kotlin/Package.kt b/model/src/main/kotlin/Package.kt
@@ -62,6 +62,16 @@ data class Package(
  @JsonInclude(JsonInclude.Include.NON_DEFAULT)
  val authors: SortedSet<String> = sortedSetOf(),
 
+ /**
+ * The set of concluded copyright statements for this package. It can be used to override the [detected copyright
+ * statements][CopyrightFinding.statement] (note that there is no such thing as a *declared* copyright statement as
+ * package managers to not support declaring them explicitly).
+ *
+ * ORT itself does not set this field, it needs to be set by the user using a [PackageCuration].
+ */
+ @JsonInclude(JsonInclude.Include.NON_DEFAULT)
+ val concludedCopyrights: SortedSet<String> = sortedSetOf(),
+
  /**
  * The set of licenses declared for this package. This does not necessarily correspond to the licenses as detected
  * by a scanner. Both need to be taken into account for any conclusions.
@@ -138,6 +148,7 @@ data class Package(
  id = Identifier.EMPTY,
  purl = "",
  authors = sortedSetOf(),
+ concludedCopyrights = sortedSetOf(),
  declaredLicenses = sortedSetOf(),
  declaredLicensesProcessed = ProcessedDeclaredLicense.EMPTY,
  concludedLicense = null,

diff --git a/model/src/main/kotlin/PackageCurationData.kt b/model/src/main/kotlin/PackageCurationData.kt
@@ -55,6 +55,12 @@ data class PackageCurationData(
  */
  val authors: SortedSet<String>? = null,
 
+ /**
+ * The set of concluded copyright statements for the package. It can be used to override the [detected copyright
+ * statements][CopyrightFinding.statement].
+ */
+ val concludedCopyrights: SortedSet<String>? = null,
+
  /**
  * The concluded license as an [SpdxExpression]. It can be used to override the [declared][Package.declaredLicenses]
  * / [detected][LicenseFinding.license] licenses of a package.
@@ -131,6 +137,7 @@ data class PackageCurationData(
  purl = purl ?: original.purl,
  cpe = cpe ?: original.cpe,
  authors = authors ?: original.authors,
+ concludedCopyrights = concludedCopyrights ?: original.concludedCopyrights,
  declaredLicenses = original.declaredLicenses,
  declaredLicensesProcessed = declaredLicensesProcessed,
  concludedLicense = concludedLicense ?: original.concludedLicense,
@@ -170,6 +177,7 @@ data class PackageCurationData(
  purl = purl ?: other.purl,
  cpe = cpe ?: other.cpe,
  authors = (authors.orEmpty() + other.authors.orEmpty()).toSortedSet(),
+ concludedCopyrights = (concludedCopyrights.orEmpty() + other.concludedCopyrights.orEmpty()).toSortedSet(),
  concludedLicense = setOfNotNull(concludedLicense, other.concludedLicense).reduce(SpdxExpression::and),
  description = description ?: other.description,
  homepageUrl = homepageUrl ?: other.homepageUrl,

diff --git a/model/src/main/kotlin/licenses/DefaultLicenseInfoProvider.kt b/model/src/main/kotlin/licenses/DefaultLicenseInfoProvider.kt
@@ -53,10 +53,11 @@ class DefaultLicenseInfoProvider(
  private fun createConcludedLicenseInfo(id: Identifier): ConcludedLicenseInfo =
  ortResult.getPackage(id)?.let { (pkg, curations) ->
  ConcludedLicenseInfo(
+ concludedCopyrights = pkg.concludedCopyrights,
  concludedLicense = pkg.concludedLicense,
  appliedCurations = curations.filter { it.curation.concludedLicense != null }
  )
- } ?: ConcludedLicenseInfo(concludedLicense = null, appliedCurations = emptyList())
+ } ?: ConcludedLicenseInfo(concludedCopyrights = null, concludedLicense = null, appliedCurations = emptyList())
 
  private fun createDeclaredLicenseInfo(id: Identifier): DeclaredLicenseInfo =
  ortResult.getProject(id)?.let { project ->

diff --git a/model/src/main/kotlin/licenses/LicenseInfo.kt b/model/src/main/kotlin/licenses/LicenseInfo.kt
@@ -62,6 +62,11 @@ data class LicenseInfo(
  * Information about the concluded license of a package or project.
  */
 data class ConcludedLicenseInfo(
+ /**
+ * The concluded copyright statements, or null if no copyrights were concluded.
+ */
+ val concludedCopyrights: SortedSet<String>?,
+
  /**
  * The concluded license, or null if no license was concluded.
  */

diff --git a/model/src/main/kotlin/licenses/LicenseInfoResolver.kt b/model/src/main/kotlin/licenses/LicenseInfoResolver.kt
@@ -96,6 +96,22 @@ class LicenseInfoResolver(
  originalDeclaredLicenses += licenseInfo.declaredLicenseInfo.processed.mapped.filterValues {
  it == license
  }.keys
+
+ licenseInfo.concludedLicenseInfo.concludedCopyrights?.takeIf { it.isNotEmpty() }?.also {
+ locations += ResolvedLicenseLocation(
+ provenance = UnknownProvenance,
+ location = UNDEFINED_TEXT_LOCATION,
+ appliedCuration = null,
+ matchingPathExcludes = emptyList(),
+ copyrights = it.mapTo(mutableSetOf()) { statement ->
+ ResolvedCopyrightFinding(
+ statement = statement,
+ location = UNDEFINED_TEXT_LOCATION,
+ matchingPathExcludes = emptyList()
+ )
+ }
+ )
+ }
  }
  }
 

diff --git a/model/src/test/kotlin/PackageCurationDataTest.kt b/model/src/test/kotlin/PackageCurationDataTest.kt
@@ -30,6 +30,7 @@ class PackageCurationDataTest : WordSpec({
  purl = "original",
  cpe = "original",
  authors = sortedSetOf("original"),
+ concludedCopyrights = sortedSetOf("original"),
  concludedLicense = "original".toSpdx(),
  description = "original",
  homepageUrl = "original",
@@ -57,6 +58,7 @@ class PackageCurationDataTest : WordSpec({
  purl = "other",
  cpe = "other",
  authors = sortedSetOf("other"),
+ concludedCopyrights = sortedSetOf("other"),
  concludedLicense = "other".toSpdx(),
  description = "other",
  homepageUrl = "other",
@@ -88,6 +90,7 @@ class PackageCurationDataTest : WordSpec({
  val originalWithSomeUnsetData = original.copy(
  comment = null,
  authors = null,
+ concludedCopyrights = null,
  concludedLicense = null,
  binaryArtifact = null,
  vcs = null,
@@ -98,6 +101,7 @@ class PackageCurationDataTest : WordSpec({
  originalWithSomeUnsetData.merge(other) shouldBe originalWithSomeUnsetData.copy(
  comment = other.comment,
  authors = other.authors,
+ concludedCopyrights = other.concludedCopyrights,
  concludedLicense = other.concludedLicense,
  binaryArtifact = other.binaryArtifact,
  vcs = other.vcs,
@@ -110,6 +114,7 @@ class PackageCurationDataTest : WordSpec({
  original.merge(other) shouldBe original.copy(
  comment = "original\nother",
  authors = sortedSetOf("original", "other"),
+ concludedCopyrights = sortedSetOf("original", "other"),
  concludedLicense = "original AND other".toSpdx(),
  declaredLicenseMapping = mapOf(
  "original" to "original".toSpdx(),
@@ -122,6 +127,7 @@ class PackageCurationDataTest : WordSpec({
  val otherWithSomeOriginalData = other.copy(
  comment = original.comment,
  authors = original.authors,
+ concludedCopyrights = original.concludedCopyrights,
  concludedLicense = original.concludedLicense,
  declaredLicenseMapping = original.declaredLicenseMapping
  )

diff --git a/model/src/test/kotlin/PackageCurationTest.kt b/model/src/test/kotlin/PackageCurationTest.kt
@@ -56,7 +56,7 @@ class PackageCurationTest : WordSpec({
  purl = "pkg:maven/org.hamcrest/[email protected]#subpath=src/main/java/org/hamcrest/core",
  cpe = "cpe:2.3:a:apache:commons_io:2.8.0:rc2:*:*:*:*:*:*",
  authors = sortedSetOf("author 1", "author 2"),
- declaredLicenseMapping = mapOf("license a" to "Apache-2.0".toSpdx()),
+ concludedCopyrights = sortedSetOf("copyright 1", "copyright 2"),
  concludedLicense = "license1 OR license2".toSpdx(),
  description = "description",
  homepageUrl = "http://home.page",
@@ -75,7 +75,8 @@ class PackageCurationTest : WordSpec({
  path = "path"
  ),
  isMetaDataOnly = true,
- isModified = true
+ isModified = true,
+ declaredLicenseMapping = mapOf("license a" to "Apache-2.0".toSpdx())
  )
  )
 
@@ -86,6 +87,7 @@ class PackageCurationTest : WordSpec({
  purl shouldBe curation.data.purl
  cpe shouldBe curation.data.cpe
  authors shouldBe curation.data.authors
+ concludedCopyrights shouldBe curation.data.concludedCopyrights
  declaredLicenses shouldBe pkg.declaredLicenses
  declaredLicensesProcessed.spdxExpression shouldBe "Apache-2.0".toSpdx()
  declaredLicensesProcessed.unmapped should containExactlyInAnyOrder("license b")
@@ -147,6 +149,7 @@ class PackageCurationTest : WordSpec({
  purl shouldBe pkg.purl
  cpe shouldBe pkg.cpe
  authors shouldBe pkg.authors
+ concludedCopyrights shouldBe pkg.concludedCopyrights
  declaredLicenses shouldBe pkg.declaredLicenses
  concludedLicense shouldBe pkg.concludedLicense
  description shouldBe pkg.description

diff --git a/model/src/test/kotlin/licenses/LicenseInfoResolverTest.kt b/model/src/test/kotlin/licenses/LicenseInfoResolverTest.kt
@@ -238,6 +238,27 @@ class LicenseInfoResolverTest : WordSpec({
  result should containFindingsForCopyrightExactly("(c) 2010 Holder 2", TextLocation("LICENSE", 3))
  }
 
+ "resolve concluded copyright statements" {
+ val licenseInfos = listOf(
+ createLicenseInfo(
+ id = pkgId,
+ concludedCopyrights = concludedCopyrights,
+ declaredLicenses = declaredLicenses
+ )
+ )
+ val resolver = createResolver(licenseInfos)
+
+ val result = resolver.resolveLicenseInfo(pkgId)
+ result should containCopyrightStatementsForLicenseExactly(
+ "LicenseRef-a",
+ "Copyright (C) 2017 foo", "Copyright (C) 2022 bar"
+ )
+ result should containCopyrightStatementsForLicenseExactly(
+ "LicenseRef-b",
+ "Copyright (C) 2017 foo", "Copyright (C) 2022 bar"
+ )
+ }
+
  "process copyrights by license" {
  val licenseInfos = listOf(
  createLicenseInfo(
@@ -625,6 +646,7 @@ private fun createResolver(
 
 private fun createLicenseInfo(
  id: Identifier,
+ concludedCopyrights: SortedSet<String>? = null,
  declaredLicenses: Set<String> = emptySet(),
  detectedLicenses: List<Findings> = emptyList(),
  concludedLicense: SpdxExpression? = null
@@ -640,6 +662,7 @@ private fun createLicenseInfo(
  findings = detectedLicenses
  ),
  concludedLicenseInfo = ConcludedLicenseInfo(
+ concludedCopyrights = concludedCopyrights,
  concludedLicense = concludedLicense,
  appliedCurations = emptyList()
  )

diff --git a/model/src/test/kotlin/licenses/TestData.kt b/model/src/test/kotlin/licenses/TestData.kt
@@ -53,6 +53,7 @@ import org.ossreviewtoolkit.utils.spdx.toSpdx
 val authors = sortedSetOf("The Author", "The Other Author")
 val projectAuthors = sortedSetOf("The Project Author")
 
+val concludedCopyrights = sortedSetOf("Copyright (C) 2017 foo", "Copyright (C) 2022 bar")
 val concludedLicense = "LicenseRef-a AND LicenseRef-b".toSpdx()
 val declaredLicenses = sortedSetOf("LicenseRef-a", "LicenseRef-b")
 val declaredLicensesProcessed = DeclaredLicenseProcessor.process(declaredLicenses)

diff --git a/scanner/src/main/kotlin/Scanner.kt b/scanner/src/main/kotlin/Scanner.kt
@@ -46,15 +46,18 @@ import org.ossreviewtoolkit.utils.ort.Environment
 const val TOOL_NAME = "scanner"
 
 private fun removeConcludedPackages(packages: Set<Package>, scanner: Scanner): Set<Package> =
- packages.takeUnless { scanner.scannerConfig.skipConcluded }
- // Remove all packages that have a concluded license and authors set.
- ?: packages.partition { it.concludedLicense != null && it.authors.isNotEmpty() }.let { (skip, keep) ->
+ packages.takeUnless { scanner.scannerConfig.skipConcluded } ?: run {
+ // Remove all packages that have a concluded license and concluded copyrights set.
+ packages.partition { it.concludedLicense != null && it.concludedCopyrights.isNotEmpty() }.let { (skip, keep) ->
  if (skip.isNotEmpty()) {
- Scanner.logger.debug { "Not scanning the following packages with concluded licenses: $skip" }
+ Scanner.logger.debug {
+ "Not scanning the following packages with concluded licenses and concluded copyrights: $skip"
+ }
  }
 
  keep.toSet()
  }
+ }
 
 /**
  * Use the [scanner] to scan the [Project]s and [Package]s specified in the [ortResult]. If [skipExcluded] is true,

diff --git a/scanner/src/main/kotlin/experimental/ExperimentalScanner.kt b/scanner/src/main/kotlin/experimental/ExperimentalScanner.kt
@@ -360,16 +360,18 @@ class ExperimentalScanner(
  }
 
  private fun Collection<Package>.filterNotConcluded(): Collection<Package> =
- takeUnless { scannerConfig.skipConcluded }
- ?: partition { it.concludedLicense != null && it.authors.isNotEmpty() }.let { (skip, keep) ->
+ takeUnless { scannerConfig.skipConcluded } ?: run {
+ // Remove all packages that have a concluded license and concluded copyrights set.
+ partition { it.concludedLicense != null && it.concludedCopyrights.isNotEmpty() }.let { (skip, keep) ->
  if (skip.isNotEmpty()) {
  logger.debug {
- "Not scanning the following package(s) with concluded licenses: $skip"
+ "Not scanning the following package(s) with concluded licenses and concluded copyrights: $skip"
  }
  }
 
  keep
  }
+ }
 
  private fun Collection<Package>.filterNotMetaDataOnly(): List<Package> =
  partition { it.isMetaDataOnly }.let { (skip, keep) ->