FossID: Limit fossid snippets

plugins/reporters/fossid/src/main/resources/templates/asciidoc/fossid_snippet.ftl

@@ -31,6 +31,15 @@ List of all the provenances with their files and snippets.
 
 [#if scanResult.scanner.name != "FossId"] [#continue] [/#if]
 
+[#assign snippetsLimitIssue = helper.getSnippetsLimitIssue()]
+
+[#if snippetsLimitIssue??]
+[WARNING]
+====
+${snippetsLimitIssue}
+====
+[/#if]
+
 [#if scanResult.provenance.vcsInfo??]
  [#assign url = scanResult.provenance.vcsInfo.url]
 [#else]

plugins/reporters/freemarker/src/main/kotlin/FreemarkerTemplateProcessor.kt

@@ -198,6 +198,7 @@ class FreemarkerTemplateProcessor(
  /**
  * A collection of helper functions for the Freemarker templates.
  */
+ @Suppress("TooManyFunctions")
  class TemplateHelper(private val input: ReporterInput) {
  /**
  * Return only those [packages] that are a dependency of at least one of the provided [projects][projectIds].
@@ -277,6 +278,18 @@ class FreemarkerTemplateProcessor(
  fun hasUnresolvedIssues(threshold: Severity = input.ortConfig.severeIssueThreshold) =
  input.ortResult.getOpenIssues(minSeverity = threshold).isNotEmpty()
 
+ /**
+ * If there are any issue caused by reaching the snippets limit, return the text of the issue. Otherwise reuturn
+ * 'null'.
+ */
+ @Suppress("UNUSED") // This function is used in the templates.
+ fun getSnippetsLimitIssue() =
+ input.ortResult.scanner?.scanResults?.flatMap { result ->
+ result.summary.issues
+ }?.firstOrNull {
+ it.message.contains("snippets limit")
+ }?.message
+
  /**
  * Return `true` if there are any unresolved and non-excluded [RuleViolation]s whose severity is equal to or
  * greater than the [threshold], or `false` otherwise.

plugins/scanners/fossid/src/main/kotlin/FossId.kt

@@ -31,7 +31,9 @@ import kotlinx.coroutines.Deferred
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.async
 import kotlinx.coroutines.awaitAll
+import kotlinx.coroutines.coroutineScope
 import kotlinx.coroutines.delay
+import kotlinx.coroutines.flow.flow
 import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.withTimeoutOrNull
 
@@ -857,36 +859,42 @@ class FossId internal constructor(
  }
 
  val matchedLines = mutableMapOf<Int, MatchedLines>()
- val snippets = runBlocking(Dispatchers.IO) {
- pendingFiles.map {
- async {
- logger.info { "Listing snippet for $it..." }
- val snippetResponse = service.listSnippets(config.user, config.apiKey, scanCode, it)
- .checkResponse("list snippets")
- val snippets = checkNotNull(snippetResponse.data) {
- "Snippet could not be listed. Response was ${snippetResponse.message}."
- }
- logger.info { "${snippets.size} snippets." }
+ val pendingFilesIterator = pendingFiles.iterator()
+ val snippets = flow {
+ while (pendingFilesIterator.hasNext()) {
+ val file = pendingFilesIterator.next()
+ logger.info { "Listing snippet for $file..." }
+
+ val snippetResponse = service.listSnippets(config.user, config.apiKey, scanCode, file)
+ .checkResponse("list snippets")
+ val snippets = checkNotNull(snippetResponse.data) {
+ "Snippet could not be listed. Response was ${snippetResponse.message}."
+ }
+ logger.info { "${snippets.size} snippets." }
 
-  val filteredSnippets = snippets.filterTo(mutableSetOf()) { it.matchType.isValidType() }
+ val filteredSnippets = snippets.filterTo(mutableSetOf()) { it.matchType.isValidType() }
 
-  if (config.fetchSnippetMatchedLines) {
-  logger.info { "Listing snippet matched lines for $it..." }
+ if (config.fetchSnippetMatchedLines) {
+ logger.info { "Listing snippet matched lines for $file..." }
 
+ coroutineScope {
  filteredSnippets.filter { it.matchType == MatchType.PARTIAL }.map { snippet ->
- val matchedLinesResponse =
- service.listMatchedLines(config.user, config.apiKey, scanCode, it, snippet.id)
- .checkResponse("list snippets matched lines")
- val lines = checkNotNull(matchedLinesResponse.data) {
- "Matched lines could not be listed. Response was ${matchedLinesResponse.message}."
+ async {
+ val matchedLinesResponse =
+ service.listMatchedLines(config.user, config.apiKey, scanCode, file, snippet.id)
+ .checkResponse("list snippets matched lines")
+ val lines = checkNotNull(matchedLinesResponse.data) {
+ "Matched lines could not be listed. Response was " +
+ "${matchedLinesResponse.message}."
+ }
+ matchedLines[snippet.id] = lines
  }
- matchedLines[snippet.id] = lines
- }
+ }.awaitAll()
  }
-
- it to filteredSnippets
  }
- }.awaitAll().toMap()
+
+ emit(file to filteredSnippets.toSet())
+ }
  }
 
  return RawResults(
@@ -903,7 +911,7 @@ class FossId internal constructor(
  * Construct the [ScanSummary] for this FossID scan.
  */
  @Suppress("LongParameterList")
- private fun createResultSummary(
+ private suspend fun createResultSummary(
  startTime: Instant,
  provenance: Provenance,
  rawResults: RawResults,
@@ -920,6 +928,7 @@ class FossId internal constructor(
  val snippetLicenseFindings = mutableSetOf<LicenseFinding>()
  val snippetFindings = mapSnippetFindings(
  rawResults,
+ config.snippetsLimit,
  issues,
  detectedLicenseMapping,
  snippetChoices,

plugins/scanners/fossid/src/main/kotlin/FossIdConfig.kt

@@ -110,46 +110,52 @@ data class FossIdConfig(
  /** Whether matched lines of snippets are to be fetched. */
  val fetchSnippetMatchedLines: Boolean,
 
+ /** A limit on the amount of snippets to fetch. **/
+ val snippetsLimit: Int,
+
  /** Stores the map with FossID-specific configuration options. */
  private val options: Map<String, String>
 ) {
  companion object {
  /** Name of the configuration property for the server URL. */
- private const val SERVER_URL_PROPERTY = "serverUrl"
+ private const val PROP_SERVER_URL = "serverUrl"
 
  /** Name of the configuration property for the username. */
- private const val USER_PROPERTY = "user"
+ private const val PROP_USER = "user"
 
  /** Name of the configuration property for the API key. */
- private const val API_KEY_PROPERTY = "apiKey"
+ private const val PROP_API_KEY = "apiKey"
 
  /** Name of the configuration property controlling whether ORT should wait for FossID results. */
- private const val WAIT_FOR_RESULT_PROPERTY = "waitForResult"
+ private const val PROP_WAIT_FOR_RESULT = "waitForResult"
 
  /** Name of the configuration property defining the naming convention for projects. */
- private const val NAMING_PROJECT_PATTERN_PROPERTY = "namingProjectPattern"
+ private const val PROP_NAMING_PROJECT_PATTERN = "namingProjectPattern"
 
  /** Name of the configuration property defining the naming convention for scans. */
- private const val NAMING_SCAN_PATTERN_PROPERTY = "namingScanPattern"
+ private const val PROP_NAMING_SCAN_PATTERN = "namingScanPattern"
 
  /** Name of the configuration property defining whether to keep failed scans. */
- private const val KEEP_FAILED_SCANS_PROPERTY = "keepFailedScans"
+ private const val PROP_KEEP_FAILED_SCANS = "keepFailedScans"
 
  /** Name of the configuration property controlling whether delta scans are to be created. */
- private const val DELTA_SCAN_PROPERTY = "deltaScans"
+ private const val PROP_DELTA_SCAN = "deltaScans"
 
  /** Name of the configuration property that limits the number of delta scans. */
- private const val DELTA_SCAN_LIMIT_PROPERTY = "deltaScanLimit"
+ private const val PROP_DELTA_SCAN_LIMIT = "deltaScanLimit"
 
- private const val DETECT_LICENSE_DECLARATIONS_PROPERTY = "detectLicenseDeclarations"
+ private const val PROP_DETECT_LICENSE_DECLARATIONS = "detectLicenseDeclarations"
 
- private const val DETECT_COPYRIGHT_STATEMENTS_PROPERTY = "detectCopyrightStatements"
+ private const val PROP_DETECT_COPYRIGHT_STATEMENTS = "detectCopyrightStatements"
 
  /** Name of the configuration property defining the timeout in minutes for communication with FossID. */
- private const val TIMEOUT = "timeout"
+ private const val PROP_TIMEOUT = "timeout"
 
  /** Name of the configuration property controlling whether matched lines of snippets are to be fetched. */
- private const val FETCH_SNIPPET_MATCHED_LINES = "fetchSnippetMatchedLines"
+ private const val PROP_FETCH_SNIPPET_MATCHED_LINES = "fetchSnippetMatchedLines"
+
+ /** Name of the configuration property defining the limit on the amount of snippets to fetch. */
+ private const val PROP_SNIPPETS_LIMIT = "snippetsLimit"
 
  /**
  * The scanner options beginning with this prefix will be used to parameterize project and scan names.
@@ -162,28 +168,35 @@ data class FossIdConfig(
  @JvmStatic
  private val DEFAULT_TIMEOUT = 60
 
+ /**
+ * Default limit on the amount of snippets to fetch.
+ */
+ @JvmStatic
+ private val DEFAULT_SNIPPETS_LIMIT = 500
+
  fun create(options: Options, secrets: Options): FossIdConfig {
  require(options.isNotEmpty()) { "No FossID Scanner configuration found." }
 
- val serverUrl = options[SERVER_URL_PROPERTY]
+ val serverUrl = options[PROP_SERVER_URL]
  ?: throw IllegalArgumentException("No FossID server URL configuration found.")
- val user = secrets[USER_PROPERTY]
+ val user = secrets[PROP_USER]
  ?: throw IllegalArgumentException("No FossID User configuration found.")
- val apiKey = secrets[API_KEY_PROPERTY]
+ val apiKey = secrets[PROP_API_KEY]
  ?: throw IllegalArgumentException("No FossID API Key configuration found.")
 
- val waitForResult = options[WAIT_FOR_RESULT_PROPERTY]?.toBoolean() != false
+ val waitForResult = options[PROP_WAIT_FOR_RESULT]?.toBoolean() != false
 
- val keepFailedScans = options[KEEP_FAILED_SCANS_PROPERTY]?.toBoolean() == true
- val deltaScans = options[DELTA_SCAN_PROPERTY]?.toBoolean() == true
- val deltaScanLimit = options[DELTA_SCAN_LIMIT_PROPERTY]?.toInt() ?: Int.MAX_VALUE
+ val keepFailedScans = options[PROP_KEEP_FAILED_SCANS]?.toBoolean() == true
+ val deltaScans = options[PROP_DELTA_SCAN]?.toBoolean() == true
+ val deltaScanLimit = options[PROP_DELTA_SCAN_LIMIT]?.toInt() ?: Int.MAX_VALUE
 
- val detectLicenseDeclarations = options[DETECT_LICENSE_DECLARATIONS_PROPERTY]?.toBoolean() == true
- val detectCopyrightStatements = options[DETECT_COPYRIGHT_STATEMENTS_PROPERTY]?.toBoolean() == true
+ val detectLicenseDeclarations = options[PROP_DETECT_LICENSE_DECLARATIONS]?.toBoolean() == true
+ val detectCopyrightStatements = options[PROP_DETECT_COPYRIGHT_STATEMENTS]?.toBoolean() == true
 
- val timeout = options[TIMEOUT]?.toInt() ?: DEFAULT_TIMEOUT
+ val timeout = options[PROP_TIMEOUT]?.toInt() ?: DEFAULT_TIMEOUT
 
- val fetchSnippetMatchedLines = options[FETCH_SNIPPET_MATCHED_LINES]?.toBoolean() == true
+ val fetchSnippetMatchedLines = options[PROP_FETCH_SNIPPET_MATCHED_LINES]?.toBoolean() == true
+ val snippetsLimit = options[PROP_SNIPPETS_LIMIT]?.toInt() ?: DEFAULT_SNIPPETS_LIMIT
 
  require(deltaScanLimit > 0) {
  "deltaScanLimit must be > 0, current value is $deltaScanLimit."
@@ -203,7 +216,8 @@ data class FossIdConfig(
  detectCopyrightStatements = detectCopyrightStatements,
  timeout = timeout,
  fetchSnippetMatchedLines = fetchSnippetMatchedLines,
- options = options
+ options = options,
+ snippetsLimit = snippetsLimit
  )
  }
  }
@@ -212,11 +226,11 @@ data class FossIdConfig(
  * Create a [FossIdNamingProvider] helper object based on the configuration stored in this object.
  */
  fun createNamingProvider(): FossIdNamingProvider {
- val namingProjectPattern = options[NAMING_PROJECT_PATTERN_PROPERTY]?.also {
+ val namingProjectPattern = options[PROP_NAMING_PROJECT_PATTERN]?.also {
  logger.info { "Naming pattern for projects is $it." }
  }
 
- val namingScanPattern = options[NAMING_SCAN_PATTERN_PROPERTY]?.also {
+ val namingScanPattern = options[PROP_NAMING_SCAN_PATTERN]?.also {
  logger.info { "Naming pattern for scans is $it." }
  }