Update output data grouping for existing supported PURL types #116

- Adjusted data output for bitbucket, cargo, npm, pypi and rubygems types to return metadata (1) for all versions when the input PURL has no version and (2) for just the specified version when the input PURL has a version. Signed-off-by: John M. Horan <[email protected]>
aboutcode-org · Jul 19, 2024 · 5226943 · 5226943
1 parent 1eb1bc3
commit 5226943
Show file tree

Hide file tree

Showing 16 changed files with 2,125 additions and 3,307 deletions.
diff --git a/src/fetchcode/package.py b/src/fetchcode/package.py
@@ -43,8 +43,8 @@
 
 def info(url):
     """
-    Return data according to the `url` string
-    `url` string can be purl too
+    Return package metadata for a URL or PURL.
+    Return None if there is no URL, or the URL or PURL is not supported.
     """
     if url:
         try:
@@ -86,13 +86,7 @@ def get_cargo_data_from_purl(purl):
     crate = response.get("crate") or {}
     homepage_url = crate.get("homepage")
     code_view_url = crate.get("repository")
-    yield Package(
-        homepage_url=homepage_url,
-        api_url=api_url,
-        code_view_url=code_view_url,
-        download_url=download_url,
-        **purl.to_dict(),
-    )
+
     versions = response.get("versions", [])
     for version in versions:
         version_purl = PackageURL(type=purl.type, name=name, version=version.get("num"))
@@ -103,6 +97,9 @@ def get_cargo_data_from_purl(purl):
             download_url = None
         declared_license = version.get("license")
 
+        if purl.version and version_purl.version != purl.version:
+            continue
+
         yield Package(
             homepage_url=homepage_url,
             api_url=api_url,
@@ -112,6 +109,9 @@ def get_cargo_data_from_purl(purl):
             **version_purl.to_dict(),
         )
 
+        if purl.version:
+            break
+
 
 @router.route("pkg:npm/.*")
 def get_npm_data_from_purl(purl):
@@ -123,28 +123,17 @@ def get_npm_data_from_purl(purl):
     name = purl.name
     version = purl.version
     api_url = f"{base_path}/{name}"
+
     response = get_response(api_url)
     vcs_data = response.get("repository") or {}
     bugs = response.get("bugs") or {}
-
     download_url = f"{base_path}/{name}/-/{name}-{version}.tgz" if version else None
     vcs_url = vcs_data.get("url")
     bug_tracking_url = bugs.get("url")
     license = response.get("license")
     homepage_url = response.get("homepage")
 
-    yield Package(
-        homepage_url=homepage_url,
-        api_url=api_url,
-        vcs_url=vcs_url,
-        bug_tracking_url=bug_tracking_url,
-        download_url=download_url,
-        declared_license=license,
-        **purl.to_dict(),
-    )
-
     versions = response.get("versions", [])
-    tags = []
     for num in versions:
         version = versions[num]
         version_purl = PackageURL(
@@ -153,11 +142,13 @@ def get_npm_data_from_purl(purl):
         repository = version.get("repository") or {}
         bugs = response.get("bugs") or {}
         dist = version.get("dist") or {}
-        licenses = version.get("licenses") or [{}]
         vcs_url = repository.get("url")
         download_url = dist.get("tarball")
         bug_tracking_url = bugs.get("url")
-        declared_license = licenses[0].get("type")
+        declared_license = license
+
+        if purl.version and version_purl.version != purl.version:
+            continue
 
         yield Package(
             homepage_url=homepage_url,
@@ -169,6 +160,9 @@ def get_npm_data_from_purl(purl):
             **version_purl.to_dict(),
         )
 
+        if purl.version:
+            break
+
 
 @router.route("pkg:pypi/.*")
 def get_pypi_data_from_purl(purl):
@@ -177,6 +171,7 @@ def get_pypi_data_from_purl(purl):
     """
     purl = PackageURL.from_string(purl)
     name = purl.name
+
     base_path = "https://pypi.org/pypi"
     api_url = f"{base_path}/{name}/json"
     response = get_response(api_url)
@@ -187,19 +182,14 @@ def get_pypi_data_from_purl(purl):
     project_urls = info.get("project_urls") or {}
     code_view_url = get_pypi_codeview_url(project_urls)
     bug_tracking_url = get_pypi_bugtracker_url(project_urls)
-    yield Package(
-        homepage_url=homepage_url,
-        api_url=api_url,
-        bug_tracking_url=bug_tracking_url,
-        code_view_url=code_view_url,
-        declared_license=license,
-        **purl.to_dict(),
-    )
+
     for num in releases:
         version_purl = PackageURL(type=purl.type, name=name, version=num)
         release = releases.get(num) or [{}]
         release = release[0]
         download_url = release.get("url")
+        if purl.version and version_purl.version != purl.version:
+            continue
         yield Package(
             homepage_url=homepage_url,
             api_url=api_url,
@@ -210,6 +200,9 @@ def get_pypi_data_from_purl(purl):
             **version_purl.to_dict(),
         )
 
+        if purl.version:
+            break
+
 
 @router.route("pkg:github/.*")
 def get_github_data_from_purl(purl):
@@ -296,19 +289,15 @@ def get_bitbucket_data_from_purl(purl):
     bitbucket_url = "https://bitbucket.org"
     bug_tracking_url = f"{bitbucket_url}/{namespace}/{name}/issues"
     code_view_url = f"{bitbucket_url}/{namespace}/{name}"
-    yield Package(
-        api_url=api_url,
-        bug_tracking_url=bug_tracking_url,
-        code_view_url=code_view_url,
-        **purl.to_dict(),
-    )
+
     links = response.get("links") or {}
     tags_url = links.get("tags") or {}
     tags_url = tags_url.get("href")
     if not tags_url:
         return []
     tags_data = get_response(tags_url)
     tags = tags_data.get("values") or {}
+
     for tag in tags:
         version = tag.get("name") or ""
         version_purl = PackageURL(
@@ -318,6 +307,10 @@ def get_bitbucket_data_from_purl(purl):
             f"{base_path}/{namespace}/{name}/downloads/{name}-{version}.tar.gz"
         )
         code_view_url = f"{bitbucket_url}/{namespace}/{name}/src/{version}"
+
+        if purl.version and version_purl.version != purl.version:
+            continue
+
         yield Package(
             api_url=api_url,
             bug_tracking_url=bug_tracking_url,
@@ -326,6 +319,9 @@ def get_bitbucket_data_from_purl(purl):
             **version_purl.to_dict(),
         )
 
+        if purl.version:
+            break
+
 
 @router.route("pkg:rubygems/.*")
 def get_rubygems_data_from_purl(purl):
@@ -334,22 +330,38 @@ def get_rubygems_data_from_purl(purl):
     """
     purl = PackageURL.from_string(purl)
     name = purl.name
-    api_url = f"https://rubygems.org/api/v1/gems/{name}.json"
-    response = get_response(api_url)
-    declared_license = response.get("licenses") or None
-    homepage_url = response.get("homepage_uri")
-    code_view_url = response.get("source_code_uri")
-    bug_tracking_url = response.get("bug_tracker_uri")
-    download_url = response.get("gem_uri")
-    yield Package(
-        homepage_url=homepage_url,
-        api_url=api_url,
-        bug_tracking_url=bug_tracking_url,
-        code_view_url=code_view_url,
-        declared_license=declared_license,
-        download_url=download_url,
-        **purl.to_dict(),
-    )
+    all_versions_url = f"https://rubygems.org/api/v1/versions/{name}.json"
+    all_versions = get_response(all_versions_url)
+
+    for vers in all_versions:
+        version_purl = PackageURL(type=purl.type, name=name, version=vers.get("number"))
+
+        if purl.version and version_purl.version != purl.version:
+            continue
+
+        number = vers.get("number")
+        version_api = f"https://rubygems.org/api/v2/rubygems/{name}/versions/{number}.json"
+        version_api_response = get_response(version_api)
+        declared_license = version_api_response.get("licenses") or None
+        homepage_url = version_api_response.get("homepage_uri")
+        code_view_url = version_api_response.get("source_code_uri")
+        bug_tracking_url = version_api_response.get("bug_tracker_uri")
+        download_url = version_api_response.get("gem_uri")
+        repository_homepage_url = version_api_response.get("project_uri")
+
+        yield Package(
+            homepage_url=homepage_url,
+            api_url=version_api,
+            bug_tracking_url=bug_tracking_url,
+            code_view_url=code_view_url,
+            declared_license=declared_license,
+            download_url=download_url,
+            repository_homepage_url=repository_homepage_url,
+            **version_purl.to_dict(),
+        )
+
+        if purl.version:
+            break
 
 
 @router.route("pkg:gnu/.*")
@@ -378,7 +390,8 @@ def get_cocoapods_data_from_purl(purl):
     data_list = get_cocoapod_tags(spec, name)
 
     for tag in data_list:
-        if purl.version and tag != purl.version:
+        version_purl = PackageURL(type=purl.type, name=name, version=tag)
+        if purl.version and version_purl.version != purl.version:
             continue
 
         gh_repo_owner = None
@@ -394,7 +407,7 @@ def get_cocoapods_data_from_purl(purl):
             gh_repo_name = podspec_homepage_split[-1]
 
         tag_pkg = construct_cocoapods_package(
-            purl,
+            version_purl,
             name,
             hashed_path,
             cocoapods_org_url,