Merge branch 'main' into add-almalinux-advisories

Author: ambuj-1211

.github/workflows/docs.yml

@@ -4,7 +4,7 @@ on: [push, pull_request]
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     strategy:
       max-parallel: 4

.github/workflows/main.yml

@@ -9,7 +9,7 @@ env:
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     services:
       postgres:

.github/workflows/pypi-release.yml

@@ -21,7 +21,7 @@ on:
 jobs:
   build-pypi-distribs:
     name: Build and publish library to PyPI
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     steps:
       - uses: actions/checkout@master

docs/source/conf.py

@@ -36,6 +36,8 @@
     "https://www.softwaretestinghelp.com/how-to-write-good-bug-report/",  # Cloudflare protection
     "https://www.openssl.org/news/vulnerabilities.xml",  # OpenSSL legacy advisory URL, not longer available
     "https://example.org/api/non-existent-packages",
+    "https://github.com/aboutcode-org/vulnerablecode/pull/495/commits",
+    "https://nvd.nist.gov/products/cpe",
 ]
 
 # Add any Sphinx extension module names here, as strings. They can be

requirements.txt

@@ -27,7 +27,7 @@ dateparser==1.1.1
 decorator==5.1.1
 defusedxml==0.7.1
 distro==1.7.0
-Django==4.2.17
+Django==4.2.20
 django-crispy-forms==2.3
 django-environ==0.11.2
 django-filter==24.3

vulnerabilities/import_runner.py

@@ -104,24 +104,30 @@ def process_advisories(
         advisories = []
         for data in advisory_datas:
             content_id = compute_content_id(advisory_data=data)
+            advisory = {
+                "summary": data.summary,
+                "affected_packages": [pkg.to_dict() for pkg in data.affected_packages],
+                "references": [ref.to_dict() for ref in data.references],
+                "date_published": data.date_published,
+                "weaknesses": data.weaknesses,
+                "created_by": importer_name,
+                "date_collected": datetime.datetime.now(tz=datetime.timezone.utc),
+            }
             try:
                 aliases = get_or_create_aliases(aliases=data.aliases)
                 obj, created = Advisory.objects.get_or_create(
                     unique_content_id=content_id,
                     url=data.url,
-                    defaults={
-                        "summary": data.summary,
-                        "affected_packages": [pkg.to_dict() for pkg in data.affected_packages],
-                        "references": [ref.to_dict() for ref in data.references],
-                        "date_published": data.date_published,
-                        "weaknesses": data.weaknesses,
-                        "created_by": importer_name,
-                        "date_collected": datetime.datetime.now(tz=datetime.timezone.utc),
-                    },
+                    defaults=advisory,
                 )
                 obj.aliases.add(*aliases)
                 if not obj.date_imported:
                     advisories.append(obj)
+            except Advisory.MultipleObjectsReturned:
+                logger.error(
+                    f"Multiple Advisories returned: unique_content_id: {content_id}, url: {data.url}, advisory: {advisory!r}"
+                )
+                raise
             except Exception as e:
                 logger.error(
                     f"Error while processing {data!r} with aliases {data.aliases!r}: {e!r} \n {traceback_format_exc()}"

vulnerabilities/importers/__init__.py

@@ -45,10 +45,25 @@
 from vulnerabilities.pipelines import pysec_importer
 
 IMPORTERS_REGISTRY = [
+    nvd_importer.NVDImporterPipeline,
+    github_importer.GitHubAPIImporterPipeline,
+    gitlab_importer.GitLabImporterPipeline,
+    github_osv.GithubOSVImporter,
+    pypa_importer.PyPaImporterPipeline,
+    npm_importer.NpmImporterPipeline,
+    nginx_importer.NginxImporterPipeline,
+    pysec_importer.PyPIImporterPipeline,
+    apache_tomcat.ApacheTomcatImporter,
+    postgresql.PostgreSQLImporter,
+    debian.DebianImporter,
+    curl.CurlImporter,
+    epss.EPSSImporter,
+    vulnrichment.VulnrichImporter,
+    alpine_linux_importer.AlpineLinuxImporterPipeline,
+    ruby.RubyImporter,
+    apache_kafka.ApacheKafkaImporter,
     openssl.OpensslImporter,
     redhat.RedhatImporter,
-    debian.DebianImporter,
-    postgresql.PostgreSQLImporter,
     archlinux.ArchlinuxImporter,
     ubuntu.UbuntuImporter,
     debian_oval.DebianOvalImporter,
@@ -60,11 +75,9 @@
     project_kb_msr2019.ProjectKBMSRImporter,
     suse_scores.SUSESeverityScoreImporter,
     elixir_security.ElixirSecurityImporter,
-    apache_tomcat.ApacheTomcatImporter,
     xen.XenImporter,
     ubuntu_usn.UbuntuUSNImporter,
     fireeye.FireyeImporter,
-    apache_kafka.ApacheKafkaImporter,
     oss_fuzz.OSSFuzzImporter,
     ruby.RubyImporter,
     github_osv.GithubOSVImporter,

vulnerabilities/importers/osv.py

@@ -224,7 +224,14 @@ def get_affected_purl(affected_pkg, raw_id):
                 f"No PackageURL possible: {purl!r} for affected_pkg {affected_pkg} for OSV id: {raw_id}"
             )
             return
-    return PackageURL.from_string(str(purl))
+    try:
+        package_url = PackageURL.from_string(str(purl))
+        return package_url
+    except:
+        logger.error(
+            f"Invalid PackageURL: {purl!r} for affected_pkg {affected_pkg} for OSV id: {raw_id}"
+        )
+        return None
 
 
 def get_affected_version_range(affected_pkg, raw_id, supported_ecosystem):

vulnerabilities/improvers/__init__.py

@@ -18,6 +18,7 @@
 from vulnerabilities.pipelines import enhance_with_kev
 from vulnerabilities.pipelines import enhance_with_metasploit
 from vulnerabilities.pipelines import flag_ghost_packages
+from vulnerabilities.pipelines import populate_vulnerability_summary_pipeline
 from vulnerabilities.pipelines import remove_duplicate_advisories
 
 IMPROVERS_REGISTRY = [
@@ -48,6 +49,7 @@
     collect_commits.CollectFixCommitsPipeline,
     add_cvss31_to_CVEs.CVEAdvisoryMappingPipeline,
     remove_duplicate_advisories.RemoveDuplicateAdvisoriesPipeline,
+    populate_vulnerability_summary_pipeline.PopulateVulnerabilitySummariesPipeline,
 ]
 
 IMPROVERS_REGISTRY = {

vulnerabilities/management/commands/export.py

@@ -6,11 +6,16 @@
 # See https://github.com/aboutcode-org/vulnerablecode for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
+import itertools
 import logging
 from itertools import groupby
 from pathlib import Path
+from timeit import default_timer as timer
+from traceback import format_exc as traceback_format_exc
 
 import saneyaml
+from aboutcode.pipeline import LoopProgress
+from aboutcode.pipeline import humanize_time
 from django.core.management.base import BaseCommand
 from django.core.management.base import CommandError
 from packageurl import PackageURL
@@ -26,7 +31,7 @@ def serialize_severity(sev):
         "score": sev.value,
         "scoring_system": sev.scoring_system,
         "scoring_elements": sev.scoring_elements,
-        "published_at": sev.published_at,
+        "published_at": str(sev.published_at),
         "url": sev.url,
     }
 
@@ -88,8 +93,22 @@ def export_data(self, base_path: Path):
         """
         i = 0
         seen_vcid = set()
+        export_start_time = timer()
 
-        for i, (purl_without_version, package_versions) in enumerate(packages_by_type_ns_name(), 1):
+        distinct_packages_count = (
+            Package.objects.values("type", "namespace", "name")
+            .distinct("type", "namespace", "name")
+            .count()
+        )
+
+        progress = LoopProgress(
+            total_iterations=distinct_packages_count,
+            progress_step=1,
+            logger=self.stdout.write,
+        )
+        for i, (purl_without_version, package_versions) in enumerate(
+            progress.iter(packages_by_type_ns_name()), 1
+        ):
             pkg_version = None
             try:
                 package_urls = []
@@ -108,7 +127,11 @@ def export_data(self, base_path: Path):
                     }
                     package_vulnerabilities.append(package_data)
 
-                    for vuln in pkg_version.vulnerabilities:
+                    vulnerabilities = itertools.chain(
+                        pkg_version.affected_by_vulnerabilities.all(),
+                        pkg_version.fixing_vulnerabilities.all(),
+                    )
+                    for vuln in vulnerabilities:
                         vcid = vuln.vulnerability_id
                         # do not write twice the same file
                         if vcid in seen_vcid:
@@ -131,9 +154,15 @@ def export_data(self, base_path: Path):
                     self.stdout.write(f"Processed {i} package. Last PURL: {purl_without_version}")
 
             except Exception as e:
-                raise Exception(f"Failed to process Package: {pkg_version}") from e
+                self.stdout.write(
+                    self.style.ERROR(
+                        f"Failed to process Package {pkg_version}: {e!r} \n {traceback_format_exc()}"
+                    )
+                )
 
         self.stdout.write(f"Exported data for: {i} package and {len(seen_vcid)} vulnerabilities.")
+        export_run_time = timer() - export_start_time
+        self.stdout.write(f"Export completed in {humanize_time(export_run_time)}")
 
 
 def by_purl_type_ns_name(package):
@@ -159,7 +188,7 @@ def packages_by_type_ns_name():
             "fixing_vulnerabilities__weaknesses",
             "fixing_vulnerabilities__severities",
         )
-        .paginated()
+        .iterator()
     )
 
     for tp_ns_name, packages in groupby(qs, key=by_purl_type_ns_name):