Fix Gentoo importer v1

Update the Gentoo get_safe_and_affected_versions function in advisory v2

Signed-off-by: ziad hany <ziadhany2016@gmail.com>

Author: ziadhany

vulnerabilities/importers/__init__.py

@@ -49,8 +49,8 @@
     elixir_security_importer as elixir_security_importer_v2,
 )
 from vulnerabilities.pipelines.v2_importers import epss_importer_v2
-from vulnerabilities.pipelines.v2_importers import gentoo_importer as gentoo_importer_v2
 from vulnerabilities.pipelines.v2_importers import fireeye_importer_v2
+from vulnerabilities.pipelines.v2_importers import gentoo_importer as gentoo_importer_v2
 from vulnerabilities.pipelines.v2_importers import github_osv_importer as github_osv_importer_v2
 from vulnerabilities.pipelines.v2_importers import gitlab_importer as gitlab_importer_v2
 from vulnerabilities.pipelines.v2_importers import istio_importer as istio_importer_v2
@@ -70,62 +70,62 @@
 
 IMPORTERS_REGISTRY = create_registry(
     [
-        archlinux_importer_v2.ArchLinuxImporterPipeline,
-        nvd_importer_v2.NVDImporterPipeline,
-        elixir_security_importer_v2.ElixirSecurityImporterPipeline,
-        npm_importer_v2.NpmImporterPipeline,
-        vulnrichment_importer_v2.VulnrichImporterPipeline,
-        apache_httpd_v2.ApacheHTTPDImporterPipeline,
-        pypa_importer_v2.PyPaImporterPipeline,
-        gitlab_importer_v2.GitLabImporterPipeline,
-        pysec_importer_v2.PyPIImporterPipeline,
-        xen_importer_v2.XenImporterPipeline,
-        curl_importer_v2.CurlImporterPipeline,
-        oss_fuzz_v2.OSSFuzzImporterPipeline,
-        istio_importer_v2.IstioImporterPipeline,
-        postgresql_importer_v2.PostgreSQLImporterPipeline,
-        mozilla_importer_v2.MozillaImporterPipeline,
-        github_osv_importer_v2.GithubOSVImporterPipeline,
-        redhat_importer_v2.RedHatImporterPipeline,
-        aosp_importer_v2.AospImporterPipeline,
-        ruby_importer_v2.RubyImporterPipeline,
-        epss_importer_v2.EPSSImporterPipeline,
+        # archlinux_importer_v2.ArchLinuxImporterPipeline,
+        # nvd_importer_v2.NVDImporterPipeline,
+        # elixir_security_importer_v2.ElixirSecurityImporterPipeline,
+        # npm_importer_v2.NpmImporterPipeline,
+        # vulnrichment_importer_v2.VulnrichImporterPipeline,
+        # apache_httpd_v2.ApacheHTTPDImporterPipeline,
+        # pypa_importer_v2.PyPaImporterPipeline,
+        # gitlab_importer_v2.GitLabImporterPipeline,
+        # pysec_importer_v2.PyPIImporterPipeline,
+        # xen_importer_v2.XenImporterPipeline,
+        # curl_importer_v2.CurlImporterPipeline,
+        # oss_fuzz_v2.OSSFuzzImporterPipeline,
+        # istio_importer_v2.IstioImporterPipeline,
+        # postgresql_importer_v2.PostgreSQLImporterPipeline,
+        # mozilla_importer_v2.MozillaImporterPipeline,
+        # github_osv_importer_v2.GithubOSVImporterPipeline,
+        # redhat_importer_v2.RedHatImporterPipeline,
+        # aosp_importer_v2.AospImporterPipeline,
+        # ruby_importer_v2.RubyImporterPipeline,
+        # epss_importer_v2.EPSSImporterPipeline,
         gentoo_importer_v2.GentooImporterPipeline,
-        mattermost_importer_v2.MattermostImporterPipeline,
-        nvd_importer.NVDImporterPipeline,
-        github_importer.GitHubAPIImporterPipeline,
-        gitlab_importer.GitLabImporterPipeline,
-        github_osv.GithubOSVImporter,
-        pypa_importer.PyPaImporterPipeline,
-        npm_importer.NpmImporterPipeline,
-        nginx_importer.NginxImporterPipeline,
-        pysec_importer.PyPIImporterPipeline,
-        fireeye_importer_v2.FireeyeImporterPipeline,
-        apache_tomcat.ApacheTomcatImporter,
-        postgresql.PostgreSQLImporter,
-        debian.DebianImporter,
-        curl.CurlImporter,
-        epss.EPSSImporter,
-        vulnrichment.VulnrichImporter,
-        alpine_linux_importer.AlpineLinuxImporterPipeline,
-        ruby.RubyImporter,
-        apache_kafka.ApacheKafkaImporter,
-        openssl.OpensslImporter,
-        redhat.RedhatImporter,
-        archlinux.ArchlinuxImporter,
-        ubuntu.UbuntuImporter,
-        debian_oval.DebianOvalImporter,
-        retiredotnet.RetireDotnetImporter,
-        apache_httpd.ApacheHTTPDImporter,
-        mozilla.MozillaImporter,
-        gentoo.GentooImporter,
-        istio.IstioImporter,
-        project_kb_msr2019.ProjectKBMSRImporter,
-        suse_scores.SUSESeverityScoreImporter,
-        elixir_security.ElixirSecurityImporter,
-        xen.XenImporter,
-        ubuntu_usn.UbuntuUSNImporter,
-        fireeye.FireyeImporter,
-        oss_fuzz.OSSFuzzImporter,
+        # mattermost_importer_v2.MattermostImporterPipeline,
+        # nvd_importer.NVDImporterPipeline,
+        # github_importer.GitHubAPIImporterPipeline,
+        # gitlab_importer.GitLabImporterPipeline,
+        # github_osv.GithubOSVImporter,
+        # pypa_importer.PyPaImporterPipeline,
+        # npm_importer.NpmImporterPipeline,
+        # nginx_importer.NginxImporterPipeline,
+        # pysec_importer.PyPIImporterPipeline,
+        # fireeye_importer_v2.FireeyeImporterPipeline,
+        # apache_tomcat.ApacheTomcatImporter,
+        # postgresql.PostgreSQLImporter,
+        # debian.DebianImporter,
+        # curl.CurlImporter,
+        # epss.EPSSImporter,
+        # vulnrichment.VulnrichImporter,
+        # alpine_linux_importer.AlpineLinuxImporterPipeline,
+        # ruby.RubyImporter,
+        # apache_kafka.ApacheKafkaImporter,
+        # openssl.OpensslImporter,
+        # redhat.RedhatImporter,
+        # archlinux.ArchlinuxImporter,
+        # ubuntu.UbuntuImporter,
+        # debian_oval.DebianOvalImporter,
+        # retiredotnet.RetireDotnetImporter,
+        # apache_httpd.ApacheHTTPDImporter,
+        # mozilla.MozillaImporter,
+        # gentoo.GentooImporter,
+        # istio.IstioImporter,
+        # project_kb_msr2019.ProjectKBMSRImporter,
+        # suse_scores.SUSESeverityScoreImporter,
+        # elixir_security.ElixirSecurityImporter,
+        # xen.XenImporter,
+        # ubuntu_usn.UbuntuUSNImporter,
+        # fireeye.FireyeImporter,
+        # oss_fuzz.OSSFuzzImporter,
     ]
 )

vulnerabilities/importers/gentoo.py

@@ -6,8 +6,7 @@
 # See https://github.com/aboutcode-org/vulnerablecode for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
-
-
+import logging
 import re
 import xml.etree.ElementTree as ET
 from pathlib import Path
@@ -17,12 +16,15 @@
 from univers.version_constraint import VersionConstraint
 from univers.version_range import EbuildVersionRange
 from univers.versions import GentooVersion
+from univers.versions import InvalidVersion
 
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import AffectedPackage
 from vulnerabilities.importer import Importer
 from vulnerabilities.importer import Reference
 
+logger = logging.getLogger(__name__)
+
 
 class GentooImporter(Importer):
     repo_url = "git+https://anongit.gentoo.org/git/data/glsa.git"
@@ -104,14 +106,20 @@ def affected_and_safe_purls(affected_elem):
             safe_versions, affected_versions = GentooImporter.get_safe_and_affected_versions(pkg)
 
             for version in safe_versions:
-                constraints.append(
-                    VersionConstraint(version=GentooVersion(version), comparator="=").invert()
-                )
+                try:
+                    constraints.append(
+                        VersionConstraint(version=GentooVersion(version), comparator="=").invert()
+                    )
+                except InvalidVersion as e:
+                    logger.error(f"Invalid safe_version {version} - error: {e}")
 
             for version in affected_versions:
-                constraints.append(
-                    VersionConstraint(version=GentooVersion(version), comparator="=")
-                )
+                try:
+                    constraints.append(
+                        VersionConstraint(version=GentooVersion(version), comparator="=")
+                    )
+                except InvalidVersion as e:
+                    logger.error(f"Invalid affected_version {version} - error: {e}")
 
             if not constraints:
                 continue

vulnerabilities/pipelines/v2_importers/gentoo_importer.py

@@ -121,43 +121,43 @@ def cves_from_reference(reference):
         return cves
 
 
+def _yield_packages(pkg_name, pkg_ns, constraints, invert):
+    """
+    Generate AffectedPackageV2 objects for a list of constraints.
+    """
+    for comparator, version, slot_value in constraints:
+        qualifiers = {"slot": slot_value} if slot_value else {}
+        purl = PackageURL(type="ebuild", name=pkg_name, namespace=pkg_ns, qualifiers=qualifiers)
+
+        try:
+            constraint = VersionConstraint(version=GentooVersion(version), comparator=comparator)
+
+            if invert:
+                constraint = constraint.invert()
+
+            yield AffectedPackageV2(
+                package=purl,
+                affected_version_range=EbuildVersionRange(constraints=[constraint]),
+                fixed_version_range=None,
+            )
+        except InvalidVersion as e:
+            logger.error(f"InvalidVersion constraints version: {version} error:{e}")
+
+
 def affected_and_safe_purls(affected_elem):
-    constraints = []
     for pkg in affected_elem:
         name = pkg.attrib.get("name")
         if not name:
             continue
         pkg_ns, _, pkg_name = name.rpartition("/")
-        purl = PackageURL(type="ebuild", name=pkg_name, namespace=pkg_ns)
-        safe_versions, affected_versions = get_safe_and_affected_versions(pkg)
-
-        for version in safe_versions:
-            try:
-                constraints.append(
-                    VersionConstraint(version=GentooVersion(version), comparator="=").invert()
-                )
-            except InvalidVersion as e:
-                logger.error(f"InvalidVersion - version: {version} - error:{e}")
-
-        for version in affected_versions:
-            try:
-                constraints.append(
-                    VersionConstraint(version=GentooVersion(version), comparator="=")
-                )
-            except InvalidVersion as e:
-                logger.error(f"InvalidVersion - version: {version} - error:{e}")
-
-        if not constraints:
-            continue
 
-        yield AffectedPackageV2(
-            package=purl,
-            affected_version_range=EbuildVersionRange(constraints=constraints),
-            fixed_version_range=None,
-        )
+        safe_constraints, affected_constraints = get_safe_and_affected_constraints(pkg)
+
+        yield from _yield_packages(pkg_name, pkg_ns, affected_constraints, invert=False)
+        yield from _yield_packages(pkg_name, pkg_ns, safe_constraints, invert=True)
 
 
-def get_safe_and_affected_versions(pkg):
+def get_safe_and_affected_constraints(pkg):
     # TODO : Revisit why we are skipping some versions in gentoo importer
     skip_versions = {"1.3*", "7.3*", "7.4*"}
     safe_versions = set()
@@ -166,27 +166,20 @@ def get_safe_and_affected_versions(pkg):
         if info.text in skip_versions:
             continue
 
-        if info.attrib.get("range"):
-            if len(info.attrib.get("range")) > 2:
-                continue
+        # All possible values of info.attrib['range'] =
+        # {'gt', 'lt', 'rle', 'rge', 'rgt', 'le', 'ge', 'eq'}, out of
+        # which ('rle', 'rge', 'rgt') are ignored, because they compare
+        # 'release' not the 'version'.
+        range_value = info.attrib.get("range")
+        slot_value = info.attrib.get("slot")
+        comparator_dict = {"gt": ">", "lt": "<", "ge": ">=", "le": "<=", "eq": "="}
+        comparator = comparator_dict.get(range_value)
+        if not comparator:
+            continue
 
         if info.tag == "unaffected":
-            # quick hack, to know whether this
-            # version lies in this range, 'e' stands for
-            # equal, which is paired with 'greater' or 'less'.
-            # All possible values of info.attrib['range'] =
-            # {'gt', 'lt', 'rle', 'rge', 'rgt', 'le', 'ge', 'eq'}, out of
-            # which ('rle', 'rge', 'rgt') are ignored, because they compare
-            # 'release' not the 'version'.
-            if "e" in info.attrib["range"]:
-                safe_versions.add(info.text)
-            else:
-                affected_versions.add(info.text)
+            safe_versions.add((comparator, info.text, slot_value))
 
         elif info.tag == "vulnerable":
-            if "e" in info.attrib["range"]:
-                affected_versions.add(info.text)
-            else:
-                safe_versions.add(info.text)
-
+            affected_versions.add((comparator, info.text, slot_value))
     return safe_versions, affected_versions

vulnerabilities/tests/pipelines/v2_importers/test_gentoo_importer_v2.py

@@ -6,7 +6,7 @@
 # See https://github.com/aboutcode-org/vulnerablecode for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
-
+import json
 from pathlib import Path
 from unittest.mock import Mock
 from unittest.mock import patch
@@ -32,7 +32,10 @@ def test_gentoo_advisories_per_file(xml_file):
     pipeline.vcs_response = Mock(dest_dir=TEST_DATA)
 
     with patch.object(Path, "glob", return_value=[xml_file]):
-        result = [adv.to_dict() for adv in pipeline.collect_advisories()]
+        results = [adv.to_dict() for adv in pipeline.collect_advisories()]
+
+    for adv in results:
+        adv["affected_packages"].sort(key=lambda x: json.dumps(x, sort_keys=True))
 
     expected_file = xml_file.with_name(xml_file.stem + "-expected.json")
-    util_tests.check_results_against_json(result, expected_file)
+    util_tests.check_results_against_json(results, expected_file)

vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json

@@ -15,7 +15,35 @@
           "qualifiers": "",
           "subpath": ""
         },
-        "affected_version_range": "vers:ebuild/0.1.1|!=1.9.7",
+        "affected_version_range": "vers:ebuild/0.1.1",
+        "fixed_version_range": null,
+        "introduced_by_commit_patches": [],
+        "fixed_by_commit_patches": []
+      },
+      {
+        "package": {
+          "type": "ebuild",
+          "namespace": "dev-vcs",
+          "name": "subversion",
+          "version": "",
+          "qualifiers": "",
+          "subpath": ""
+        },
+        "affected_version_range": "vers:ebuild/<1.9.7",
+        "fixed_version_range": null,
+        "introduced_by_commit_patches": [],
+        "fixed_by_commit_patches": []
+      },
+      {
+        "package": {
+          "type": "ebuild",
+          "namespace": "dev-vcs",
+          "name": "subversion",
+          "version": "",
+          "qualifiers": "",
+          "subpath": ""
+        },
+        "affected_version_range": "vers:ebuild/<1.9.7",
         "fixed_version_range": null,
         "introduced_by_commit_patches": [],
         "fixed_by_commit_patches": []