diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index be084844b..fad9236b9 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -73,6 +73,7 @@ from vulnerabilities.pipelines.v2_importers import pypa_importer as pypa_importer_v2 from vulnerabilities.pipelines.v2_importers import pysec_importer as pysec_importer_v2 from vulnerabilities.pipelines.v2_importers import redhat_importer as redhat_importer_v2 +from vulnerabilities.pipelines.v2_importers import retiredotnet_importer as retiredotnet_importer_v2 from vulnerabilities.pipelines.v2_importers import ruby_importer as ruby_importer_v2 from vulnerabilities.pipelines.v2_importers import ubuntu_osv_importer as ubuntu_osv_importer_v2 from vulnerabilities.pipelines.v2_importers import vulnrichment_importer as vulnrichment_importer_v2 @@ -108,6 +109,7 @@ debian_importer_v2.DebianImporterPipeline, mattermost_importer_v2.MattermostImporterPipeline, apache_tomcat_v2.ApacheTomcatImporterPipeline, + retiredotnet_importer_v2.RetireDotnetImporterPipeline, ubuntu_osv_importer_v2.UbuntuOSVImporterPipeline, nvd_importer.NVDImporterPipeline, github_importer.GitHubAPIImporterPipeline, diff --git a/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py b/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py new file mode 100644 index 000000000..bb92ed884 --- /dev/null +++ b/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py @@ -0,0 +1,138 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import json +import re +from collections import defaultdict +from pathlib import Path + +from fetchcode.vcs import fetch_via_vcs +from packageurl import PackageURL +from univers.version_range import NugetVersionRange + +from vulnerabilities.importer import AdvisoryDataV2 +from vulnerabilities.importer import AffectedPackageV2 +from vulnerabilities.importer import ReferenceV2 +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 +from vulnerabilities.utils import get_advisory_url + + +class RetireDotnetImporterPipeline(VulnerableCodeBaseImporterPipelineV2): + license_url = "https://github.com/RetireNet/Packages/blob/master/LICENSE" + spdx_license_expression = "MIT" + repo_url = "git+https://github.com/RetireNet/Packages/" + pipeline_id = "retiredotnet_importer_v2" + run_once = True + + @classmethod + def steps(cls): + return ( + cls.clone, + cls.collect_and_store_advisories, + cls.clean_downloads, + ) + + def clone(self): + self.log(f"Cloning `{self.repo_url}`") + self.vcs_response = fetch_via_vcs(self.repo_url) + + def advisories_count(self): + root = Path(self.vcs_response.dest_dir) / "Content" + return sum(1 for _ in root.rglob("*.json")) + + def collect_advisories(self): + base_path = Path(self.vcs_response.dest_dir) + vuln = base_path / "Content" + affected_packages = [] + + for file in vuln.glob("*.json"): + advisory_id = "retiredotnet-" + file.stem + advisory_url = get_advisory_url( + file=file, + base_path=base_path, + url="https://github.com/RetireNet/Packages/blob/master/", + ) + with open(file) as f: + json_doc = json.load(f) + description = json_doc.get("description") or "" + aliases = self.vuln_id_from_desc(description) + + # group by package name `id` + # { pkg_id: {'affected_versions': set(), 'fixed': set()} } + grouped_packages = defaultdict( + lambda: {"affected_versions": set(), "fixed_versions": set()} + ) + for pkg in json_doc.get("packages") or []: + name = pkg.get("id") + if not name: + continue + + affected_version = pkg.get("affected") + if affected_version: + grouped_packages[name]["affected_versions"].add(affected_version) + + fixed_version = pkg.get("fix") + if fixed_version: + grouped_packages[name]["fixed_versions"].add(fixed_version) + + for pkg in grouped_packages: + affected_version_range = None + affected_versions = grouped_packages[pkg]["affected_versions"] + if affected_versions: + affected_version_range = NugetVersionRange.from_versions(affected_versions) + + fixed_version_range = None + fixed_versions = grouped_packages[pkg]["fixed_versions"] + if fixed_versions: + fixed_version_range = NugetVersionRange.from_versions(fixed_versions) + + if affected_version_range or fixed_version_range: + affected_packages.append( + AffectedPackageV2( + package=PackageURL(type="nuget", name=pkg), + affected_version_range=affected_version_range, + fixed_version_range=fixed_version_range, + ) + ) + + link = json_doc.get("link") + if link: + vuln_reference = [ + ReferenceV2( + url=link, + ) + ] + + yield AdvisoryDataV2( + advisory_id=advisory_id, + aliases=[aliases] if aliases else [], + summary=description, + affected_packages=affected_packages, + references=vuln_reference, + url=advisory_url, + ) + + @staticmethod + def vuln_id_from_desc(desc): + cve_regex = re.compile(r"CVE-\d+-\d+") + res = cve_regex.search(desc) + if res: + return desc[res.start() : res.end()] + else: + return None + + def clean_downloads(self): + """Cleanup any temporary repository data.""" + if self.vcs_response: + self.log(f"Removing cloned repository") + self.vcs_response.delete() + + def on_failure(self): + """Ensure cleanup is always performed on failure.""" + self.clean_downloads() diff --git a/vulnerabilities/tests/pipelines/v2_importers/test_retiredotnet_imprter.py b/vulnerabilities/tests/pipelines/v2_importers/test_retiredotnet_imprter.py new file mode 100644 index 000000000..3a66a14e3 --- /dev/null +++ b/vulnerabilities/tests/pipelines/v2_importers/test_retiredotnet_imprter.py @@ -0,0 +1,45 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from pathlib import Path +from unittest.mock import Mock +from unittest.mock import patch + +import pytest + +from vulnerabilities.pipelines.v2_importers.retiredotnet_importer import ( + RetireDotnetImporterPipeline, +) +from vulnerabilities.tests import util_tests + +TEST_DATA = Path(__file__).parent.parent.parent / "test_data" / "retiredotnet_v2" + + +def test_vuln_id_from_desc(): + importer = RetireDotnetImporterPipeline() + gibberish = "xyzabcpqr123" * 50 + "\n" * 100 + res = importer.vuln_id_from_desc(gibberish) + assert res is None + + desc = "abcdef CVE-2002-1968 pqrstuvwxyz:_|-|" + res = importer.vuln_id_from_desc(desc) + assert res == "CVE-2002-1968" + + +@pytest.mark.django_db +def test_retiredotnet_advisories_per_file(): + pipeline = RetireDotnetImporterPipeline() + test_file = TEST_DATA / "12.json" + expected_file = TEST_DATA / "expected_file.json" + pipeline.vcs_response = Mock(dest_dir=TEST_DATA) + + with patch.object(Path, "glob", return_value=[test_file]): + result = [adv.to_dict() for adv in pipeline.collect_advisories()] + + util_tests.check_results_against_json(result, expected_file) diff --git a/vulnerabilities/tests/test_data/retiredotnet_v2/12.json b/vulnerabilities/tests/test_data/retiredotnet_v2/12.json new file mode 100644 index 000000000..5b21f8b26 --- /dev/null +++ b/vulnerabilities/tests/test_data/retiredotnet_v2/12.json @@ -0,0 +1,176 @@ +{ + "link": "https://github.com/aspnet/Announcements/issues/334", + "description": "Microsoft Security Advisory CVE-2019-0564: ASP.NET Core Denial of Service Vulnerability", + "packages": [ + { + "id": "Microsoft.AspNetCore.WebSockets", + "affected": "2.1.0", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.WebSockets", + "affected": "2.1.1", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.WebSockets", + "affected": "2.2.0", + "fix": "2.2.1" + }, + { + "id": "Microsoft.AspNetCore.Server.Kestrel.Core", + "affected": "2.1.0", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.Server.Kestrel.Core", + "affected": "2.1.1", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.Server.Kestrel.Core", + "affected": "2.1.2", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.Server.Kestrel.Core", + "affected": "2.1.3", + "fix": "2.1.7" + }, + { + "id": "System.Net.WebSockets.WebSocketProtocol", + "affected": "4.5.0", + "fix": "4.5.3" + }, + { + "id": "System.Net.WebSockets.WebSocketProtocol", + "affected": "4.5.1", + "fix": "4.5.3" + }, + { + "id": "System.Net.WebSockets.WebSocketProtocol", + "affected": "4.5.2", + "fix": "4.5.3" + }, + { + "id": "Microsoft.NETCore.App", + "affected": "2.1.0", + "fix": "2.1.7" + }, + { + "id": "Microsoft.NETCore.App", + "affected": "2.1.1", + "fix": "2.1.7" + }, + { + "id": "Microsoft.NETCore.App", + "affected": "2.1.2", + "fix": "2.1.7" + }, + { + "id": "Microsoft.NETCore.App", + "affected": "2.1.3", + "fix": "2.1.7" + }, + { + "id": "Microsoft.NETCore.App", + "affected": "2.1.4", + "fix": "2.1.7" + }, + { + "id": "Microsoft.NETCore.App", + "affected": "2.1.5", + "fix": "2.1.7" + }, + { + "id": "Microsoft.NETCore.App", + "affected": "2.1.6", + "fix": "2.1.7" + }, + { + "id": "Microsoft.NETCore.App", + "affected": "2.2.0", + "fix": "2.2.1" + }, + { + "id": "Microsoft.AspNetCore.App", + "affected": "2.1.0", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.App", + "affected": "2.1.1", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.App", + "affected": "2.1.2", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.App", + "affected": "2.1.3", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.App", + "affected": "2.1.4", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.App", + "affected": "2.1.5", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.App", + "affected": "2.1.6", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.App", + "affected": "2.2.0", + "fix": "2.2.1" + }, + { + "id": "Microsoft.AspNetCore.All", + "affected": "2.1.0", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.All", + "affected": "2.1.1", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.All", + "affected": "2.1.2", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.All", + "affected": "2.1.3", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.All", + "affected": "2.1.4", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.All", + "affected": "2.1.5", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.All", + "affected": "2.1.6", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.All", + "affected": "2.2.0", + "fix": "2.2.1" + } + ] +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/retiredotnet_v2/expected_file.json b/vulnerabilities/tests/test_data/retiredotnet_v2/expected_file.json new file mode 100644 index 000000000..7c48a66b5 --- /dev/null +++ b/vulnerabilities/tests/test_data/retiredotnet_v2/expected_file.json @@ -0,0 +1,107 @@ +[ + { + "advisory_id": "retiredotnet-12", + "aliases": [ + "CVE-2019-0564" + ], + "summary": "Microsoft Security Advisory CVE-2019-0564: ASP.NET Core Denial of Service Vulnerability", + "affected_packages": [ + { + "package": { + "type": "nuget", + "namespace": "", + "name": "Microsoft.AspNetCore.WebSockets", + "version": "", + "qualifiers": "", + "subpath": "" + }, + "affected_version_range": "vers:nuget/2.1.0|2.1.1|2.2.0", + "fixed_version_range": "vers:nuget/2.1.7|2.2.1", + "introduced_by_commit_patches": [], + "fixed_by_commit_patches": [] + }, + { + "package": { + "type": "nuget", + "namespace": "", + "name": "Microsoft.AspNetCore.Server.Kestrel.Core", + "version": "", + "qualifiers": "", + "subpath": "" + }, + "affected_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3", + "fixed_version_range": "vers:nuget/2.1.7", + "introduced_by_commit_patches": [], + "fixed_by_commit_patches": [] + }, + { + "package": { + "type": "nuget", + "namespace": "", + "name": "System.Net.WebSockets.WebSocketProtocol", + "version": "", + "qualifiers": "", + "subpath": "" + }, + "affected_version_range": "vers:nuget/4.5.0|4.5.1|4.5.2", + "fixed_version_range": "vers:nuget/4.5.3", + "introduced_by_commit_patches": [], + "fixed_by_commit_patches": [] + }, + { + "package": { + "type": "nuget", + "namespace": "", + "name": "Microsoft.NETCore.App", + "version": "", + "qualifiers": "", + "subpath": "" + }, + "affected_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3|2.1.4|2.1.5|2.1.6|2.2.0", + "fixed_version_range": "vers:nuget/2.1.7|2.2.1", + "introduced_by_commit_patches": [], + "fixed_by_commit_patches": [] + }, + { + "package": { + "type": "nuget", + "namespace": "", + "name": "Microsoft.AspNetCore.App", + "version": "", + "qualifiers": "", + "subpath": "" + }, + "affected_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3|2.1.4|2.1.5|2.1.6|2.2.0", + "fixed_version_range": "vers:nuget/2.1.7|2.2.1", + "introduced_by_commit_patches": [], + "fixed_by_commit_patches": [] + }, + { + "package": { + "type": "nuget", + "namespace": "", + "name": "Microsoft.AspNetCore.All", + "version": "", + "qualifiers": "", + "subpath": "" + }, + "affected_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3|2.1.4|2.1.5|2.1.6|2.2.0", + "fixed_version_range": "vers:nuget/2.1.7|2.2.1", + "introduced_by_commit_patches": [], + "fixed_by_commit_patches": [] + } + ], + "references": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://github.com/aspnet/Announcements/issues/334" + } + ], + "patches": [], + "severities": [], + "date_published": null, + "weaknesses": [], + "url": "https://github.com/RetireNet/Packages/blob/master/12.json" + } +] \ No newline at end of file