Skip to content

Commit 812bc00

Browse files
michaelehabmichaelehab
authored
Merge branch 'main' into 1953-pypa-importer-package-first
2 parents 7a34791 + dcb0511 commit 812bc00

22 files changed

+11251
-1
lines changed

‎setup.cfg‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ install_requires =
101101
#vulntotal
102102
python-dotenv
103103
texttable
104+
extractcode[full]==31.0.0
104105

105106

106107
[options.extras_require]

‎vulnerabilities/importer.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,8 @@ def from_url(cls, url):
187187
reference_id = get_reference_id(url)
188188
if "GHSA-" in reference_id.upper():
189189
return cls(reference_id=reference_id, url=url)
190+
if reference_id.startswith(("RHSA-", "RHEA-", "RHBA-")):
191+
return cls(reference_id=reference_id, url=url)
190192
if is_cve(reference_id):
191193
return cls(url=url, reference_id=reference_id.upper())
192194
return cls(url=url)
@@ -458,6 +460,24 @@ def clean_summary(self, summary):
458460
return summary
459461

460462
def to_dict(self):
463+
is_adv_v2 = (
464+
self.advisory_id
465+
or self.severities
466+
or self.references_v2
467+
or (self.affected_packages and isinstance(self.affected_packages[0], AffectedPackageV2))
468+
)
469+
if is_adv_v2:
470+
return {
471+
"advisory_id": self.advisory_id,
472+
"aliases": self.aliases,
473+
"summary": self.summary,
474+
"affected_packages": [pkg.to_dict() for pkg in self.affected_packages],
475+
"references_v2": [ref.to_dict() for ref in self.references_v2],
476+
"severities": [sev.to_dict() for sev in self.severities],
477+
"date_published": self.date_published.isoformat() if self.date_published else None,
478+
"weaknesses": self.weaknesses,
479+
"url": self.url if self.url else "",
480+
}
461481
return {
462482
"aliases": self.aliases,
463483
"summary": self.summary,

‎vulnerabilities/importers/__init__.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
from vulnerabilities.pipelines import pypa_importer
4343
from vulnerabilities.pipelines import pysec_importer
4444
from vulnerabilities.pipelines.v2_importers import apache_httpd_importer as apache_httpd_v2
45+
from vulnerabilities.pipelines.v2_importers import archlinux_importer as archlinux_importer_v2
4546
from vulnerabilities.pipelines.v2_importers import curl_importer as curl_importer_v2
4647
from vulnerabilities.pipelines.v2_importers import (
4748
elixir_security_importer as elixir_security_importer_v2,
@@ -57,12 +58,14 @@
5758
from vulnerabilities.pipelines.v2_importers import pypa_importer as pypa_importer_v2
5859
from vulnerabilities.pipelines.v2_importers import pypa_live_importer as pypa_live_importer_v2
5960
from vulnerabilities.pipelines.v2_importers import pysec_importer as pysec_importer_v2
61+
from vulnerabilities.pipelines.v2_importers import redhat_importer as redhat_importer_v2
6062
from vulnerabilities.pipelines.v2_importers import vulnrichment_importer as vulnrichment_importer_v2
6163
from vulnerabilities.pipelines.v2_importers import xen_importer as xen_importer_v2
6264
from vulnerabilities.utils import create_registry
6365

6466
IMPORTERS_REGISTRY = create_registry(
6567
[
68+
archlinux_importer_v2.ArchLinuxImporterPipeline,
6669
nvd_importer_v2.NVDImporterPipeline,
6770
elixir_security_importer_v2.ElixirSecurityImporterPipeline,
6871
npm_importer_v2.NpmImporterPipeline,
@@ -78,6 +81,7 @@
7881
postgresql_importer_v2.PostgreSQLImporterPipeline,
7982
mozilla_importer_v2.MozillaImporterPipeline,
8083
github_osv_importer_v2.GithubOSVImporterPipeline,
84+
redhat_importer_v2.RedHatImporterPipeline,
8185
nvd_importer.NVDImporterPipeline,
8286
github_importer.GitHubAPIImporterPipeline,
8387
gitlab_importer.GitLabImporterPipeline,

‎vulnerabilities/improvers/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
from vulnerabilities.pipelines import flag_ghost_packages
2020
from vulnerabilities.pipelines import populate_vulnerability_summary_pipeline
2121
from vulnerabilities.pipelines import remove_duplicate_advisories
22+
from vulnerabilities.pipelines.v2_improvers import compute_advisory_todo as compute_advisory_todo_v2
2223
from vulnerabilities.pipelines.v2_improvers import compute_package_risk as compute_package_risk_v2
2324
from vulnerabilities.pipelines.v2_improvers import (
2425
computer_package_version_rank as compute_version_rank_v2,
@@ -65,6 +66,7 @@
6566
enhance_with_metasploit_v2.MetasploitImproverPipeline,
6667
compute_package_risk_v2.ComputePackageRiskPipeline,
6768
compute_version_rank_v2.ComputeVersionRankPipeline,
69+
compute_advisory_todo_v2.ComputeToDo,
6870
compute_advisory_todo.ComputeToDo,
6971
]
7072
)

‎vulnerabilities/migrations/0101_advisorytodov2_todorelatedadvisoryv2_and_more.py‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
# Generated by Django 4.2.22 on 2025-07-24 12:05
2+
3+
from django.db import migrations, models
4+
import django.db.models.deletion
5+
6+
7+
class Migration(migrations.Migration):
8+
9+
dependencies = [
10+
("vulnerabilities", "0100_remove_advisoryv2_affecting_packages_and_more"),
11+
]
12+
13+
operations = [
14+
migrations.CreateModel(
15+
name="AdvisoryToDoV2",
16+
fields=[
17+
(
18+
"id",
19+
models.AutoField(
20+
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
21+
),
22+
),
23+
(
24+
"related_advisories_id",
25+
models.CharField(
26+
help_text="SHA1 digest of the unique_content_id field of the applicable advisories.",
27+
max_length=40,
28+
),
29+
),
30+
(
31+
"issue_type",
32+
models.CharField(
33+
choices=[
34+
("MISSING_AFFECTED_PACKAGE", "Advisory is missing affected package"),
35+
("MISSING_FIXED_BY_PACKAGE", "Advisory is missing fixed-by package"),
36+
(
37+
"MISSING_AFFECTED_AND_FIXED_BY_PACKAGES",
38+
"Advisory is missing both affected and fixed-by packages",
39+
),
40+
("MISSING_SUMMARY", "Advisory is missing summary"),
41+
(
42+
"CONFLICTING_FIXED_BY_PACKAGES",
43+
"Advisories have conflicting fixed-by packages",
44+
),
45+
(
46+
"CONFLICTING_AFFECTED_PACKAGES",
47+
"Advisories have conflicting affected packages",
48+
),
49+
(
50+
"CONFLICTING_AFFECTED_AND_FIXED_BY_PACKAGES",
51+
"Advisories have conflicting affected and fixed-by packages",
52+
),
53+
(
54+
"CONFLICTING_SEVERITY_SCORES",
55+
"Advisories have conflicting severity scores",
56+
),
57+
],
58+
db_index=True,
59+
help_text="Select the issue that needs to be addressed from the available options.",
60+
max_length=50,
61+
),
62+
),
63+
(
64+
"issue_detail",
65+
models.TextField(blank=True, help_text="Additional details about the issue."),
66+
),
67+
(
68+
"created_at",
69+
models.DateTimeField(
70+
auto_now_add=True,
71+
help_text="Timestamp indicating when this TODO was created.",
72+
),
73+
),
74+
(
75+
"is_resolved",
76+
models.BooleanField(
77+
db_index=True, default=False, help_text="This TODO is resolved or not."
78+
),
79+
),
80+
(
81+
"resolved_at",
82+
models.DateTimeField(
83+
blank=True,
84+
help_text="Timestamp indicating when this TODO was resolved.",
85+
null=True,
86+
),
87+
),
88+
(
89+
"resolution_detail",
90+
models.TextField(
91+
blank=True, help_text="Additional detail on how this TODO was resolved."
92+
),
93+
),
94+
],
95+
),
96+
migrations.CreateModel(
97+
name="ToDoRelatedAdvisoryV2",
98+
fields=[
99+
(
100+
"id",
101+
models.AutoField(
102+
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
103+
),
104+
),
105+
(
106+
"advisory",
107+
models.ForeignKey(
108+
on_delete=django.db.models.deletion.CASCADE, to="vulnerabilities.advisoryv2"
109+
),
110+
),
111+
(
112+
"todo",
113+
models.ForeignKey(
114+
on_delete=django.db.models.deletion.CASCADE,
115+
to="vulnerabilities.advisorytodov2",
116+
),
117+
),
118+
],
119+
options={
120+
"unique_together": {("todo", "advisory")},
121+
},
122+
),
123+
migrations.AddField(
124+
model_name="advisorytodov2",
125+
name="advisories",
126+
field=models.ManyToManyField(
127+
help_text="Advisory/ies where this TODO is applicable.",
128+
related_name="advisory_todos",
129+
through="vulnerabilities.ToDoRelatedAdvisoryV2",
130+
to="vulnerabilities.advisoryv2",
131+
),
132+
),
133+
migrations.AlterUniqueTogether(
134+
name="advisorytodov2",
135+
unique_together={("related_advisories_id", "issue_type")},
136+
),
137+
]

‎vulnerabilities/models.py‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2493,6 +2493,62 @@ class Meta:
24932493
unique_together = ("related_advisories_id", "issue_type")
24942494

24952495

2496+
class AdvisoryToDoV2(models.Model):
2497+
"""Track the TODOs for advisory/ies that need to be addressed."""
2498+
2499+
# Since we can not make advisories field (M2M field) unique
2500+
# (see https://code.djangoproject.com/ticket/702), we use related_advisories_id
2501+
# to avoid creating duplicate issue for same set of advisories,
2502+
related_advisories_id = models.CharField(
2503+
max_length=40,
2504+
help_text="SHA1 digest of the unique_content_id field of the applicable advisories.",
2505+
)
2506+
2507+
advisories = models.ManyToManyField(
2508+
"AdvisoryV2",
2509+
through="ToDoRelatedAdvisoryV2",
2510+
related_name="advisory_todos",
2511+
help_text="Advisory/ies where this TODO is applicable.",
2512+
)
2513+
2514+
issue_type = models.CharField(
2515+
max_length=50,
2516+
choices=ISSUE_TYPE_CHOICES,
2517+
db_index=True,
2518+
help_text="Select the issue that needs to be addressed from the available options.",
2519+
)
2520+
2521+
issue_detail = models.TextField(
2522+
blank=True,
2523+
help_text="Additional details about the issue.",
2524+
)
2525+
2526+
created_at = models.DateTimeField(
2527+
auto_now_add=True,
2528+
help_text="Timestamp indicating when this TODO was created.",
2529+
)
2530+
2531+
is_resolved = models.BooleanField(
2532+
default=False,
2533+
db_index=True,
2534+
help_text="This TODO is resolved or not.",
2535+
)
2536+
2537+
resolved_at = models.DateTimeField(
2538+
null=True,
2539+
blank=True,
2540+
help_text="Timestamp indicating when this TODO was resolved.",
2541+
)
2542+
2543+
resolution_detail = models.TextField(
2544+
blank=True,
2545+
help_text="Additional detail on how this TODO was resolved.",
2546+
)
2547+
2548+
class Meta:
2549+
unique_together = ("related_advisories_id", "issue_type")
2550+
2551+
24962552
class AdvisorySeverity(models.Model):
24972553
url = models.URLField(
24982554
max_length=1024,
@@ -2834,6 +2890,7 @@ def to_advisory_data(self) -> "AdvisoryData":
28342890
from vulnerabilities.importer import AdvisoryData
28352891

28362892
return AdvisoryData(
2893+
advisory_id=self.advisory_id,
28372894
aliases=[item.alias for item in self.aliases.all()],
28382895
summary=self.summary,
28392896
affected_packages=[
@@ -2933,6 +2990,21 @@ class Meta:
29332990
unique_together = ("todo", "advisory")
29342991

29352992

2993+
class ToDoRelatedAdvisoryV2(models.Model):
2994+
todo = models.ForeignKey(
2995+
AdvisoryToDoV2,
2996+
on_delete=models.CASCADE,
2997+
)
2998+
2999+
advisory = models.ForeignKey(
3000+
AdvisoryV2,
3001+
on_delete=models.CASCADE,
3002+
)
3003+
3004+
class Meta:
3005+
unique_together = ("todo", "advisory")
3006+
3007+
29363008
class PackageQuerySetV2(BaseQuerySet, PackageURLQuerySet):
29373009
def search(self, query: str = None):
29383010
"""

0 commit comments

Comments
 (0)