Add locations/commits and push protection estimate

enrich_code_scanning_alerts.py

@@ -129,6 +129,8 @@ def enrich_alerts(alerts: list, metadata: dict) -> None:
 
 def format_header(key: str) -> str:
     """Format the heading depending on its value."""
+    output = ""
+
     if key not in ["cwe", "language"]:
         output = PUNCTUATION_RE.sub(" ", key).title()
     elif key == "cwe":
@@ -668,6 +670,8 @@ def main() -> None:
     enrich_alerts(alerts, metadata)
     fixup_alerts(alerts)
 
+    fields = []
+
     if args.format in ["html", "pdf"]:
         fields = (
             [

estimate_push_protection_rate.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+
+"""Estimate how many secrets would have been detected in a list of existing secret detections, and a list of which patterns have push protection now."""
+
+import argparse
+import json
+from datetime import datetime, timezone
+
+
+def add_args(parser: argparse.ArgumentParser) -> None:
+    """Add command line arguments to the parser."""
+    parser.add_argument(
+        "secrets_file",
+        type=str,
+        help="Path to the file containing the list of secrets",
+    )
+    parser.add_argument(
+        "patterns_file",
+        type=str,
+        help="Path to the file containing the list of patterns with push protection",
+    )
+
+
+def main() -> None:
+    """Command line entry point."""
+    parser = argparse.ArgumentParser(
+        description="Estimate push protection rate for secrets"
+    )
+    add_args(parser)
+    args = parser.parse_args()
+
+    with open(args.patterns_file, "r") as f:
+        patterns: set = {line.strip() for line in f if line.strip()}
+
+    with open(args.secrets_file, "r") as f:
+        secrets = json.load(f)
+
+    total_secrets = len(secrets)
+    protected_secrets = [secret for secret in secrets if secret.get("secret_type") in patterns]
+
+    print(f"Total secrets: {total_secrets}")
+    print(f"Protected secrets: {len(protected_secrets)}")
+
+    if total_secrets > 0:
+        protection_rate = (len(protected_secrets) / total_secrets) * 100
+        print(f"Estimated push protection rate: {protection_rate:.2f}%")
+    else:
+        print("No secrets found to evaluate.")
+
+    # now evaluate how often we'd expect to block pushes, using the `first_commit_date` field
+    # that's in ISO format with a Z suffix
+    now = datetime.now(timezone.utc)
+
+    # find the oldest blocked commit
+    earliest_blocked_commit_date = min([
+        datetime.fromisoformat(secret["first_commit_date"].replace("Z", "+00:00"))
+        for secret in protected_secrets
+    ])
+
+    blocking_timespan = now - earliest_blocked_commit_date
+    rate = len(protected_secrets) / blocking_timespan.days if blocking_timespan.days > 0 else len(protected_secrets)
+
+    print(f"Estimated secrets blocked per day since {earliest_blocked_commit_date.date()}: {rate:.2f}")
+
+
+if __name__ == "__main__":
+    main()

githubapi.py

@@ -26,6 +26,20 @@
 ISO_NO_TZ_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}$")
 VALID_NAME_RE = re.compile(r"^[A-Za-z0-9_.-]{1,39}$")
 
+GENERIC_SECRET_TYPES = ",".join(
+    [
+        "http_basic_authentication_header",
+        "http_bearer_authentication_header",
+        "mongodb_connection_string",
+        "mysql_connection_string",
+        "openssh_private_key",
+        "pgp_private_key",
+        "postgres_connection_string",
+        "rsa_private_key",
+        "password",     # Copilot powered secret detection
+    ]
+)
+
 
 class RateLimited(Exception):
     """Rate limited exception."""
@@ -54,7 +68,7 @@ def __init__(self, token: str | None = None, hostname="github.com") -> None:
         self.hostname = hostname
 
     @classmethod
-    def check_name(self, name: str, scope: str) -> bool:
+    def check_name(cls, name: str, scope: str) -> bool:
         """Check the name is valid."""
         # check repo slug has <owner</<repo> format or org/Enterprise name is valid
         if scope == "repo":
@@ -112,7 +126,7 @@ def query(
         if paging is None:
             try:
                 result = self._do(url, method, data=data)
-                yield result
+                yield result.json()
             except Exception as e:
                 LOG.error("Error: %s", e)
                 # show traceback without raising the exception
@@ -161,6 +175,8 @@ def construct_api_url(
 
         path = api_path + scope_path + endpoint
 
+        query_params = {}
+
         if paging is None:
             query_params = {}
         elif paging == "cursor":
@@ -314,7 +330,7 @@ def paginate(
                     break
 
                 if progress:
-                    pbar.update(1)
+                    pbar.update(1)  # type: ignore
 
                 LOG.debug(data)
 
@@ -413,9 +429,14 @@ def list_secret_scanning_alerts(
         since: datetime.datetime | None = None,
         scope: str = "org",
         bypassed: bool = False,
+        generic: bool = False,
     ) -> Generator[dict, None, None]:
         """List secret scanning alerts for a GitHub repository, organization or Enterprise."""
         query = {"state": state} if state is not None else {}
+
+        if generic:
+            query["secret_type"] = GENERIC_SECRET_TYPES
+
         alerts = self.query(
             scope,
             name,

list_code_scanning_alerts.py

@@ -4,7 +4,6 @@
 
 import sys
 import argparse
-import re
 import logging
 import datetime
 import json