feat: burpference recon scanner for remote domains

.github/workflows/rigging_pr_description.yml

@@ -21,10 +21,11 @@ jobs:
         id: diff
         # shellcheck disable=SC2102
         run: |
-          git fetch origin "${GITHUB_BASE_REF}"
-          MERGE_BASE="$(git merge-base HEAD "origin/${GITHUB_BASE_REF}")"
-          DIFF="$(git diff "${MERGE_BASE}" HEAD | base64 --wrap=0)"
-          echo "diff=${DIFF}" >> "${GITHUB_OUTPUT}"
+          git fetch origin "${{ github.base_ref }}"
+          MERGE_BASE=$(git merge-base HEAD "origin/${{ github.base_ref }}")
+          # Use separate diff arguments instead of range notation
+          DIFF=$(git diff "$MERGE_BASE" HEAD | base64 --wrap=0)
+          echo "diff=${DIFF}" >> "$GITHUB_OUTPUT"
       - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b #v5.0.3
         with:
           python-version: "3.11"
@@ -45,14 +46,6 @@ jobs:
           GIT_DIFF: ${{ steps.diff.outputs.diff }}
         run: |
           python .github/scripts/rigging_pr_decorator.py
-      # Extract PR body
-      - name: Extract PR body
-        id: pr
-        run: |
-          PR_BODY="$(gh pr view "${GITHUB_EVENT_PULL_REQUEST_NUMBER}" --json body --jq .body)"
-          echo "body=${PR_BODY}" >> "${GITHUB_OUTPUT}"
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       # Update the PR description
       - name: Update PR Description
         uses: nefrob/pr-description@4dcc9f3ad5ec06b2a197c5f8f93db5e69d2fdca7 #v1.2.0

README.md

@@ -6,7 +6,7 @@ Experimenting with yarrr' Burp Proxy tab going brrrrrrrrrrrrr.
 
 [![GitHub release (latest by date)](https://img.shields.io/github/v/release/dreadnode/burpference)](https://github.com/dreadnode/burpference/releases)
 [![GitHub stars](https://img.shields.io/github/stars/dreadnode/burpference?style=social)](https://github.com/dreadnode/burpference/stargazers)
-[![GitHub license](https://img.shields.io/github/license/dreadnode/burpference)](https://github.com/dreadnode/burpference/blob/main/LICENSE)
+[![GitHub license](https://img.shields.io/github/license/dreadnode/burpference)](https://img.shields.io/github/license/dreadnode/burpference)
 [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](https://github.com/dreadnode/burpference/pulls)
 
 •
@@ -40,6 +40,14 @@ Some key features:
   - Only in-scope items are sent, optimizing resource usage and avoiding unnecessary API calls.
   - By default, [certain MIME types are excluded](https://github.com/dreadnode/burpference/blob/7e81641e263bbdfe4a38e30746eb3c27f3454190/burpference/burpference.py#L616).
   - Color-coded tabs display `critical/high/medium/low/informational` findings from your model for easy visualization.
+- **Scanner Analysis**: A dedicated scanner tab provides focused security analysis capabilities:
+  - Direct analysis of URLs and OpenAPI specifications
+  - Load the configuration files using the API adapter, the same as usual in burpference for efficient management of API keys/model selection etc
+  - Automated extraction of security headers and server information
+  - Real-time security header assessment (X-Frame-Options, CSP, HSTS, etc.)
+  - Custom system prompts for specialized analysis scenarios
+  - Support for both single-endpoint and full domain scanning
+  - Integration with Burp's native issue reporting system
 - **Comprehensive Logging**: A logging system allows you to review intercepted responses, API requests sent, and replies received—all clearly displayed for analysis.
   - A clean table interface displaying all logs, intercepted responses, API calls, and status codes for comprehensive engagement tracking.
   - Stores inference logs in both the "_Inference Logger_" tab as a live preview and a timestamped file in the /logs directory.

burpference/burpference.py

@@ -1,21 +1,24 @@
 # -*- coding: utf-8 -*-
 # type: ignore[import]
+from datetime import datetime
 from burp import IBurpExtender, ITab, IHttpListener, IScanIssue
-from java.awt import BorderLayout, GridBagLayout, GridBagConstraints, Font
+from java.awt import BorderLayout, GridBagLayout, GridBagConstraints, Font, Dimension
 from javax.swing import (
     JPanel, JTextArea, JScrollPane,
     BorderFactory, JSplitPane, JButton, JComboBox,
     JTable, table, ListSelectionModel, JOptionPane, JTextField, JTabbedPane)
+from javax.swing import BoxLayout, JLabel
 from javax.swing.table import DefaultTableCellRenderer, TableRowSorter
 from javax.swing.border import TitledBorder
 from java.util import Comparator
 import json
 import urllib2
 import os
-from datetime import datetime
 from consts import *
 from api_adapters import get_api_adapter
 from issues import BurpferenceIssue
+from threading import Thread
+from scanner import BurpferenceScanner
 
 
 def load_ascii_art(file_path):
@@ -51,6 +54,8 @@ def __init__(self):
         self.temp_log_messages = []
         self.request_counter = 0
         self.log_message("Extension initialized and running.")
+        self._hosts = set()
+        self.scanner = None  # Will initialize after callbacks
 
     def registerExtenderCallbacks(self, callbacks):
         self._callbacks = callbacks
@@ -264,12 +269,35 @@ def compare(self, s1, s2):
         self._panel.add(diffSplitPane, BorderLayout.NORTH)
 
         self.inference_tab = self.create_inference_logger_tab()
+        self.scanner_tab = self.create_scanner_tab()
+
         self.tabbedPane = JTabbedPane()
         self.tabbedPane.setBackground(DARK_BACKGROUND)
         self.tabbedPane.setForeground(DREADNODE_GREY)
         self.tabbedPane.addTab("burpference", self._panel)
         self.tabbedPane.addTab("Inference Logger", self.inference_tab)
 
+        # Initialize scanner AFTER loading config
+        self.scanner = None
+
+        # Now initialize scanner with current config
+        colors = {
+            'DARK_BACKGROUND': DARK_BACKGROUND,
+            'LIGHTER_BACKGROUND': LIGHTER_BACKGROUND,
+            'DREADNODE_GREY': DREADNODE_GREY,
+            'DREADNODE_ORANGE': DREADNODE_ORANGE
+        }
+        self.scanner = BurpferenceScanner(
+            callbacks=self._callbacks,
+            helpers=self._helpers,
+            config=None,
+            api_adapter=None,
+            colors=colors
+        )
+
+        self.scanner_tab = self.scanner.create_scanner_tab()
+        self.tabbedPane.addTab("Scanner", self.scanner_tab)
+
         for i in range(self.tabbedPane.getTabCount()):
             self.tabbedPane.setBackgroundAt(i, DREADNODE_GREY)
             self.tabbedPane.setForegroundAt(i, DREADNODE_ORANGE)
@@ -333,33 +361,50 @@ def loadConfiguration(self, event):
             try:
                 with open(config_path, 'r') as config_file:
                     self.config = json.load(config_file)
-                self.log_message("Loaded configuration: %s" %
-                                 json.dumps(self.config, indent=2))
+                    self.config["config_file"] = selected_config
+
                 try:
                     self.api_adapter = get_api_adapter(self.config)
+                    if self.scanner:
+                        self.scanner.config = self.config
+                        self.scanner.api_adapter = self.api_adapter
+                        self.scanner.update_config_display()
                     self.log_message("API adapter initialized successfully")
                 except ValueError as e:
                     self.log_message("Error initializing API adapter: %s" % str(e))
                     self.api_adapter = None
+                    if self.scanner:
+                        self.scanner.api_adapter = None
                 except Exception as e:
                     self.log_message(
                         "Unexpected error initializing API adapter: %s" % str(e))
                     self.api_adapter = None
+                    if self.scanner:
+                        self.scanner.api_adapter = None
             except ValueError as e:
                 self.log_message(
                     "Error parsing JSON in configuration file: %s" % str(e))
                 self.config = None
                 self.api_adapter = None
+                if self.scanner:
+                    self.scanner.config = None
+                    self.scanner.api_adapter = None
             except Exception as e:
                 self.log_message(
                     "Unexpected error loading configuration: %s" % str(e))
                 self.config = None
                 self.api_adapter = None
+                if self.scanner:
+                    self.scanner.config = None
+                    self.scanner.api_adapter = None
         else:
             self.log_message(
                 "Configuration file %s not found." % selected_config)
             self.config = None
             self.api_adapter = None
+            if self.scanner:
+                self.scanner.config = None
+                self.scanner.api_adapter = None
 
     def create_inference_logger_tab(self):
         panel = JPanel(BorderLayout())
@@ -434,6 +479,133 @@ def create_inference_logger_tab(self):
 
         return panel
 
+    def create_scanner_tab(self):
+        """Creates the burpference scanner tab with domain filtering and direct model interaction"""
+        panel = JPanel()
+        panel.setLayout(BoxLayout(panel, BoxLayout.Y_AXIS))
+        panel.setBackground(DARK_BACKGROUND)
+
+        # Create top control panel
+        top_panel = JPanel()
+        top_panel.setLayout(BoxLayout(top_panel, BoxLayout.X_AXIS))
+        top_panel.setBackground(DARK_BACKGROUND)
+
+        # Domain selector
+        domain_panel = JPanel()
+        domain_panel.setBackground(DARK_BACKGROUND)
+        domain_label = JLabel("Target Domain:")
+        domain_label.setForeground(DREADNODE_GREY)
+        self._domain_selector = JComboBox(list(self._hosts))
+        self._domain_selector.setBackground(LIGHTER_BACKGROUND)
+        self._domain_selector.setForeground(DREADNODE_GREY)
+        domain_panel.add(domain_label)
+        domain_panel.add(self._domain_selector)
+        top_panel.add(domain_panel)
+
+        # Optional prompt input
+        middle_panel = JPanel()
+        middle_panel.setBackground(DARK_BACKGROUND)
+        middle_panel.setLayout(BoxLayout(middle_panel, BoxLayout.Y_AXIS))
+        prompt_label = JLabel("Custom Analysis Prompt:")
+        prompt_label.setForeground(DREADNODE_GREY)
+        self._custom_prompt = JTextArea(5, 50)
+        self._custom_prompt.setLineWrap(True)
+        self._custom_prompt.setWrapStyleWord(True)
+        self._custom_prompt.setBackground(LIGHTER_BACKGROUND)
+        self._custom_prompt.setForeground(DREADNODE_ORANGE)
+        prompt_scroll = JScrollPane(self._custom_prompt)
+
+        # Analyze button
+        analyze_button = JButton("Analyze Domain", actionPerformed=self.analyze_domain)
+        analyze_button.setBackground(DREADNODE_ORANGE)
+        analyze_button.setForeground(DREADNODE_GREY)
+
+        middle_panel.add(prompt_label)
+        middle_panel.add(prompt_scroll)
+        middle_panel.add(analyze_button)
+
+        # Results area
+        self._scanner_output = JTextArea(20, 50)
+        self._scanner_output.setEditable(False)
+        self._scanner_output.setLineWrap(True)
+        self._scanner_output.setWrapStyleWord(True)
+        self._scanner_output.setBackground(LIGHTER_BACKGROUND)
+        self._scanner_output.setForeground(DREADNODE_ORANGE)
+        scanner_scroll = JScrollPane(self._scanner_output)
+
+        # Add all components
+        panel.add(top_panel)
+        panel.add(middle_panel)
+        panel.add(scanner_scroll)
+
+        return panel
+
+    def analyze_domain(self, event):
+        """Handles the domain analysis button click"""
+        domain = self._domain_selector.getSelectedItem()
+        custom_prompt = self._custom_prompt.getText()
+
+        def run_analysis():
+            self._scanner_output.setText("Analyzing domain: %s...\n" % domain)
+            try:
+                # Get all requests for selected domain
+                http_pairs = self.get_domain_traffic(domain)
+                if not http_pairs:
+                    self._scanner_output.append("\nNo traffic found for domain.")
+                    return
+
+                # Use custom prompt if provided, otherwise use default
+                prompt = custom_prompt if custom_prompt else "Analyze this domain's traffic for security issues:"
+
+                # Prepare and send to current model
+                analysis_request = self.api_adapter.prepare_request(
+                    user_content=json.dumps(http_pairs, indent=2),
+                    system_content=prompt
+                )
+
+                # Make request and process response
+                req = urllib2.Request(self.config.get("host", ""))
+                for header, value in self.config.get("headers", {}).items():
+                    req.add_header(header, value)
+
+                response = urllib2.urlopen(req, json.dumps(analysis_request))
+                response_data = response.read()
+                analysis = self.api_adapter.process_response(response_data)
+
+                # Update UI
+                self._scanner_output.setText("Analysis for %s:\n\n%s" % (domain, analysis))
+
+            except Exception as e:
+                self._scanner_output.setText("Error analyzing domain: %s" % str(e))
+
+        # Run analysis in background thread
+        Thread(target=run_analysis).start()
+
+    def get_domain_traffic(self, domain):
+        """Gets all traffic for a specific domain"""
+        traffic = []
+        for message in self._callbacks.getProxyHistory():
+            if domain in message.getHttpService().getHost():
+                analyzed_request = self._helpers.analyzeRequest(message)
+                analyzed_response = self._helpers.analyzeResponse(message.getResponse())
+
+                # Extract request/response data
+                request_info = {
+                    "method": analyzed_request.getMethod(),
+                    "url": str(message.getUrl()),
+                    "headers": dict(header.split(': ', 1) for header in analyzed_request.getHeaders()[1:] if ': ' in header),
+                    "body": message.getRequest()[analyzed_request.getBodyOffset():].tostring()
+                }
+
+                response_info = {
+                    "status": analyzed_response.getStatusCode(),
+                    "headers": dict(header.split(': ', 1) for header in analyzed_response.getHeaders()[1:] if ': ' in header),
+                    "body": message.getResponse()[analyzed_response.getBodyOffset():].tostring()
+                }
+
+                traffic.append({"request": request_info, "response": response_info})
+        return traffic
+
     def inferenceLogSelectionChanged(self, event):
         selectedRow = self.inferenceLogTable.getSelectedRow()
         if selectedRow != -1:
@@ -451,9 +623,7 @@ def inferenceLogSelectionChanged(self, event):
                 except (ValueError, TypeError):
                     formatted_request = str(request)
 
-                # For response, try to extract the message content if it's a model response
                 try:
-                    # Handle case where response is already a dict
                     if isinstance(response, dict):
                         response_obj = response
                     else:
@@ -552,7 +722,7 @@ def getTableCellRendererComponent(self, table, value, isSelected, hasFocus, row,
 
     def log_message(self, message):
         timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        log_entry = "[{0}] {1}\n".format(timestamp, message)  # Python2 format strings
+        log_entry = "[{0}] {1}\n".format(timestamp, message)
 
         if self.logArea is None:
             self.temp_log_messages.append(log_entry)
@@ -562,12 +732,10 @@ def log_message(self, message):
                 self.logArea.getDocument().getLength())
 
         try:
-            # Try to create/write to log file with explicit permissions
             log_dir = os.path.dirname(self.log_file_path)
             if not os.path.exists(log_dir):
-                os.makedirs(log_dir, 0755)  # Python2 octal notation
+                os.makedirs(log_dir, 0755)
 
-            # Open with explicit write permissions
             with open(self.log_file_path, 'a+') as log_file:
                 log_file.write(log_entry)
         except (IOError, OSError) as e:
@@ -629,7 +797,7 @@ def create_scan_issue(self, messageInfo, processed_response):
                 detail = str(processed_response)
 
             if detail.startswith('"') and detail.endswith('"'):
-                detail = detail[1:-1]  # Remove surrounding quotes
+                detail = detail[1:-1]
 
             # Create properly formatted issue name
             issue_name = "burpference: %s Security Finding" % severity
@@ -651,6 +819,13 @@ def create_scan_issue(self, messageInfo, processed_response):
             self.log_message("Error creating scan issue: %s" % str(e))
 
     def processHttpMessage(self, toolFlag, messageIsRequest, messageInfo):
+        if messageIsRequest:
+            # Add new domains to both main extension and scanner
+            host = messageInfo.getHttpService().getHost()
+            if host not in self._hosts:
+                self._hosts.add(host)
+                if self.scanner:
+                    self.scanner.add_host(host)
         if not self.is_running:
             return
         if not self.api_adapter:
@@ -661,6 +836,13 @@ def processHttpMessage(self, toolFlag, messageIsRequest, messageInfo):
         if messageIsRequest:
             # Store the request for later use
             self.current_request = messageInfo
+            host = messageInfo.getHttpService().getHost()
+            if host not in self._hosts:
+                self._hosts.add(host)
+                if hasattr(self, '_domain_selector'):
+                    self._domain_selector.addItem(host)
+            if self.scanner:
+                self.scanner.add_host(host)
         else:
             request = self.current_request
             response = messageInfo
@@ -799,7 +981,6 @@ def processHttpMessage(self, toolFlag, messageIsRequest, messageInfo):
 
                     self.requestArea.append("\n\n=== Request #" + str(self.request_counter) + " ===\n")
                     try:
-                        # Format the request nicely
                         formatted_request = json.dumps(http_pair, indent=2)
                         formatted_request = formatted_request.replace('\\n', '\n')
                         formatted_request = formatted_request.replace('\\"', '"')
@@ -810,7 +991,6 @@ def processHttpMessage(self, toolFlag, messageIsRequest, messageInfo):
 
                     self.responseArea.append("\n\n=== Response #" + str(self.request_counter) + " ===\n")
                     try:
-                        # Format the response nicely
                         if isinstance(processed_response, dict) and 'message' in processed_response and 'content' in processed_response['message']:
                             formatted_response = processed_response['message']['content']
                         else: