diff --git a/fixtures/github.py b/fixtures/github.py index c070ea6cb1826a..fdaa3d9c868984 100644 --- a/fixtures/github.py +++ b/fixtures/github.py @@ -3541,3 +3541,30 @@ "site_admin": false } }""" + +# Simplified example of a check_run rerequested action event +# Note: installation.id must match the external_id used in create_github_integration (default: "12345") +# Note: repository.id must match a Repository created for the organization (use create_repo in tests) +CHECK_RUN_REREQUESTED_ACTION_EVENT_EXAMPLE = b"""{ + "action": "rerequested", + "installation": {"id": 12345}, + "repository": { + "id": 35129377, + "full_name": "getsentry/sentry", + "html_url": "https://github.com/getsentry/sentry" + }, + "check_run": { + "external_id": "4663713", + "html_url": "https://github.com/test/repo/runs/4" + } +}""" + +CHECK_RUN_COMPLETED_EVENT_EXAMPLE = b"""{ + "action": "completed", + "installation": {"id": 12345}, + "repository": { + "id": 35129377, + "full_name": "getsentry/sentry", + "html_url": "https://github.com/getsentry/sentry" + } +}""" diff --git a/src/sentry/integrations/github/webhook.py b/src/sentry/integrations/github/webhook.py index c0936aeb2d0e85..97146bddb688bf 100644 --- a/src/sentry/integrations/github/webhook.py +++ b/src/sentry/integrations/github/webhook.py @@ -85,8 +85,11 @@ def provider(self) -> str: def _handle(self, integration: RpcIntegration, event: Mapping[str, Any], **kwargs) -> None: pass - def __call__(self, event: Mapping[str, Any], **kwargs) -> None: - external_id = get_github_external_id(event=event, host=kwargs.get("host")) + def __call__(self, event: Mapping[str, Any], **kwargs: Mapping[str, Any]) -> None: + host = kwargs.get("host") + external_id = get_github_external_id( + event=event, host=host if isinstance(host, str) else None + ) result = integration_service.organization_contexts( external_id=external_id, provider=self.provider @@ -784,6 +787,34 @@ def _handle( handle_github_pr_webhook_for_autofix(organization, action, pull_request, user) +class CheckRunEventWebhook(GitHubWebhook): + """ + Handles GitHub check_run webhook events. + https://docs.github.com/en/webhooks/webhook-events-and-payloads#check_run + """ + + @property + def event_type(self) -> IntegrationWebhookEventType: + return IntegrationWebhookEventType.PULL_REQUEST + + def _handle( + self, + integration: RpcIntegration, + event: Mapping[str, Any], + **kwargs, + ) -> None: + # Get organization from kwargs (populated by GitHubWebhook base class) + organization = kwargs.get("organization") + if organization is None: + logger.warning("github.webhook.check_run.missing-organization") + return + + # XXX: Add support for registering functions to call + from sentry.seer.error_prediction.webhooks import handle_github_check_run_event + + handle_github_check_run_event(organization=organization, event=event) + + @all_silo_endpoint class GitHubIntegrationsWebhookEndpoint(Endpoint): """ @@ -804,6 +835,7 @@ class GitHubIntegrationsWebhookEndpoint(Endpoint): GithubWebhookType.PULL_REQUEST: PullRequestEventWebhook, GithubWebhookType.INSTALLATION: InstallationEventWebhook, GithubWebhookType.ISSUE: IssuesEventWebhook, + GithubWebhookType.CHECK_RUN: CheckRunEventWebhook, } def get_handler(self, event_type: str) -> type[GitHubWebhook] | None: diff --git a/src/sentry/integrations/github/webhook_types.py b/src/sentry/integrations/github/webhook_types.py index 06a1575eca1d1b..33fba1347d2d9a 100644 --- a/src/sentry/integrations/github/webhook_types.py +++ b/src/sentry/integrations/github/webhook_types.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from enum import StrEnum GITHUB_WEBHOOK_TYPE_HEADER = "HTTP_X_GITHUB_EVENT" @@ -14,3 +16,4 @@ class GithubWebhookType(StrEnum): PULL_REQUEST_REVIEW_COMMENT = "pull_request_review_comment" PULL_REQUEST_REVIEW = "pull_request_review" PUSH = "push" + CHECK_RUN = "check_run" diff --git a/src/sentry/seer/error_prediction/__init__.py b/src/sentry/seer/error_prediction/__init__.py new file mode 100644 index 00000000000000..d68ab5a3f20605 --- /dev/null +++ b/src/sentry/seer/error_prediction/__init__.py @@ -0,0 +1 @@ +# Error prediction module for Seer integration diff --git a/src/sentry/seer/error_prediction/webhooks.py b/src/sentry/seer/error_prediction/webhooks.py new file mode 100644 index 00000000000000..c804cbb2dff22e --- /dev/null +++ b/src/sentry/seer/error_prediction/webhooks.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import logging +from collections.abc import Mapping +from enum import StrEnum +from typing import Any + +from sentry.models.organization import Organization +from sentry.seer.signed_seer_api import post_to_seer +from sentry.utils import metrics +from sentry.utils.seer import can_use_prevent_ai_features + +logger = logging.getLogger(__name__) + + +# https://docs.github.com/en/webhooks/webhook-events-and-payloads#check_run +class CheckRunAction(StrEnum): + COMPLETED = "completed" + CREATED = "created" + REQUESTED_ACTION = "requested_action" + REREQUESTED = "rerequested" + + +HANDLED_ACTIONS = [CheckRunAction.REREQUESTED] +# This needs to match the value defined in the Seer API: +# https://github.com/getsentry/seer/blob/main/src/seer/automation/codegen/pr_review_coding_agent.py +SEER_PR_REVIEW_RERUN_PATH = "/v1/automation/codegen/pr-review/rerun" + + +def handle_github_check_run_event( + organization: Organization, + event: Mapping[str, Any], +) -> bool: + """ + Handle GitHub check_run webhook events for PR review rerun. + + This is called when a check_run event is received from GitHub. + When a user clicks "Re-run" on a check run in GitHub UI, we forward + the original run ID to Seer so it can rerun the PR review. + + Args: + organization: The Sentry organization + event: The webhook event payload + + Returns: + True if the event was handled successfully, False otherwise + """ + check_run = event.get("check_run") + action = event.get("action") + + extra = { + "organization_id": organization.id, + "action": action, + "check_run_html_url": check_run.get("html_url") if check_run else None, + "check_run_name": check_run.get("name") if check_run else None, + "check_run_external_id": check_run.get("external_id") if check_run else None, + } + + # Check if the org has opted in to Prevent AI features (Code Review) + # This checks feature flags, org options, and billing plan type + if not can_use_prevent_ai_features(organization): + logger.debug("seer.error_prediction.check_run.feature_disabled", extra=extra) + return False + + # Only handle relevant actions + if action not in HANDLED_ACTIONS: + logger.debug("seer.error_prediction.check_run.skipped_action", extra=extra) + return False + + # Validate required fields after feature/action checks + if not check_run: + logger.warning("seer.error_prediction.check_run.missing_check_run", extra=extra) + return False + + # Extract and validate external_id (required for Seer to identify the original run) + raw_external_id = check_run.get("external_id") + if not raw_external_id: + logger.warning("seer.error_prediction.check_run.missing_external_id", extra=extra) + return False + + try: + original_run_id = int(raw_external_id) + except (TypeError, ValueError): + logger.warning( + "seer.error_prediction.check_run.invalid_external_id", + extra={**extra, "raw_external_id": raw_external_id}, + ) + return False + + # Forward the original run ID to Seer for PR review rerun + payload = {"original_run_id": original_run_id} + outcome = "failure" + try: + post_to_seer(path=SEER_PR_REVIEW_RERUN_PATH, payload=payload) + outcome = "success" + except Exception: + logger.exception("seer.error_prediction.check_run.forward.exception", extra=extra) + finally: + metrics.incr("seer.error_prediction.check_run.forward.outcome", tags={"outcome": outcome}) + + return outcome == "success" diff --git a/src/sentry/seer/signed_seer_api.py b/src/sentry/seer/signed_seer_api.py index 9bd2c7f045a5fd..35f5467df9423f 100644 --- a/src/sentry/seer/signed_seer_api.py +++ b/src/sentry/seer/signed_seer_api.py @@ -1,10 +1,13 @@ import hashlib import hmac import logging +from collections.abc import Mapping from random import random from typing import Any from urllib.parse import urlparse +import orjson +import requests import sentry_sdk from django.conf import settings from urllib3 import BaseHTTPResponse, HTTPConnectionPool, Retry @@ -53,6 +56,53 @@ def make_signed_seer_api_request( ) +@sentry_sdk.tracing.trace +def post_to_seer( + path: str, + payload: Mapping[str, Any], + timeout: int | float = 5, + base_url: str | None = None, +) -> requests.Response: + """ + Simple wrapper to POST data to Seer with automatic signing. + + This is a simpler alternative to make_signed_seer_api_request for one-off requests. + Use this for webhook forwarding and simple API calls. Use make_signed_seer_api_request + if you need connection pooling for high-volume requests. + + Args: + path: The API path (e.g., "/v1/automation/codegen/pr-review/github") + payload: The data to send (will be JSON-serialized) + timeout: Request timeout in seconds (default: 5) + base_url: Base URL override (defaults to settings.SEER_AUTOFIX_URL) + + Returns: + requests.Response object + + Raises: + requests.HTTPError: If the response status indicates an error + """ + base_url = base_url or settings.SEER_AUTOFIX_URL + body = orjson.dumps(payload) + + with metrics.timer( + "seer.request_to_seer", + sample_rate=1.0, + tags={"endpoint": path}, + ): + response = requests.post( + f"{base_url}{path}", + data=body, + headers={ + "content-type": "application/json;charset=utf-8", + **sign_with_seer_secret(body), + }, + timeout=timeout, + ) + response.raise_for_status() + return response + + def sign_with_seer_secret(body: bytes) -> dict[str, str]: auth_headers: dict[str, str] = {} if random() < options.get("seer.api.use-shared-secret"): diff --git a/src/sentry/testutils/helpers/github.py b/src/sentry/testutils/helpers/github.py new file mode 100644 index 00000000000000..0e57b0182ce6cc --- /dev/null +++ b/src/sentry/testutils/helpers/github.py @@ -0,0 +1,137 @@ +""" +Utilities for testing GitHub integration webhooks. +""" + +from __future__ import annotations + +import hashlib +import hmac +from datetime import datetime, timedelta +from typing import Any +from uuid import uuid4 + +from sentry import options +from sentry.silo.base import SiloMode +from sentry.testutils.cases import APITestCase +from sentry.testutils.silo import assume_test_silo_mode + + +def compute_github_webhook_signature(body: bytes, secret: str, method: str = "sha256") -> str: + """ + Compute GitHub webhook signature for testing. + + This uses the same HMAC logic as GitHub's webhook signature validation. + + Args: + body: The request body as bytes + secret: The webhook secret + method: Hash method ('sha1' or 'sha256') + + Returns: + Signature string in format "method=hexdigest" + """ + if method == "sha256": + mod = hashlib.sha256 + elif method == "sha1": + mod = hashlib.sha1 + else: + raise ValueError(f"Unsupported hash method: {method}") + + signature = hmac.new(key=secret.encode("utf-8"), msg=body, digestmod=mod).hexdigest() + return f"{method}={signature}" + + +class GitHubWebhookTestCase(APITestCase): + """ + Base test case for GitHub webhook tests. + + Provides common utilities for: + - Setting up GitHub integrations + - Computing webhook signatures + - Sending webhook events with proper authentication + """ + + def setUp(self) -> None: + super().setUp() + self.github_webhook_url = "/extensions/github/webhook/" + self.github_webhook_secret = "b3002c3e321d4b7880360d397db2ccfd" + options.set("github-app.webhook-secret", self.github_webhook_secret) + + def create_github_integration( + self, + external_id: str = "12345", + access_token: str = "1234", + **metadata_overrides: Any, + ): + """ + Create a GitHub integration for testing. + + Args: + external_id: GitHub installation ID + access_token: GitHub access token + **metadata_overrides: Additional metadata fields + + Returns: + The created integration + """ + future_expires = datetime.now().replace(microsecond=0) + timedelta(minutes=5) + metadata = { + "access_token": access_token, + "expires_at": future_expires.isoformat(), + **metadata_overrides, + } + + with assume_test_silo_mode(SiloMode.CONTROL): + integration = self.create_integration( + organization=self.organization, + external_id=external_id, + provider="github", + metadata=metadata, + ) + integration.add_organization(self.project.organization.id, self.user) + + return integration + + def send_github_webhook_event( + self, + event_type: str, + event_data: str | bytes, + **extra_headers: str, + ): + """ + Send a GitHub webhook event with proper signatures and headers. + + Args: + event_type: GitHub event type (e.g., "push", "pull_request", "check_run") + event_data: The webhook event payload (as JSON string or bytes) + **extra_headers: Additional HTTP headers + + Returns: + Response from the webhook endpoint + """ + # Convert to bytes if needed + event_bytes = event_data.encode("utf-8") if isinstance(event_data, str) else event_data + + # Compute signatures + sha1_sig = compute_github_webhook_signature(event_bytes, self.github_webhook_secret, "sha1") + sha256_sig = compute_github_webhook_signature( + event_bytes, self.github_webhook_secret, "sha256" + ) + + # Build headers + headers = { + "HTTP_X_GITHUB_EVENT": event_type, + "HTTP_X_HUB_SIGNATURE": sha1_sig, + "HTTP_X_HUB_SIGNATURE_256": sha256_sig, + "HTTP_X_GITHUB_DELIVERY": str(uuid4()), + **extra_headers, + } + + # The DRF APIClient stubs can misinterpret **extra headers as a positional arg + client: Any = self.client + return client.post( + self.github_webhook_url, + data=event_data, + content_type="application/json", + **headers, + ) diff --git a/tests/sentry/integrations/github/test_check_run_webhook.py b/tests/sentry/integrations/github/test_check_run_webhook.py new file mode 100644 index 00000000000000..156cfbd89b109f --- /dev/null +++ b/tests/sentry/integrations/github/test_check_run_webhook.py @@ -0,0 +1,102 @@ +from unittest.mock import MagicMock, patch + +import orjson +import responses +from django.conf import settings +from rest_framework.response import Response + +from fixtures.github import ( + CHECK_RUN_COMPLETED_EVENT_EXAMPLE, + CHECK_RUN_REREQUESTED_ACTION_EVENT_EXAMPLE, +) +from sentry.testutils.helpers.features import with_feature +from sentry.testutils.helpers.github import GitHubWebhookTestCase + + +class CheckRunEventWebhookTest(GitHubWebhookTestCase): + def _send_check_run_event(self, event_data: bytes | str) -> Response: + """Helper to send check_run event.""" + integration = self.create_github_integration() + # Create a repository that matches the fixture's repository.id (35129377) + self.create_repo( + project=self.project, + provider="integrations:github", + external_id="35129377", + integration_id=integration.id, + ) + response = self.send_github_webhook_event("check_run", event_data) + assert response.status_code == 204 + return response + + @patch("sentry.integrations.github.webhook.CheckRunEventWebhook.__call__") + def test_check_run_requested_action_event_triggers_handler( + self, mock_event_handler: MagicMock + ) -> None: + """Test that check_run requested_action events trigger the webhook handler.""" + self._send_check_run_event(CHECK_RUN_REREQUESTED_ACTION_EVENT_EXAMPLE) + assert mock_event_handler.called + + @responses.activate + @with_feature({"organizations:gen-ai-features", "organizations:seat-based-seer-enabled"}) + def test_check_run_rerequested_forwards_to_seer(self) -> None: + """Test that rerequested check_run events forward original_run_id to Seer.""" + # Mock the Seer API endpoint + responses.add( + responses.POST, + f"{settings.SEER_AUTOFIX_URL}/v1/automation/codegen/pr-review/rerun", + json={"success": True}, + status=200, + ) + + self._send_check_run_event(CHECK_RUN_REREQUESTED_ACTION_EVENT_EXAMPLE) + + # Verify the request was made to Seer + assert len(responses.calls) == 1 + request = responses.calls[0].request + + # Verify request body contains original_run_id extracted from external_id + body = orjson.loads(request.body) + assert body == {"original_run_id": 4663713} + + @responses.activate + @with_feature({"organizations:gen-ai-features", "organizations:seat-based-seer-enabled"}) + def test_check_run_completed_is_skipped(self) -> None: + """Test that completed check_run events are skipped (not handled).""" + # Mock the Seer API endpoint (should NOT be called) + responses.add( + responses.POST, + f"{settings.SEER_AUTOFIX_URL}/v1/automation/codegen/pr-review/rerun", + json={"success": True}, + status=200, + ) + + self._send_check_run_event(CHECK_RUN_COMPLETED_EVENT_EXAMPLE) + + # Verify NO request was made to Seer (completed action is not handled) + assert len(responses.calls) == 0 + + @responses.activate + @with_feature({"organizations:gen-ai-features", "organizations:seat-based-seer-enabled"}) + def test_check_run_handles_seer_error_gracefully(self) -> None: + """Test that Seer API errors are caught and logged without failing the webhook.""" + # Mock Seer API to return an error + responses.add( + responses.POST, + f"{settings.SEER_AUTOFIX_URL}/v1/automation/codegen/pr-review/rerun", + json={"error": "Internal server error"}, + status=500, + ) + + # Should not raise an exception, just log it + self._send_check_run_event(CHECK_RUN_REREQUESTED_ACTION_EVENT_EXAMPLE) + + # Verify the request was attempted + assert len(responses.calls) == 1 + + def test_check_run_without_integration_returns_204(self) -> None: + """Test that check_run events without integration return 204.""" + # Don't create an integration, just send the event + response = self.send_github_webhook_event("check_run", CHECK_RUN_COMPLETED_EVENT_EXAMPLE) + + # Should still return 204 even without integration + assert response.status_code == 204 diff --git a/tests/sentry/seer/error_prediction/test_webhooks.py b/tests/sentry/seer/error_prediction/test_webhooks.py new file mode 100644 index 00000000000000..49838638bfaaf6 --- /dev/null +++ b/tests/sentry/seer/error_prediction/test_webhooks.py @@ -0,0 +1,159 @@ +from unittest.mock import patch + +import orjson +import responses +from django.conf import settings + +from sentry.seer.error_prediction.webhooks import handle_github_check_run_event +from sentry.testutils.cases import TestCase +from sentry.testutils.helpers.features import with_feature + + +class HandleGithubCheckRunEventTest(TestCase): + def setUp(self): + super().setUp() + self.organization = self.create_organization() + self.action_rerequested_event = { + "action": "rerequested", + "check_run": { + "external_id": "4663713", + "html_url": "https://github.com/test/repo/runs/4", + }, + } + + def test_skips_when_prevent_ai_features_disabled(self): + """Test that the handler returns early when AI features are not enabled.""" + # Without enabling feature flags, can_use_prevent_ai_features returns False + with patch("sentry.seer.error_prediction.webhooks.logger") as mock_logger: + success = handle_github_check_run_event( + organization=self.organization, + event=self.action_rerequested_event, + ) + assert not success + # Verify debug log for disabled feature + mock_logger.debug.assert_called_once() + assert "feature_disabled" in mock_logger.debug.call_args[0][0] + + @with_feature({"organizations:gen-ai-features", "organizations:seat-based-seer-enabled"}) + def test_skips_non_handled_actions(self): + """Test that non-handled actions are skipped.""" + non_handled_actions = ["created", "completed", "requested_action", None] + + for action in non_handled_actions: + with patch("sentry.seer.error_prediction.webhooks.logger") as mock_logger: + event = { + "action": action, + "check_run": { + "external_id": "4663713", + "html_url": "https://github.com/test/repo/runs/4", + }, + } + success = handle_github_check_run_event( + organization=self.organization, + event=event, + ) + assert not success + # Verify debug log for skipped action + mock_logger.debug.assert_called_once() + assert "skipped_action" in mock_logger.debug.call_args[0][0] + + @responses.activate + @with_feature({"organizations:gen-ai-features", "organizations:seat-based-seer-enabled"}) + def test_forwards_rerequested_action_to_seer(self): + """Test that rerequested action forwards original_run_id to Seer.""" + responses.add( + responses.POST, + f"{settings.SEER_AUTOFIX_URL}/v1/automation/codegen/pr-review/rerun", + json={"success": True}, + status=200, + ) + + success = handle_github_check_run_event( + organization=self.organization, + event=self.action_rerequested_event, + ) + assert success + + # Verify request was made with correct payload + assert len(responses.calls) == 1 + body = orjson.loads(responses.calls[0].request.body) + assert body == {"original_run_id": 4663713} + + @with_feature({"organizations:gen-ai-features", "organizations:seat-based-seer-enabled"}) + def test_fails_when_external_id_missing(self): + """Test that missing external_id returns False.""" + event = { + "action": "rerequested", + "check_run": {"html_url": "https://github.com/test/repo/runs/4"}, + } + + with patch("sentry.seer.error_prediction.webhooks.logger") as mock_logger: + success = handle_github_check_run_event( + organization=self.organization, + event=event, + ) + assert not success + mock_logger.warning.assert_called_once() + assert "missing_external_id" in mock_logger.warning.call_args[0][0] + + @with_feature({"organizations:gen-ai-features", "organizations:seat-based-seer-enabled"}) + def test_fails_when_external_id_not_numeric(self): + """Test that non-numeric external_id returns False.""" + event = { + "action": "rerequested", + "check_run": { + "external_id": "not-a-number", + "html_url": "https://github.com/test/repo/runs/4", + }, + } + + with patch("sentry.seer.error_prediction.webhooks.logger") as mock_logger: + success = handle_github_check_run_event( + organization=self.organization, + event=event, + ) + assert not success + mock_logger.warning.assert_called_once() + assert "invalid_external_id" in mock_logger.warning.call_args[0][0] + + @responses.activate + @with_feature({"organizations:gen-ai-features", "organizations:seat-based-seer-enabled"}) + def test_handles_seer_error_response(self): + """Test that Seer errors are caught and logged.""" + responses.add( + responses.POST, + f"{settings.SEER_AUTOFIX_URL}/v1/automation/codegen/pr-review/rerun", + json={"error": "Internal server error"}, + status=500, + ) + + with patch("sentry.seer.error_prediction.webhooks.logger") as mock_logger: + success = handle_github_check_run_event( + organization=self.organization, + event=self.action_rerequested_event, + ) + assert not success + # Verify exception logging + mock_logger.exception.assert_called_once() + assert "check_run.forward.exception" in mock_logger.exception.call_args[0][0] + + @responses.activate + @with_feature({"organizations:gen-ai-features", "organizations:seat-based-seer-enabled"}) + def test_includes_signed_headers(self): + """Test that request includes signed headers for Seer authentication.""" + responses.add( + responses.POST, + f"{settings.SEER_AUTOFIX_URL}/v1/automation/codegen/pr-review/rerun", + json={"success": True}, + status=200, + ) + + success = handle_github_check_run_event( + organization=self.organization, + event=self.action_rerequested_event, + ) + assert success + + # Verify request has content-type header + request = responses.calls[0].request + assert request.headers["content-type"] == "application/json;charset=utf-8"