Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions lambdas/handlers/im_alerting_handler.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import json
import os

from services.im_alerting_service import IMAlertingService
from utils.audit_logging_setup import LoggingService
Expand All @@ -21,6 +22,7 @@
"SLACK_CHANNEL_ID",
"SLACK_BOT_TOKEN",
"WORKSPACE",
"VIRUS_SCANNER_TOPIC_ARN",
]
)
def lambda_handler(event, context):
Expand All @@ -32,4 +34,15 @@ def lambda_handler(event, context):
logger.info(f"Processing message: {message}")

message_service = IMAlertingService(message)

if is_virus_scanner_topic(sns_message["Sns"]):
message_service.handle_virus_scanner_alert()
return

message_service.handle_alarm_alert()


def is_virus_scanner_topic(message):

topic_arn = message.get("TopicArn", "")
return topic_arn == os.environ["VIRUS_SCANNER_TOPIC_ARN"]
1 change: 0 additions & 1 deletion lambdas/models/fhir/R4/fhir_document_reference.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
)
from pydantic import BaseModel, Field
from utils.exceptions import FhirDocumentReferenceException

from utils.ods_utils import PCSE_ODS_CODE

# Constants
Expand Down
2 changes: 1 addition & 1 deletion lambdas/models/templates/slack_alert_blocks.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
"type": "section",
"text": {
"type": "mrkdwn",
"text": "*Info:*\n <{{ action_url }}>"
"text": "{% if is_initial_message %}*Info:*\n <{{ action_url }}>{% endif %}"
}
}
]
33 changes: 33 additions & 0 deletions lambdas/models/templates/virus_scanner_alert_slack_blocks.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
[
{
"type": "header",
"text": {
"type": "plain_text",
"text": "{{ topic }}: {{ scan_result }} {{ severity }}"
}
},
{
"type": "divider"
},
{
"type": "section",
"text": {
"type": "plain_text",
"text": "Scan date and time: {{ scan_date }}"
}
},
{
"type": "section",
"text": {
"type": "plain_text",
"text": "Scan ID: {{ scan_id }}"
}
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "*Info:*\n <{{ action_url }}>"
}
}
]
66 changes: 62 additions & 4 deletions lambdas/services/im_alerting_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,19 @@ def handle_ok_action_trigger(self, tags: dict, alarm_entry: AlarmEntry):
f"Alarm entry for {alarm_entry.alarm_name_metric} has been updated since reaching OK state"
)

def handle_virus_scanner_alert(self):

slack_blocks = {
"blocks": self.compose_virus_scanner_slack_blocks(),
"channel": os.environ["SLACK_CHANNEL_ID"],
}

requests.post(
url=self.SLACK_POST_CHAT_API,
headers=self.slack_headers,
data=json.dumps(slack_blocks),
)

"""
We want to wait for a set time (ALARM_OK_WAIT_SECONDS) to allow the alarm's OK state to stabilise before updating
the teams & slack alerts to display OK. This will prevent a situation where an alarm temporarily reaches an OK
Expand Down Expand Up @@ -383,6 +396,10 @@ def extract_alarm_names_from_arns(self, arn_list: list) -> list:
alarm_names.append(match.group(1))
return alarm_names

def extract_topic_name_from_arn(self, arn: str) -> str:
components = arn.split(":")
return components[-1]

def add_ttl_to_alarm_entry(self, alarm_entry: AlarmEntry):
alarm_entry.time_to_exist = int(
(
Expand Down Expand Up @@ -471,7 +488,9 @@ def compose_teams_message(self, alarm_entry: AlarmEntry):
def send_initial_slack_alert(self, alarm_entry: AlarmEntry):
slack_message = {
"channel": alarm_entry.channel_id,
"blocks": self.compose_slack_message_blocks(alarm_entry),
"blocks": self.compose_slack_message_blocks(
alarm_entry=alarm_entry, is_initial_message=True
),
}

try:
Expand Down Expand Up @@ -505,7 +524,9 @@ def send_slack_response(self, alarm_entry: AlarmEntry):
slack_message = {
"channel": alarm_entry.channel_id,
"thread_ts": alarm_entry.slack_timestamp,
"blocks": self.compose_slack_message_blocks(alarm_entry),
"blocks": self.compose_slack_message_blocks(
alarm_entry, is_initial_message=False
),
}

try:
Expand All @@ -529,7 +550,9 @@ def update_original_slack_message(self, alarm_entry: AlarmEntry):
slack_message = {
"channel": alarm_entry.channel_id,
"ts": alarm_entry.slack_timestamp,
"blocks": self.compose_slack_message_blocks(alarm_entry),
"blocks": self.compose_slack_message_blocks(
alarm_entry=alarm_entry, is_initial_message=True
),
}

requests.post(
Expand All @@ -547,7 +570,11 @@ def update_original_slack_message(self, alarm_entry: AlarmEntry):
f"Unexpected error updating original Slack message for alarm {alarm_entry.alarm_name_metric}: {e}"
)

def compose_slack_message_blocks(self, alarm_entry: AlarmEntry):
def compose_slack_message_blocks(
self,
alarm_entry: AlarmEntry,
is_initial_message: bool,
):
with open(f"{os.getcwd()}/models/templates/slack_alert_blocks.json", "r") as f:
template_content = f.read()

Expand All @@ -560,6 +587,37 @@ def compose_slack_message_blocks(self, alarm_entry: AlarmEntry):
"action_url": self.create_action_url(
self.confluence_base_url, alarm_entry.alarm_name_metric
),
"is_initial_message": is_initial_message,
}

rendered_json = template.render(context)
return json.loads(rendered_json)

def compose_virus_scanner_slack_blocks(self):
with open(
f"{os.getcwd()}/models/templates/virus_scanner_alert_slack_blocks.json", "r"
) as f:
template_content = f.read()

template = Template(template_content)

topic = self.extract_topic_name_from_arn(self.message["TopicArn"])
result = self.message["Message"].get("result", "")

timestamp = self.create_alarm_timestamp(
self.message["Message"].get("dateScanned", "")
)
scan_date = self.format_time_string(timestamp)

context = {
"topic": topic,
"scan_result": result,
"scan_date": scan_date,
"severity": f":{AlarmSeverity.HIGH.additional_value}:",
"scan_id": self.message["Message"].get("id", ""),
"action_url": self.create_action_url(
self.confluence_base_url, f"{topic} {result}"
),
}

rendered_json = template.render(context)
Expand Down
1 change: 1 addition & 0 deletions lambdas/tests/unit/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,7 @@ def set_env(monkeypatch):
monkeypatch.setenv("SLACK_BOT_TOKEN", MOCK_SLACK_BOT_TOKEN)
monkeypatch.setenv("SLACK_CHANNEL_ID", MOCK_ALERTING_SLACK_CHANNEL_ID)
monkeypatch.setenv("ITOC_TESTING_ODS_CODES", MOCK_ITOC_ODS_CODES)
monkeypatch.setenv("VIRUS_SCANNER_TOPIC_ARN", "virus_scanner_topic_arn")
monkeypatch.setenv("STAGING_STORE_BUCKET_NAME", MOCK_STAGING_STORE_BUCKET)
monkeypatch.setenv("METADATA_SQS_QUEUE_URL", MOCK_LG_METADATA_SQS_QUEUE)

Expand Down
64 changes: 64 additions & 0 deletions lambdas/tests/unit/handlers/test_im_alerting_handler.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
import json

import pytest
from handlers.im_alerting_handler import is_virus_scanner_topic, lambda_handler
from tests.unit.helpers.data.alerting.mock_sns_alerts import (
MOCK_LAMBDA_ALERT_MESSAGE,
MOCK_VIRUS_SCANNER_ALERT_SNS_MESSAGE,
)


@pytest.fixture
def mock_service_with_alarm_alert(mocker):
mocked_class = mocker.patch("handlers.im_alerting_handler.IMAlertingService")
mocked_instance = mocked_class.return_value
mocker.patch.object(mocked_instance, "dynamo_service")
mocked_class.return_value.message = MOCK_LAMBDA_ALERT_MESSAGE
return mocked_instance


@pytest.fixture
def mock_service_with_virus_scanner_alert(mocker):
mocked_class = mocker.patch("handlers.im_alerting_handler.IMAlertingService")
mocked_instance = mocked_class.return_value
mocker.patch.object(mocked_instance, "dynamo_service")
mocked_class.return_value.message = MOCK_VIRUS_SCANNER_ALERT_SNS_MESSAGE
return mocked_instance


def test_handler_calls_handle_alarm_message_lambda_triggered_by_alarm_message(
mock_service_with_alarm_alert, context, set_env
):

event = {"Records": [{"Sns": {"Message": json.dumps(MOCK_LAMBDA_ALERT_MESSAGE)}}]}
lambda_handler(event, context)

mock_service_with_alarm_alert.handle_virus_scanner_alert.assert_not_called()
mock_service_with_alarm_alert.handle_alarm_alert.assert_called()


def test_handler_calls_handle_virus_scanner_alert_lambda_triggered_by_virus_scanner_sns(
mock_service_with_virus_scanner_alert, context, set_env
):

event = {
"Records": [
{
"Sns": {
"TopicArn": MOCK_VIRUS_SCANNER_ALERT_SNS_MESSAGE["TopicArn"],
"Message": json.dumps(
MOCK_VIRUS_SCANNER_ALERT_SNS_MESSAGE["Message"]
),
}
}
]
}
lambda_handler(event, context)
mock_service_with_virus_scanner_alert.handle_virus_scanner_alert.assert_called()
mock_service_with_virus_scanner_alert.handle_alarm_alert.assert_not_called()


def test_is_virus_scanner_topic(set_env):

assert is_virus_scanner_topic(MOCK_VIRUS_SCANNER_ALERT_SNS_MESSAGE)
assert not is_virus_scanner_topic(MOCK_LAMBDA_ALERT_MESSAGE)
33 changes: 33 additions & 0 deletions lambdas/tests/unit/helpers/data/alerting/mock_slack_reply.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
[
{
"type": "header",
"text": {
"type": "plain_text",
"text": "dev-test_bulk_upload_metadata_queue ApproximateAgeOfOldestMessage Alert: :red_circle:"
}
},
{
"type": "divider"
},
{
"type": "section",
"text": {
"type": "plain_text",
"text": "History: :red_circle:"
}
},
{
"type": "section",
"text": {
"type": "plain_text",
"text": "Last state change: 15:10:41 17-04-2025 UTC"
}
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": ""
}
}
]
66 changes: 66 additions & 0 deletions lambdas/tests/unit/helpers/data/alerting/mock_sns_alerts.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
from tests.unit.conftest import MOCK_LG_METADATA_SQS_QUEUE, TEST_UUID

ALERT_TIME = "2025-04-17T15:10:41.433+0000"

QUEUE_ALERT_MESSAGE = {
"AlarmName": "dev_lg_bulk_main_oldest_message_alarm_6d",
"AlarmDescription": f"Alarm when a message in queue dev-{MOCK_LG_METADATA_SQS_QUEUE} is older than 6 days.",
"NewStateValue": "ALARM",
"StateChangeTime": ALERT_TIME,
"OldStateValue": "OK",
"Trigger": {
"MetricName": "ApproximateAgeOfOldestMessage",
"Namespace": "AWS/SQS",
"StatisticType": "Statistic",
"Statistic": "Maximum",
"Unit": None,
"Dimensions": [
{
"QueueName": f"dev-{MOCK_LG_METADATA_SQS_QUEUE}",
}
],
},
}

MOCK_LAMBDA_ALERT_MESSAGE = {
"AlarmName": "dev-alarm_search_patient_details_handler_error",
"AlarmDescription": "Triggers when an error has occurred in dev_SearchPatientDetailsLambda.",
"AlarmConfigurationUpdatedTimestamp": "2025-04-17T15:08:51.604+0000",
"NewStateValue": "ALARM",
"StateChangeTime": ALERT_TIME,
"OldStateValue": "OK",
"Trigger": {
"MetricName": "Errors",
"Namespace": "AWS/Lambda",
"StatisticType": "Statistic",
"Statistic": "SUM",
"Unit": None,
"Dimensions": [
{
"value": "dev_SearchPatientDetailsLambda",
"name": "FunctionName",
}
],
},
}

MOCK_LAMBDA_ALARM_SNS_ALERT = {
"EventSource": "aws:sns",
"EventVersion": "1.0",
"EventSubscriptionArn": "arn:aws:sns:region:xxxxxx:dev-sns-search_patient_details_alarms-topicxxxxx:xxxxxx",
"Sns": {
"Type": "Notification",
"MessageId": "xxxxxx",
"TopicArn": "arn:aws:sns:region:xxxxxx:dev-sns-search_patient_details_alarms-topicxxxxx",
"Subject": 'ALARM: "dev-alarm_search_patient_details_handler_error"',
"Message": MOCK_LAMBDA_ALERT_MESSAGE,
},
}

MOCK_VIRUS_SCANNER_ALERT_SNS_MESSAGE = {
"Type": "Notification",
"MessageId": "xxxxxx",
"TopicArn": "virus_scanner_topic_arn",
"Subject": "",
"Message": {"id": TEST_UUID, "dateScanned": ALERT_TIME, "result": "Error"},
}
Loading
Loading