NRL-1386 Add infra and config for prod dashboards

terraform/account-wide-infrastructure/README.md

@@ -134,9 +134,11 @@ If deploying the EC2 set up to a new environment, these steps need to be followe
 aws ssm start-session --target <AMI> --document-name AWS-StartPortForwardingSession --parameters "localPortNumber=13389,portNumber=3389"
 ```
 
-2. Install Athena ODBC driver and Power BI personal on premises gateway
-3. Configure ODBC driver to connect to relevant Athena instance and log in to the gateway using NHS email
-4. Log into power bi and test the refresh on the relevant data sources
+2. Install Athena ODBC driver and Power BI standard on premises gateway
+3. Configure ODBC driver to connect to relevant Athena instance
+4. Log in to the gateway using NHS email, name the cluster to nhsd-nrlf-{env}--reporting-gw
+5. Log on to power bi, navigate to Manage Connections and Gateways in settings and set up Athena connector with authentication method: Anonymous and privacy level: Private
+6. Set dataset to point to this gateway, define schedule as needed
 
 ## Tear down account wide resources
 

terraform/account-wide-infrastructure/dev/athena.tf

@@ -1,5 +1,7 @@
 module "dev-athena" {
+  count              = var.enable_reporting ? 1 : 0
   source             = "../modules/athena"
   name_prefix        = "nhsd-nrlf--dev"
   target_bucket_name = module.dev-glue.target_bucket_name
+  glue_database      = module.dev-glue.glue_database
 }

terraform/account-wide-infrastructure/dev/ec2.tf

@@ -1,4 +1,5 @@
 module "vpc" {
+  count                          = var.enable_reporting && var.enable_powerbi_auto_push ? 1 : 0
   source                         = "../modules/vpc"
   vpc_cidr_block                 = var.vpc_cidr_block
   enable_dns_hostnames           = var.enable_dns_hostnames
@@ -8,16 +9,17 @@ module "vpc" {
   name_prefix                    = "nhsd-nrlf--dev"
 }
 
-module "powerbi_gw_instance_v2" {
-  source             = "../modules/ec2"
-  use_custom_ami     = true
-  instance_type      = var.instance_type
-  name_prefix        = "nhsd-nrlf--dev-powerbi-gw-v2"
+module "powerbi_gw_instance" {
+  count              = var.enable_reporting && var.enable_powerbi_auto_push ? 1 : 0
+  source             = "../modules/powerbi-gw-ec2"
+  use_custom_ami     = var.use_powerbi_gw_custom_ami
+  instance_type      = var.powerbi_gw_instance_type
+  name_prefix        = "nhsd-nrlf--dev-powerbi-gw"
   target_bucket_arn  = module.dev-glue.target_bucket_arn
   glue_kms_key_arn   = module.dev-glue.aws_kms_key_arn
-  athena_kms_key_arn = module.dev-athena.kms_key_arn
-  athena_bucket_arn  = module.dev-athena.bucket_arn
+  athena_kms_key_arn = module.dev-athena[0].kms_key_arn
+  athena_bucket_arn  = module.dev-athena[0].bucket_arn
 
-  subnet_id       = module.vpc.private_subnet_id
-  security_groups = [module.vpc.powerbi_gw_security_group_id]
+  subnet_id       = module.vpc[0].private_subnet_id
+  security_groups = [module.vpc[0].powerbi_gw_security_group_id]
 }

terraform/account-wide-infrastructure/dev/glue.tf

@@ -1,4 +1,5 @@
 module "dev-glue" {
+  is_enabled     = var.enable_reporting
   source         = "../modules/glue"
   name_prefix    = "nhsd-nrlf--dev"
   python_version = 3

terraform/account-wide-infrastructure/dev/vars.tf

@@ -14,6 +14,12 @@ variable "devsandbox_api_domain_name" {
   default     = "dev-sandbox.api.record-locator.dev.national.nhs.uk"
 }
 
+variable "enable_reporting" {
+  type        = bool
+  description = "Enable account-wide reporting processes in the dev account"
+  default     = true
+}
+
 variable "aws_azs" {
   type        = string
   description = "AWS Availability Zones"
@@ -44,14 +50,20 @@ variable "vpc_private_subnets_cidr_block" {
   default     = "10.0.1.0/24"
 }
 
-variable "instance_type" {
+variable "enable_powerbi_auto_push" {
+  type        = bool
+  description = "Enable automatic pushing of info into PowerBI"
+  default     = true
+}
+
+variable "powerbi_gw_instance_type" {
   type        = string
-  description = "Type for EC2 Instance"
+  description = "Type for PowerBI GW EC2 Instance"
   default     = "t2.micro"
 }
 
-variable "use_custom_ami" {
+variable "use_powerbi_gw_custom_ami" {
   type        = bool
-  description = "Use custom image"
-  default     = false
+  description = "Use custom image for PowerBI GW instance"
+  default     = true
 }

terraform/account-wide-infrastructure/modules/athena/athena.tf

@@ -16,3 +16,17 @@ resource "aws_athena_workgroup" "athena" {
   }
 
 }
+
+resource "aws_athena_named_query" "rep_consumer" {
+  name      = "rep_consumer"
+  workgroup = aws_athena_workgroup.athena.id
+  database  = var.glue_database
+  query     = file("${path.module}/sql/rep_consumer.sql")
+}
+
+resource "aws_athena_named_query" "rep_producer" {
+  name      = "rep_producer"
+  workgroup = aws_athena_workgroup.athena.id
+  database  = var.glue_database
+  query     = file("${path.module}/sql/rep_producer.sql")
+}

terraform/account-wide-infrastructure/modules/athena/sql/rep_consumer.sql

@@ -0,0 +1,118 @@
+CREATE OR REPLACE VIEW "rep_consumer" AS
+WITH
+  cc AS (
+   SELECT
+     time
+   , event_timestamp
+   , date
+   , host
+   , event_log_reference
+   , event_level
+   , event_location
+   , event_message
+   , event_service
+   , event_function_request_id
+   , event_correlation_id
+   , event_xray_trace_id
+   , event_pointer_types
+   , COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods
+   FROM
+     consumer_countdocumentreference
+)
+, cr AS (
+   SELECT
+     time
+   , event_timestamp
+   , date
+   , host
+   , event_log_reference
+   , event_level
+   , event_location
+   , event_message
+   , event_service
+   , event_function_request_id
+   , event_correlation_id
+   , event_xray_trace_id
+   , event_pointer_types
+   , COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods
+   FROM
+     consumer_readdocumentreference
+)
+, cs AS (
+   SELECT
+     time
+   , event_timestamp
+   , date
+   , host
+   , event_log_reference
+   , event_level
+   , event_location
+   , event_message
+   , event_service
+   , event_function_request_id
+   , event_correlation_id
+   , event_xray_trace_id
+   , event_pointer_types
+   , COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods
+   FROM
+     consumer_searchdocumentreference
+)
+, csp AS (
+   SELECT
+     time
+   , event_timestamp
+   , date
+   , host
+   , event_log_reference
+   , event_level
+   , event_location
+   , event_message
+   , event_service
+   , event_function_request_id
+   , event_correlation_id
+   , event_xray_trace_id
+   , event_pointer_types
+   , COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods
+   FROM
+     consumer_searchpostdocumentreference
+)
+, base AS (
+   SELECT *
+   FROM
+     cc
+UNION    SELECT *
+   FROM
+     cr
+UNION    SELECT *
+   FROM
+     cs
+UNION    SELECT *
+   FROM
+     csp
+)
+, ods_codes AS (
+   SELECT DISTINCT
+     user_ods
+   , event_xray_trace_id
+   FROM
+     base
+   WHERE (user_ods IS NOT NULL)
+)
+SELECT
+  time
+, event_timestamp
+, date
+, host
+, event_log_reference
+, event_level
+, event_location
+, event_message
+, event_service
+, event_function_request_id
+, b.event_correlation_id
+, b.event_xray_trace_id
+, event_pointer_types
+, oc.user_ods
+FROM
+  (base b
+LEFT JOIN ods_codes oc ON (b.event_xray_trace_id = oc.event_xray_trace_id))