NRL 1188 power bi infra

.gitignore

@@ -42,6 +42,9 @@ override.tf.json
 *_override.tf
 *_override.tf.json
 
+# Ignore output of data object
+terraform/account-wide-infrastructure/modules/glue/files/src.zip
+
 # Include override files you do wish to add to version control using negated pattern
 #
 # !example_override.tf

terraform/account-wide-infrastructure/dev/athena.tf

@@ -0,0 +1,5 @@
+module "dev-athena" {
+  source             = "../modules/athena"
+  name_prefix        = "nhsd-nrlf--dev"
+  target_bucket_name = module.dev-glue.target_bucket_name
+}

terraform/account-wide-infrastructure/dev/glue.tf

@@ -0,0 +1,5 @@
+module "dev-glue" {
+  source         = "../modules/glue"
+  name_prefix    = "nhsd-nrlf--dev"
+  python_version = 3
+}

terraform/account-wide-infrastructure/modules/athena/athena.tf

@@ -0,0 +1,31 @@
+resource "aws_athena_database" "reporting-db" {
+  name = var.database
+
+  bucket = var.target_bucket_name
+
+  encryption_configuration {
+    encryption_option = "SSE_KMS"
+    kms_key           = aws_kms_key.athena.arn
+  }
+
+  force_destroy = true
+}
+
+resource "aws_athena_workgroup" "athena" {
+  name = "${var.name_prefix}-athena-wg"
+
+  configuration {
+    enforce_workgroup_configuration    = true
+    publish_cloudwatch_metrics_enabled = true
+
+    result_configuration {
+      output_location = "s3://{aws_s3_bucket.athena.bucket}/output/"
+
+      encryption_configuration {
+        encryption_option = "SSE_KMS"
+        kms_key_arn       = aws_kms_key.athena.arn
+      }
+    }
+  }
+
+}

terraform/account-wide-infrastructure/modules/athena/kms.tf

@@ -0,0 +1,7 @@
+resource "aws_kms_key" "athena" {
+}
+
+resource "aws_kms_alias" "athena" {
+  name          = "alias/${var.name_prefix}-athena"
+  target_key_id = aws_kms_key.athena.key_id
+}

terraform/account-wide-infrastructure/modules/athena/outputs.tf

@@ -0,0 +1,11 @@
+output "workgroup" {
+  value = aws_athena_workgroup.athena
+}
+
+output "bucket" {
+  value = aws_s3_bucket.athena
+}
+
+output "database" {
+  value = aws_athena_database.reporting-db
+}

terraform/account-wide-infrastructure/modules/athena/s3.tf

@@ -0,0 +1,52 @@
+resource "aws_s3_bucket" "athena" {
+  bucket = "${var.name_prefix}-athena"
+}
+
+resource "aws_s3_bucket_policy" "athena" {
+  bucket = "${var.name_prefix}-athena"
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Id      = "athena-policy"
+    Statement = [
+      {
+        Sid    = "HTTPSOnly"
+        Effect = "Deny"
+        Principal = {
+          "AWS" : "*"
+        }
+        Action = "s3:*"
+        Resource = [
+          aws_s3_bucket.athena.arn,
+          "${aws_s3_bucket.athena.arn}/*",
+        ]
+        Condition = {
+          Bool = {
+            "aws:SecureTransport" = "false"
+          }
+        }
+      },
+    ]
+  })
+}
+
+resource "aws_s3_bucket_public_access_block" "athena-public-access-block" {
+  bucket = aws_s3_bucket.athena.id
+
+  block_public_acls       = true
+  block_public_policy     = true
+  ignore_public_acls      = true
+  restrict_public_buckets = true
+}
+
+
+resource "aws_s3_bucket_server_side_encryption_configuration" "athena" {
+  bucket = aws_s3_bucket.athena.bucket
+  rule {
+    apply_server_side_encryption_by_default {
+      sse_algorithm     = "aws:kms"
+      kms_master_key_id = aws_kms_key.athena.arn
+    }
+  }
+
+}

terraform/account-wide-infrastructure/modules/athena/vars.tf

@@ -0,0 +1,13 @@
+variable "database" {
+  description = "What the db will be called"
+  default     = "nrl_reporting"
+}
+
+variable "name_prefix" {
+  type        = string
+  description = "The prefix to apply to all resources in the module."
+}
+
+variable "target_bucket_name" {
+  type = string
+}

terraform/account-wide-infrastructure/modules/glue/glue.tf

@@ -0,0 +1,59 @@
+# Create Glue Data Catalog Database
+resource "aws_glue_catalog_database" "raw_log_database" {
+  name         = "${var.name_prefix}-raw_log"
+  location_uri = "${aws_s3_bucket.source-data-bucket.id}/"
+}
+
+# Create Glue Crawler
+resource "aws_glue_crawler" "raw_log_crawler" {
+  name          = "${var.name_prefix}-raw-log-crawler"
+  database_name = aws_glue_catalog_database.raw_log_database.name
+  role          = aws_iam_role.glue_service_role.name
+  s3_target {
+    path = "${aws_s3_bucket.source-data-bucket.id}/"
+  }
+  schema_change_policy {
+    delete_behavior = "LOG"
+  }
+  configuration = jsonencode({
+    "Version" : 1.0,
+    "Grouping" : {
+      "TableGroupingPolicy" : "CombineCompatibleSchemas"
+    }
+  })
+}
+resource "aws_glue_trigger" "raw_log_trigger" {
+  name = "${var.name_prefix}-org-report-trigger"
+  type = "ON_DEMAND"
+  actions {
+    crawler_name = aws_glue_crawler.raw_log_crawler.name
+  }
+}
+
+resource "aws_glue_job" "glue_job" {
+  name              = "${var.name_prefix}-glue-job"
+  role_arn          = aws_iam_role.glue_service_role.arn
+  description       = "Transfer logs from source to bucket"
+  glue_version      = "4.0"
+  worker_type       = "G.1X"
+  timeout           = 2880
+  max_retries       = 1
+  number_of_workers = 2
+  command {
+    name            = "glueetl"
+    python_version  = var.python_version
+    script_location = "s3://${aws_s3_bucket.code-bucket.id}/main.py"
+  }
+
+  default_arguments = {
+    "--enable-auto-scaling"             = "true"
+    "--enable-continous-cloudwatch-log" = "true"
+    "--datalake-formats"                = "delta"
+    "--source-path"                     = "s3://${aws_s3_bucket.source-data-bucket.id}/" # Specify the source S3 path
+    "--destination-path"                = "s3://${aws_s3_bucket.target-data-bucket.id}/" # Specify the destination S3 path
+    "--job-name"                        = "poc-glue-job"
+    "--enable-continuous-log-filter"    = "true"
+    "--enable-metrics"                  = "true"
+    "--extra-py-files"                  = "s3://${aws_s3_bucket.code-bucket.id}/src.zip"
+  }
+}

terraform/account-wide-infrastructure/modules/glue/iam.tf

@@ -0,0 +1,21 @@
+resource "aws_iam_role" "glue_service_role" {
+  name = "${var.name_prefix}-glue_service_role"
+
+  assume_role_policy = jsonencode({
+    "Version" : "2012-10-17",
+    "Statement" : [
+      {
+        "Effect" : "Allow",
+        "Principal" : {
+          "Service" : "glue.amazonaws.com"
+        },
+        "Action" : "sts:AssumeRole"
+      }
+    ]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "glue_service" {
+  role       = aws_iam_role.glue_service_role.id
+  policy_arn = "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole"
+}

terraform/account-wide-infrastructure/modules/glue/kms.tf

@@ -0,0 +1,7 @@
+resource "aws_kms_key" "glue" {
+}
+
+resource "aws_kms_alias" "glue" {
+  name          = "alias/${var.name_prefix}-glue"
+  target_key_id = aws_kms_key.glue.key_id
+}

terraform/account-wide-infrastructure/modules/glue/outputs.tf

@@ -0,0 +1,13 @@
+output "target_bucket_name" {
+  description = "Name of destination bucket"
+  value       = aws_s3_bucket.target-data-bucket.id
+}
+
+output "source_bucket_name" {
+  description = "Name of source bucket"
+  value       = aws_s3_bucket.source-data-bucket.id
+}
+
+output "glue_crawler_name" {
+  value = "s3//${aws_s3_bucket.source-data-bucket.id}/"
+}