From cb4822a444330161831811fdce7d7c1e03f0152c Mon Sep 17 00:00:00 2001 From: Gareth Allan <157592212+gareth-allan@users.noreply.github.com> Date: Tue, 10 Feb 2026 12:00:21 +0000 Subject: [PATCH 1/2] CCM-13304: Added a helper to trigger Glue table refresh --- package-lock.json | 301 +++++++++++------- .../playwright/constants/backend-constants.ts | 7 + .../report-generator.component.spec.ts | 28 ++ tests/playwright/helpers/athena-helpers.ts | 61 ++++ tests/playwright/package.json | 1 + 5 files changed, 281 insertions(+), 117 deletions(-) create mode 100644 tests/playwright/digital-letters-component-tests/report-generator.component.spec.ts create mode 100644 tests/playwright/helpers/athena-helpers.ts diff --git a/package-lock.json b/package-lock.json index d70f8efd..0da092e5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4841,6 +4841,72 @@ "node": ">=14.0.0" } }, + "node_modules/@aws-sdk/client-athena": { + "version": "3.986.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-athena/-/client-athena-3.986.0.tgz", + "integrity": "sha512-mZWMvRzr6H9yQ5qKNSe2Fc1lfCNuhdhU+9eV0/qsNyT1t6r1yIfual/o0nBWTNVD9yt1k7J+tjxAY2ElyANAQw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "^3.973.7", + "@aws-sdk/credential-provider-node": "^3.972.6", + "@aws-sdk/middleware-host-header": "^3.972.3", + "@aws-sdk/middleware-logger": "^3.972.3", + "@aws-sdk/middleware-recursion-detection": "^3.972.3", + "@aws-sdk/middleware-user-agent": "^3.972.7", + "@aws-sdk/region-config-resolver": "^3.972.3", + "@aws-sdk/types": "^3.973.1", + "@aws-sdk/util-endpoints": "3.986.0", + "@aws-sdk/util-user-agent-browser": "^3.972.3", + "@aws-sdk/util-user-agent-node": "^3.972.5", + "@smithy/config-resolver": "^4.4.6", + "@smithy/core": "^3.22.1", + "@smithy/fetch-http-handler": "^5.3.9", + "@smithy/hash-node": "^4.2.8", + "@smithy/invalid-dependency": "^4.2.8", + "@smithy/middleware-content-length": "^4.2.8", + "@smithy/middleware-endpoint": "^4.4.13", + "@smithy/middleware-retry": "^4.4.30", + "@smithy/middleware-serde": "^4.2.9", + "@smithy/middleware-stack": "^4.2.8", + "@smithy/node-config-provider": "^4.3.8", + "@smithy/node-http-handler": "^4.4.9", + "@smithy/protocol-http": "^5.3.8", + "@smithy/smithy-client": "^4.11.2", + "@smithy/types": "^4.12.0", + "@smithy/url-parser": "^4.2.8", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-body-length-browser": "^4.2.0", + "@smithy/util-body-length-node": "^4.2.1", + "@smithy/util-defaults-mode-browser": "^4.3.29", + "@smithy/util-defaults-mode-node": "^4.2.32", + "@smithy/util-endpoints": "^3.2.8", + "@smithy/util-middleware": "^4.2.8", + "@smithy/util-retry": "^4.2.8", + "@smithy/util-utf8": "^4.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/client-athena/node_modules/@aws-sdk/util-endpoints": { + "version": "3.986.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.986.0.tgz", + "integrity": "sha512-Mqi79L38qi1gCG3adlVdbNrSxvcm1IPDLiJPA3OBypY5ewxUyWbaA3DD4goG+EwET6LSFgZJcRSIh6KBNpP5pA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.973.1", + "@smithy/types": "^4.12.0", + "@smithy/url-parser": "^4.2.8", + "@smithy/util-endpoints": "^3.2.8", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@aws-sdk/client-cloudwatch-logs": { "version": "3.981.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-cloudwatch-logs/-/client-cloudwatch-logs-3.981.0.tgz", @@ -5313,44 +5379,44 @@ } }, "node_modules/@aws-sdk/client-sso": { - "version": "3.980.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.980.0.tgz", - "integrity": "sha512-AhNXQaJ46C1I+lQ+6Kj+L24il5K9lqqIanJd8lMszPmP7bLnmX0wTKK0dxywcvrLdij3zhWttjAKEBNgLtS8/A==", + "version": "3.985.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.985.0.tgz", + "integrity": "sha512-81J8iE8MuXhdbMfIz4sWFj64Pe41bFi/uqqmqOC5SlGv+kwoyLsyKS/rH2tW2t5buih4vTUxskRjxlqikTD4oQ==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.973.5", + "@aws-sdk/core": "^3.973.7", "@aws-sdk/middleware-host-header": "^3.972.3", "@aws-sdk/middleware-logger": "^3.972.3", "@aws-sdk/middleware-recursion-detection": "^3.972.3", - "@aws-sdk/middleware-user-agent": "^3.972.5", + "@aws-sdk/middleware-user-agent": "^3.972.7", "@aws-sdk/region-config-resolver": "^3.972.3", "@aws-sdk/types": "^3.973.1", - "@aws-sdk/util-endpoints": "3.980.0", + "@aws-sdk/util-endpoints": "3.985.0", "@aws-sdk/util-user-agent-browser": "^3.972.3", - "@aws-sdk/util-user-agent-node": "^3.972.3", + "@aws-sdk/util-user-agent-node": "^3.972.5", "@smithy/config-resolver": "^4.4.6", - "@smithy/core": "^3.22.0", + "@smithy/core": "^3.22.1", "@smithy/fetch-http-handler": "^5.3.9", "@smithy/hash-node": "^4.2.8", "@smithy/invalid-dependency": "^4.2.8", "@smithy/middleware-content-length": "^4.2.8", - "@smithy/middleware-endpoint": "^4.4.12", - "@smithy/middleware-retry": "^4.4.29", + "@smithy/middleware-endpoint": "^4.4.13", + "@smithy/middleware-retry": "^4.4.30", "@smithy/middleware-serde": "^4.2.9", "@smithy/middleware-stack": "^4.2.8", "@smithy/node-config-provider": "^4.3.8", - "@smithy/node-http-handler": "^4.4.8", + "@smithy/node-http-handler": "^4.4.9", "@smithy/protocol-http": "^5.3.8", - "@smithy/smithy-client": "^4.11.1", + "@smithy/smithy-client": "^4.11.2", "@smithy/types": "^4.12.0", "@smithy/url-parser": "^4.2.8", "@smithy/util-base64": "^4.3.0", "@smithy/util-body-length-browser": "^4.2.0", "@smithy/util-body-length-node": "^4.2.1", - "@smithy/util-defaults-mode-browser": "^4.3.28", - "@smithy/util-defaults-mode-node": "^4.2.31", + "@smithy/util-defaults-mode-browser": "^4.3.29", + "@smithy/util-defaults-mode-node": "^4.2.32", "@smithy/util-endpoints": "^3.2.8", "@smithy/util-middleware": "^4.2.8", "@smithy/util-retry": "^4.2.8", @@ -5362,9 +5428,9 @@ } }, "node_modules/@aws-sdk/client-sso/node_modules/@aws-sdk/util-endpoints": { - "version": "3.980.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.980.0.tgz", - "integrity": "sha512-AjKBNEc+rjOZQE1HwcD9aCELqg1GmUj1rtICKuY8cgwB73xJ4U/kNyqKKpN2k9emGqlfDY2D8itIp/vDc6OKpw==", + "version": "3.985.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.985.0.tgz", + "integrity": "sha512-vth7UfGSUR3ljvaq8V4Rc62FsM7GUTH/myxPWkaEgOrprz1/Pc72EgTXxj+cPPPDAfHFIpjhkB7T7Td0RJx+BA==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/types": "^3.973.1", @@ -5378,19 +5444,19 @@ } }, "node_modules/@aws-sdk/core": { - "version": "3.973.5", - "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.973.5.tgz", - "integrity": "sha512-IMM7xGfLGW6lMvubsA4j6BHU5FPgGAxoQ/NA63KqNLMwTS+PeMBcx8DPHL12Vg6yqOZnqok9Mu4H2BdQyq7gSA==", + "version": "3.973.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.973.7.tgz", + "integrity": "sha512-wNZZQQNlJ+hzD49cKdo+PY6rsTDElO8yDImnrI69p2PLBa7QomeUKAJWYp9xnaR38nlHqWhMHZuYLCQ3oSX+xg==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/types": "^3.973.1", - "@aws-sdk/xml-builder": "^3.972.2", - "@smithy/core": "^3.22.0", + "@aws-sdk/xml-builder": "^3.972.4", + "@smithy/core": "^3.22.1", "@smithy/node-config-provider": "^4.3.8", "@smithy/property-provider": "^4.2.8", "@smithy/protocol-http": "^5.3.8", "@smithy/signature-v4": "^5.3.8", - "@smithy/smithy-client": "^4.11.1", + "@smithy/smithy-client": "^4.11.2", "@smithy/types": "^4.12.0", "@smithy/util-base64": "^4.3.0", "@smithy/util-middleware": "^4.2.8", @@ -5415,12 +5481,12 @@ } }, "node_modules/@aws-sdk/credential-provider-env": { - "version": "3.972.3", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.3.tgz", - "integrity": "sha512-OBYNY4xQPq7Rx+oOhtyuyO0AQvdJSpXRg7JuPNBJH4a1XXIzJQl4UHQTPKZKwfJXmYLpv4+OkcFen4LYmDPd3g==", + "version": "3.972.5", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.5.tgz", + "integrity": "sha512-LxJ9PEO4gKPXzkufvIESUysykPIdrV7+Ocb9yAhbhJLE4TiAYqbCVUE+VuKP1leGR1bBfjWjYgSV5MxprlX3mQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.5", + "@aws-sdk/core": "^3.973.7", "@aws-sdk/types": "^3.973.1", "@smithy/property-provider": "^4.2.8", "@smithy/types": "^4.12.0", @@ -5431,20 +5497,20 @@ } }, "node_modules/@aws-sdk/credential-provider-http": { - "version": "3.972.5", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.5.tgz", - "integrity": "sha512-GpvBgEmSZPvlDekd26Zi+XsI27Qz7y0utUx0g2fSTSiDzhnd1FSa1owuodxR0BcUKNL7U2cOVhhDxgZ4iSoPVg==", + "version": "3.972.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.7.tgz", + "integrity": "sha512-L2uOGtvp2x3bTcxFTpSM+GkwFIPd8pHfGWO1764icMbo7e5xJh0nfhx1UwkXLnwvocTNEf8A7jISZLYjUSNaTg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.5", + "@aws-sdk/core": "^3.973.7", "@aws-sdk/types": "^3.973.1", "@smithy/fetch-http-handler": "^5.3.9", - "@smithy/node-http-handler": "^4.4.8", + "@smithy/node-http-handler": "^4.4.9", "@smithy/property-provider": "^4.2.8", "@smithy/protocol-http": "^5.3.8", - "@smithy/smithy-client": "^4.11.1", + "@smithy/smithy-client": "^4.11.2", "@smithy/types": "^4.12.0", - "@smithy/util-stream": "^4.5.10", + "@smithy/util-stream": "^4.5.11", "tslib": "^2.6.2" }, "engines": { @@ -5452,19 +5518,19 @@ } }, "node_modules/@aws-sdk/credential-provider-ini": { - "version": "3.972.3", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.972.3.tgz", - "integrity": "sha512-rMQAIxstP7cLgYfsRGrGOlpyMl0l8JL2mcke3dsIPLWke05zKOFyR7yoJzWCsI/QiIxjRbxpvPiAeKEA6CoYkg==", + "version": "3.972.5", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.972.5.tgz", + "integrity": "sha512-SdDTYE6jkARzOeL7+kudMIM4DaFnP5dZVeatzw849k4bSXDdErDS188bgeNzc/RA2WGrlEpsqHUKP6G7sVXhZg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.5", - "@aws-sdk/credential-provider-env": "^3.972.3", - "@aws-sdk/credential-provider-http": "^3.972.5", - "@aws-sdk/credential-provider-login": "^3.972.3", - "@aws-sdk/credential-provider-process": "^3.972.3", - "@aws-sdk/credential-provider-sso": "^3.972.3", - "@aws-sdk/credential-provider-web-identity": "^3.972.3", - "@aws-sdk/nested-clients": "3.980.0", + "@aws-sdk/core": "^3.973.7", + "@aws-sdk/credential-provider-env": "^3.972.5", + "@aws-sdk/credential-provider-http": "^3.972.7", + "@aws-sdk/credential-provider-login": "^3.972.5", + "@aws-sdk/credential-provider-process": "^3.972.5", + "@aws-sdk/credential-provider-sso": "^3.972.5", + "@aws-sdk/credential-provider-web-identity": "^3.972.5", + "@aws-sdk/nested-clients": "3.985.0", "@aws-sdk/types": "^3.973.1", "@smithy/credential-provider-imds": "^4.2.8", "@smithy/property-provider": "^4.2.8", @@ -5477,13 +5543,13 @@ } }, "node_modules/@aws-sdk/credential-provider-login": { - "version": "3.972.3", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.3.tgz", - "integrity": "sha512-Gc3O91iVvA47kp2CLIXOwuo5ffo1cIpmmyIewcYjAcvurdFHQ8YdcBe1KHidnbbBO4/ZtywGBACsAX5vr3UdoA==", + "version": "3.972.5", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.5.tgz", + "integrity": "sha512-uYq1ILyTSI6ZDCMY5+vUsRM0SOCVI7kaW4wBrehVVkhAxC6y+e9rvGtnoZqCOWL1gKjTMouvsf4Ilhc5NCg1Aw==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.5", - "@aws-sdk/nested-clients": "3.980.0", + "@aws-sdk/core": "^3.973.7", + "@aws-sdk/nested-clients": "3.985.0", "@aws-sdk/types": "^3.973.1", "@smithy/property-provider": "^4.2.8", "@smithy/protocol-http": "^5.3.8", @@ -5496,17 +5562,17 @@ } }, "node_modules/@aws-sdk/credential-provider-node": { - "version": "3.972.4", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.4.tgz", - "integrity": "sha512-UwerdzosMSY7V5oIZm3NsMDZPv2aSVzSkZxYxIOWHBeKTZlUqW7XpHtJMZ4PZpJ+HMRhgP+MDGQx4THndgqJfQ==", + "version": "3.972.6", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.6.tgz", + "integrity": "sha512-DZ3CnAAtSVtVz+G+ogqecaErMLgzph4JH5nYbHoBMgBkwTUV+SUcjsjOJwdBJTHu3Dm6l5LBYekZoU2nDqQk2A==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/credential-provider-env": "^3.972.3", - "@aws-sdk/credential-provider-http": "^3.972.5", - "@aws-sdk/credential-provider-ini": "^3.972.3", - "@aws-sdk/credential-provider-process": "^3.972.3", - "@aws-sdk/credential-provider-sso": "^3.972.3", - "@aws-sdk/credential-provider-web-identity": "^3.972.3", + "@aws-sdk/credential-provider-env": "^3.972.5", + "@aws-sdk/credential-provider-http": "^3.972.7", + "@aws-sdk/credential-provider-ini": "^3.972.5", + "@aws-sdk/credential-provider-process": "^3.972.5", + "@aws-sdk/credential-provider-sso": "^3.972.5", + "@aws-sdk/credential-provider-web-identity": "^3.972.5", "@aws-sdk/types": "^3.973.1", "@smithy/credential-provider-imds": "^4.2.8", "@smithy/property-provider": "^4.2.8", @@ -5519,12 +5585,12 @@ } }, "node_modules/@aws-sdk/credential-provider-process": { - "version": "3.972.3", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.3.tgz", - "integrity": "sha512-xkSY7zjRqeVc6TXK2xr3z1bTLm0wD8cj3lAkproRGaO4Ku7dPlKy843YKnHrUOUzOnMezdZ4xtmFc0eKIDTo2w==", + "version": "3.972.5", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.5.tgz", + "integrity": "sha512-HDKF3mVbLnuqGg6dMnzBf1VUOywE12/N286msI9YaK9mEIzdsGCtLTvrDhe3Up0R9/hGFbB+9l21/TwF5L1C6g==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.5", + "@aws-sdk/core": "^3.973.7", "@aws-sdk/types": "^3.973.1", "@smithy/property-provider": "^4.2.8", "@smithy/shared-ini-file-loader": "^4.4.3", @@ -5536,14 +5602,14 @@ } }, "node_modules/@aws-sdk/credential-provider-sso": { - "version": "3.972.3", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.972.3.tgz", - "integrity": "sha512-8Ww3F5Ngk8dZ6JPL/V5LhCU1BwMfQd3tLdoEuzaewX8FdnT633tPr+KTHySz9FK7fFPcz5qG3R5edVEhWQD4AA==", + "version": "3.972.5", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.972.5.tgz", + "integrity": "sha512-8urj3AoeNeQisjMmMBhFeiY2gxt6/7wQQbEGun0YV/OaOOiXrIudTIEYF8ZfD+NQI6X1FY5AkRsx6O/CaGiybA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/client-sso": "3.980.0", - "@aws-sdk/core": "^3.973.5", - "@aws-sdk/token-providers": "3.980.0", + "@aws-sdk/client-sso": "3.985.0", + "@aws-sdk/core": "^3.973.7", + "@aws-sdk/token-providers": "3.985.0", "@aws-sdk/types": "^3.973.1", "@smithy/property-provider": "^4.2.8", "@smithy/shared-ini-file-loader": "^4.4.3", @@ -5555,13 +5621,13 @@ } }, "node_modules/@aws-sdk/credential-provider-web-identity": { - "version": "3.972.3", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.3.tgz", - "integrity": "sha512-62VufdcH5rRfiRKZRcf1wVbbt/1jAntMj1+J0qAd+r5pQRg2t0/P9/Rz16B1o5/0Se9lVL506LRjrhIJAhYBfA==", + "version": "3.972.5", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.5.tgz", + "integrity": "sha512-OK3cULuJl6c+RcDZfPpaK5o3deTOnKZbxm7pzhFNGA3fI2hF9yDih17fGRazJzGGWaDVlR9ejZrpDef4DJCEsw==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.5", - "@aws-sdk/nested-clients": "3.980.0", + "@aws-sdk/core": "^3.973.7", + "@aws-sdk/nested-clients": "3.985.0", "@aws-sdk/types": "^3.973.1", "@smithy/property-provider": "^4.2.8", "@smithy/shared-ini-file-loader": "^4.4.3", @@ -5817,15 +5883,15 @@ } }, "node_modules/@aws-sdk/middleware-user-agent": { - "version": "3.972.5", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.972.5.tgz", - "integrity": "sha512-TVZQ6PWPwQbahUI8V+Er+gS41ctIawcI/uMNmQtQ7RMcg3JYn6gyKAFKUb3HFYx2OjYlx1u11sETSwwEUxVHTg==", + "version": "3.972.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.972.7.tgz", + "integrity": "sha512-HUD+geASjXSCyL/DHPQc/Ua7JhldTcIglVAoCV8kiVm99IaFSlAbTvEnyhZwdE6bdFyTL+uIaWLaCFSRsglZBQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.5", + "@aws-sdk/core": "^3.973.7", "@aws-sdk/types": "^3.973.1", - "@aws-sdk/util-endpoints": "3.980.0", - "@smithy/core": "^3.22.0", + "@aws-sdk/util-endpoints": "3.985.0", + "@smithy/core": "^3.22.1", "@smithy/protocol-http": "^5.3.8", "@smithy/types": "^4.12.0", "tslib": "^2.6.2" @@ -5835,9 +5901,9 @@ } }, "node_modules/@aws-sdk/middleware-user-agent/node_modules/@aws-sdk/util-endpoints": { - "version": "3.980.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.980.0.tgz", - "integrity": "sha512-AjKBNEc+rjOZQE1HwcD9aCELqg1GmUj1rtICKuY8cgwB73xJ4U/kNyqKKpN2k9emGqlfDY2D8itIp/vDc6OKpw==", + "version": "3.985.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.985.0.tgz", + "integrity": "sha512-vth7UfGSUR3ljvaq8V4Rc62FsM7GUTH/myxPWkaEgOrprz1/Pc72EgTXxj+cPPPDAfHFIpjhkB7T7Td0RJx+BA==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/types": "^3.973.1", @@ -5851,44 +5917,44 @@ } }, "node_modules/@aws-sdk/nested-clients": { - "version": "3.980.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.980.0.tgz", - "integrity": "sha512-/dONY5xc5/CCKzOqHZCTidtAR4lJXWkGefXvTRKdSKMGaYbbKsxDckisd6GfnvPSLxWtvQzwgRGRutMRoYUApQ==", + "version": "3.985.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.985.0.tgz", + "integrity": "sha512-TsWwKzb/2WHafAY0CE7uXgLj0FmnkBTgfioG9HO+7z/zCPcl1+YU+i7dW4o0y+aFxFgxTMG+ExBQpqT/k2ao8g==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.973.5", + "@aws-sdk/core": "^3.973.7", "@aws-sdk/middleware-host-header": "^3.972.3", "@aws-sdk/middleware-logger": "^3.972.3", "@aws-sdk/middleware-recursion-detection": "^3.972.3", - "@aws-sdk/middleware-user-agent": "^3.972.5", + "@aws-sdk/middleware-user-agent": "^3.972.7", "@aws-sdk/region-config-resolver": "^3.972.3", "@aws-sdk/types": "^3.973.1", - "@aws-sdk/util-endpoints": "3.980.0", + "@aws-sdk/util-endpoints": "3.985.0", "@aws-sdk/util-user-agent-browser": "^3.972.3", - "@aws-sdk/util-user-agent-node": "^3.972.3", + "@aws-sdk/util-user-agent-node": "^3.972.5", "@smithy/config-resolver": "^4.4.6", - "@smithy/core": "^3.22.0", + "@smithy/core": "^3.22.1", "@smithy/fetch-http-handler": "^5.3.9", "@smithy/hash-node": "^4.2.8", "@smithy/invalid-dependency": "^4.2.8", "@smithy/middleware-content-length": "^4.2.8", - "@smithy/middleware-endpoint": "^4.4.12", - "@smithy/middleware-retry": "^4.4.29", + "@smithy/middleware-endpoint": "^4.4.13", + "@smithy/middleware-retry": "^4.4.30", "@smithy/middleware-serde": "^4.2.9", "@smithy/middleware-stack": "^4.2.8", "@smithy/node-config-provider": "^4.3.8", - "@smithy/node-http-handler": "^4.4.8", + "@smithy/node-http-handler": "^4.4.9", "@smithy/protocol-http": "^5.3.8", - "@smithy/smithy-client": "^4.11.1", + "@smithy/smithy-client": "^4.11.2", "@smithy/types": "^4.12.0", "@smithy/url-parser": "^4.2.8", "@smithy/util-base64": "^4.3.0", "@smithy/util-body-length-browser": "^4.2.0", "@smithy/util-body-length-node": "^4.2.1", - "@smithy/util-defaults-mode-browser": "^4.3.28", - "@smithy/util-defaults-mode-node": "^4.2.31", + "@smithy/util-defaults-mode-browser": "^4.3.29", + "@smithy/util-defaults-mode-node": "^4.2.32", "@smithy/util-endpoints": "^3.2.8", "@smithy/util-middleware": "^4.2.8", "@smithy/util-retry": "^4.2.8", @@ -5900,9 +5966,9 @@ } }, "node_modules/@aws-sdk/nested-clients/node_modules/@aws-sdk/util-endpoints": { - "version": "3.980.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.980.0.tgz", - "integrity": "sha512-AjKBNEc+rjOZQE1HwcD9aCELqg1GmUj1rtICKuY8cgwB73xJ4U/kNyqKKpN2k9emGqlfDY2D8itIp/vDc6OKpw==", + "version": "3.985.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.985.0.tgz", + "integrity": "sha512-vth7UfGSUR3ljvaq8V4Rc62FsM7GUTH/myxPWkaEgOrprz1/Pc72EgTXxj+cPPPDAfHFIpjhkB7T7Td0RJx+BA==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/types": "^3.973.1", @@ -5949,13 +6015,13 @@ } }, "node_modules/@aws-sdk/token-providers": { - "version": "3.980.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.980.0.tgz", - "integrity": "sha512-1nFileg1wAgDmieRoj9dOawgr2hhlh7xdvcH57b1NnqfPaVlcqVJyPc6k3TLDUFPY69eEwNxdGue/0wIz58vjA==", + "version": "3.985.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.985.0.tgz", + "integrity": "sha512-+hwpHZyEq8k+9JL2PkE60V93v2kNhUIv7STFt+EAez1UJsJOQDhc5LpzEX66pNjclI5OTwBROs/DhJjC/BtMjQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.5", - "@aws-sdk/nested-clients": "3.980.0", + "@aws-sdk/core": "^3.973.7", + "@aws-sdk/nested-clients": "3.985.0", "@aws-sdk/types": "^3.973.1", "@smithy/property-provider": "^4.2.8", "@smithy/shared-ini-file-loader": "^4.4.3", @@ -6047,12 +6113,12 @@ } }, "node_modules/@aws-sdk/util-user-agent-node": { - "version": "3.972.3", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.972.3.tgz", - "integrity": "sha512-gqG+02/lXQtO0j3US6EVnxtwwoXQC5l2qkhLCrqUrqdtcQxV7FDMbm9wLjKqoronSHyELGTjbFKK/xV5q1bZNA==", + "version": "3.972.5", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.972.5.tgz", + "integrity": "sha512-GsUDF+rXyxDZkkJxUsDxnA67FG+kc5W1dnloCFLl6fWzceevsCYzJpASBzT+BPjwUgREE6FngfJYYYMQUY5fZQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/middleware-user-agent": "^3.972.5", + "@aws-sdk/middleware-user-agent": "^3.972.7", "@aws-sdk/types": "^3.973.1", "@smithy/node-config-provider": "^4.3.8", "@smithy/types": "^4.12.0", @@ -6071,9 +6137,9 @@ } }, "node_modules/@aws-sdk/xml-builder": { - "version": "3.972.3", - "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.3.tgz", - "integrity": "sha512-bCk63RsBNCWW4tt5atv5Sbrh+3J3e8YzgyF6aZb1JeXcdzG4k5SlPLeTMFOIXFuuFHIwgphUhn4i3uS/q49eww==", + "version": "3.972.4", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.4.tgz", + "integrity": "sha512-0zJ05ANfYqI6+rGqj8samZBFod0dPPousBjLEqg8WdxSgbMAkRgLyn81lP215Do0rFJ/17LIXwr7q0yK24mP6Q==", "license": "Apache-2.0", "dependencies": { "@smithy/types": "^4.12.0", @@ -11620,13 +11686,13 @@ } }, "node_modules/axios": { - "version": "1.13.4", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.13.4.tgz", - "integrity": "sha512-1wVkUaAO6WyaYtCkcYCOx12ZgpGf9Zif+qXa4n+oYzK558YryKqiL6UWwd5DqiH3VRW0GYhTZQ/vlgJrCoNQlg==", + "version": "1.13.5", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.13.5.tgz", + "integrity": "sha512-cz4ur7Vb0xS4/KUN0tPWe44eqxrIu31me+fbang3ijiNscE129POzipJJA6zniq2C/Z6sJCjMimjS8Lc/GAs8Q==", "license": "MIT", "dependencies": { - "follow-redirects": "^1.15.6", - "form-data": "^4.0.4", + "follow-redirects": "^1.15.11", + "form-data": "^4.0.5", "proxy-from-env": "^1.1.0" } }, @@ -24509,6 +24575,7 @@ "name": "nhs-notify-digital-letters-integration-tests", "version": "0.0.1", "dependencies": { + "@aws-sdk/client-athena": "^3.900.0", "@aws-sdk/client-cloudwatch-logs": "^3.900.0", "@aws-sdk/client-dynamodb": "^3.900.0", "@aws-sdk/client-lambda": "^3.900.0", diff --git a/tests/playwright/constants/backend-constants.ts b/tests/playwright/constants/backend-constants.ts index 1edb7f02..574f8059 100644 --- a/tests/playwright/constants/backend-constants.ts +++ b/tests/playwright/constants/backend-constants.ts @@ -58,3 +58,10 @@ export const FILE_SCANNER_LAMBDA_LOG_GROUP_NAME = `/aws/lambda/${CSI}-file-scann export const PRINT_STATUS_HANDLER_LAMBDA_LOG_GROUP_NAME = `/aws/lambda/${CSI}-print-status-handler`; export const PRINT_ANALYSER_LAMBDA_LOG_GROUP_NAME = `/aws/lambda/${CSI}-print-analyser`; export const MOVE_SCANNED_FILES_LAMBDA_LOG_GROUP_NAME = `/aws/lambda/${CSI}-move-scanned-files`; + +// Glue +export const GLUE_DATABASE_NAME = `${CSI}-reporting`; +export const GLUE_TABLE_NAME = 'event_record'; + +// Athena +export const ATHENA_WORKGROUP_NAME = CSI; diff --git a/tests/playwright/digital-letters-component-tests/report-generator.component.spec.ts b/tests/playwright/digital-letters-component-tests/report-generator.component.spec.ts new file mode 100644 index 00000000..22c38326 --- /dev/null +++ b/tests/playwright/digital-letters-component-tests/report-generator.component.spec.ts @@ -0,0 +1,28 @@ +import { expect, test } from '@playwright/test'; +import { + ATHENA_WORKGROUP_NAME, + GLUE_DATABASE_NAME, + GLUE_TABLE_NAME, +} from 'constants/backend-constants'; +import { + QueryExecutionState, + getQueryState, + triggerTableMetadataRefresh, +} from 'helpers/athena-helpers'; +import expectToPassEventually from 'helpers/expectations'; + +test.describe('Digital Letters - Report Generator', () => { + test('should refresh data in the Glue table', async () => { + const refreshQueryExecutionId = await triggerTableMetadataRefresh( + GLUE_DATABASE_NAME, + GLUE_TABLE_NAME, + ATHENA_WORKGROUP_NAME, + ); + + await expectToPassEventually(async () => { + const refreshQueryState = await getQueryState(refreshQueryExecutionId); + + expect(refreshQueryState).toEqual(QueryExecutionState.SUCCEEDED); + }); + }); +}); diff --git a/tests/playwright/helpers/athena-helpers.ts b/tests/playwright/helpers/athena-helpers.ts new file mode 100644 index 00000000..2d15ff24 --- /dev/null +++ b/tests/playwright/helpers/athena-helpers.ts @@ -0,0 +1,61 @@ +import { + AthenaClient, + GetQueryExecutionCommand, + QueryExecutionState, + StartQueryExecutionCommand, +} from '@aws-sdk/client-athena'; + +export { QueryExecutionState } from '@aws-sdk/client-athena'; + +const client = new AthenaClient(); + +/** + * Triggers a metadata refresh for an Athena table using the MSCK REPAIR TABLE command. + * + * This will cause any new files in S3 to be picked up. + * + * @param database - The name of the Athena database + * @param tableName - The name of the table to repair + * @param workgroup - The Athena workgroup to run the query in + * @returns The query execution ID + */ +export async function triggerTableMetadataRefresh( + database: string, + tableName: string, + workgroup: string, +): Promise { + const command = new StartQueryExecutionCommand({ + QueryString: `MSCK REPAIR TABLE ${tableName};`, + QueryExecutionContext: { + Database: database, + Catalog: 'AwsDataCatalog', + }, + WorkGroup: workgroup, + }); + + const response = await client.send(command); + + if (!response.QueryExecutionId) { + throw new Error( + 'Failed to start query execution - no query execution ID returned', + ); + } + + return response.QueryExecutionId; +} + +export async function getQueryState( + queryExecutionId: string, +): Promise { + const queryExecutionInfo = await client.send( + new GetQueryExecutionCommand({ + QueryExecutionId: queryExecutionId, + }), + ); + + if (!queryExecutionInfo.QueryExecution?.Status?.State) { + throw new Error('Failed to get query execution state'); + } + + return queryExecutionInfo.QueryExecution?.Status?.State; +} diff --git a/tests/playwright/package.json b/tests/playwright/package.json index 156975af..3a26193e 100644 --- a/tests/playwright/package.json +++ b/tests/playwright/package.json @@ -1,5 +1,6 @@ { "dependencies": { + "@aws-sdk/client-athena": "^3.900.0", "@aws-sdk/client-cloudwatch-logs": "^3.900.0", "@aws-sdk/client-dynamodb": "^3.900.0", "@aws-sdk/client-lambda": "^3.900.0", From 531bd8d46a011965effb0fb162d492c80376a97a Mon Sep 17 00:00:00 2001 From: Gareth Allan <157592212+gareth-allan@users.noreply.github.com> Date: Tue, 10 Feb 2026 17:10:21 +0000 Subject: [PATCH 2/2] CCM-13304: Added a component test for report generation --- package-lock.json | 67 ++++++++++ .../config/component/component.config.ts | 11 +- .../config/component/firehose.setup.ts | 21 ++++ .../config/component/firehose.teardown.ts | 21 ++++ .../playwright/constants/backend-constants.ts | 15 ++- .../report-generator.component.spec.ts | 59 ++++++++- .../helpers/data-firehose-helpers.ts | 116 ++++++++++++++++++ tests/playwright/helpers/s3-helpers.ts | 10 +- tests/playwright/package.json | 1 + 9 files changed, 315 insertions(+), 6 deletions(-) create mode 100644 tests/playwright/config/component/firehose.setup.ts create mode 100644 tests/playwright/config/component/firehose.teardown.ts create mode 100644 tests/playwright/helpers/data-firehose-helpers.ts diff --git a/package-lock.json b/package-lock.json index 0da092e5..7df0c3e6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5153,6 +5153,72 @@ "node": ">=20.0.0" } }, + "node_modules/@aws-sdk/client-firehose": { + "version": "3.986.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-firehose/-/client-firehose-3.986.0.tgz", + "integrity": "sha512-jWaPine3srErqaFLlPicLSNvFVKtBzWf/RVQS6CW3hOO4RrB6kgweIU+V0aUycj0CjmuBft+Z54BMCLgb1eo7A==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "^3.973.7", + "@aws-sdk/credential-provider-node": "^3.972.6", + "@aws-sdk/middleware-host-header": "^3.972.3", + "@aws-sdk/middleware-logger": "^3.972.3", + "@aws-sdk/middleware-recursion-detection": "^3.972.3", + "@aws-sdk/middleware-user-agent": "^3.972.7", + "@aws-sdk/region-config-resolver": "^3.972.3", + "@aws-sdk/types": "^3.973.1", + "@aws-sdk/util-endpoints": "3.986.0", + "@aws-sdk/util-user-agent-browser": "^3.972.3", + "@aws-sdk/util-user-agent-node": "^3.972.5", + "@smithy/config-resolver": "^4.4.6", + "@smithy/core": "^3.22.1", + "@smithy/fetch-http-handler": "^5.3.9", + "@smithy/hash-node": "^4.2.8", + "@smithy/invalid-dependency": "^4.2.8", + "@smithy/middleware-content-length": "^4.2.8", + "@smithy/middleware-endpoint": "^4.4.13", + "@smithy/middleware-retry": "^4.4.30", + "@smithy/middleware-serde": "^4.2.9", + "@smithy/middleware-stack": "^4.2.8", + "@smithy/node-config-provider": "^4.3.8", + "@smithy/node-http-handler": "^4.4.9", + "@smithy/protocol-http": "^5.3.8", + "@smithy/smithy-client": "^4.11.2", + "@smithy/types": "^4.12.0", + "@smithy/url-parser": "^4.2.8", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-body-length-browser": "^4.2.0", + "@smithy/util-body-length-node": "^4.2.1", + "@smithy/util-defaults-mode-browser": "^4.3.29", + "@smithy/util-defaults-mode-node": "^4.2.32", + "@smithy/util-endpoints": "^3.2.8", + "@smithy/util-middleware": "^4.2.8", + "@smithy/util-retry": "^4.2.8", + "@smithy/util-utf8": "^4.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/client-firehose/node_modules/@aws-sdk/util-endpoints": { + "version": "3.986.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.986.0.tgz", + "integrity": "sha512-Mqi79L38qi1gCG3adlVdbNrSxvcm1IPDLiJPA3OBypY5ewxUyWbaA3DD4goG+EwET6LSFgZJcRSIh6KBNpP5pA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.973.1", + "@smithy/types": "^4.12.0", + "@smithy/url-parser": "^4.2.8", + "@smithy/util-endpoints": "^3.2.8", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@aws-sdk/client-lambda": { "version": "3.981.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-lambda/-/client-lambda-3.981.0.tgz", @@ -24578,6 +24644,7 @@ "@aws-sdk/client-athena": "^3.900.0", "@aws-sdk/client-cloudwatch-logs": "^3.900.0", "@aws-sdk/client-dynamodb": "^3.900.0", + "@aws-sdk/client-firehose": "^3.900.0", "@aws-sdk/client-lambda": "^3.900.0", "@aws-sdk/client-s3": "^3.900.0", "@aws-sdk/client-sqs": "^3.900.0", diff --git a/tests/playwright/config/component/component.config.ts b/tests/playwright/config/component/component.config.ts index 340e7bb3..87ca74fa 100644 --- a/tests/playwright/config/component/component.config.ts +++ b/tests/playwright/config/component/component.config.ts @@ -13,10 +13,19 @@ export default defineConfig({ name: 'senders:setup', testMatch: 'senders.setup.ts', }, + { + name: 'firehose:setup', + testMatch: 'firehose.setup.ts', + teardown: 'firehose:teardown', + }, + { + name: 'firehose:teardown', + testMatch: 'firehose.teardown.ts', + }, { name: 'component', testMatch: '*.component.spec.ts', - dependencies: ['senders:setup'], + dependencies: ['senders:setup', 'firehose:setup'], teardown: 'component:teardown', }, { diff --git a/tests/playwright/config/component/firehose.setup.ts b/tests/playwright/config/component/firehose.setup.ts new file mode 100644 index 00000000..f7ecf537 --- /dev/null +++ b/tests/playwright/config/component/firehose.setup.ts @@ -0,0 +1,21 @@ +import { test as setup } from '@playwright/test'; +import { + MINIMUM_DESTINATION_BUFFER_INTERVAL, + MINIMUM_PROCESSOR_BUFFER_INTERVAL, + TERRAFORM_DESTINATION_BUFFER_INTERVAL, + TERRAFORM_PROCESSOR_BUFFER_INTERVAL, +} from 'constants/backend-constants'; +import { alterFirehoseBufferIntervals } from 'helpers/data-firehose-helpers'; + +setup('Reduce Firehose buffer intervals', async () => { + await alterFirehoseBufferIntervals({ + expected: { + destination: TERRAFORM_DESTINATION_BUFFER_INTERVAL, + processor: TERRAFORM_PROCESSOR_BUFFER_INTERVAL, + }, + update: { + destination: MINIMUM_DESTINATION_BUFFER_INTERVAL, + processor: MINIMUM_PROCESSOR_BUFFER_INTERVAL, + }, + }); +}); diff --git a/tests/playwright/config/component/firehose.teardown.ts b/tests/playwright/config/component/firehose.teardown.ts new file mode 100644 index 00000000..11844458 --- /dev/null +++ b/tests/playwright/config/component/firehose.teardown.ts @@ -0,0 +1,21 @@ +import { test as teardown } from '@playwright/test'; +import { + MINIMUM_DESTINATION_BUFFER_INTERVAL, + MINIMUM_PROCESSOR_BUFFER_INTERVAL, + TERRAFORM_DESTINATION_BUFFER_INTERVAL, + TERRAFORM_PROCESSOR_BUFFER_INTERVAL, +} from 'constants/backend-constants'; +import { alterFirehoseBufferIntervals } from 'helpers/data-firehose-helpers'; + +teardown('Restore Firehose buffer intervals', async () => { + await alterFirehoseBufferIntervals({ + expected: { + destination: MINIMUM_DESTINATION_BUFFER_INTERVAL, + processor: MINIMUM_PROCESSOR_BUFFER_INTERVAL, + }, + update: { + destination: TERRAFORM_DESTINATION_BUFFER_INTERVAL, + processor: TERRAFORM_PROCESSOR_BUFFER_INTERVAL, + }, + }); +}); diff --git a/tests/playwright/constants/backend-constants.ts b/tests/playwright/constants/backend-constants.ts index 574f8059..4792467d 100644 --- a/tests/playwright/constants/backend-constants.ts +++ b/tests/playwright/constants/backend-constants.ts @@ -40,10 +40,16 @@ export const EVENT_BUS_LOG_GROUP_NAME = `/aws/vendedlogs/events/event-bus/${CSI} // DynamoDB export const TTL_TABLE_NAME = `${CSI}-ttl`; +// Glue +export const GLUE_DATABASE_NAME = `${CSI}-reporting`; +export const GLUE_TABLE_NAME = 'event_record'; + // S3 export const LETTERS_S3_BUCKET_NAME = `nhs-${process.env.AWS_ACCOUNT_ID}-${REGION}-${ENV}-dl-letters`; export const NON_PII_S3_BUCKET_NAME = `nhs-${process.env.AWS_ACCOUNT_ID}-${REGION}-${ENV}-dl-non-pii-data`; export const PII_S3_BUCKET_NAME = `nhs-${process.env.AWS_ACCOUNT_ID}-${REGION}-${ENV}-dl-pii-data`; +export const REPORTING_S3_BUCKET_NAME = `nhs-${process.env.AWS_ACCOUNT_ID}-${REGION}-${ENV}-dl-reporting`; +export const REPORTING_S3_KEY_PREFIX = `kinesis-firehose-output/reporting/parquet/${GLUE_TABLE_NAME}`; export const FILE_SAFE_S3_BUCKET_NAME = `nhs-${process.env.AWS_ACCOUNT_ID}-${REGION}-${ENV}-dl-file-safe`; export const UNSCANNED_FILES_S3_BUCKET_NAME = `nhs-${process.env.AWS_ACCOUNT_ID}-${REGION}-main-acct-digi-unscanned-files`; export const FILE_QUARANTINE_S3_BUCKET_NAME = `nhs-${process.env.AWS_ACCOUNT_ID}-${REGION}-${ENV}-dl-file-quarantine`; @@ -59,9 +65,12 @@ export const PRINT_STATUS_HANDLER_LAMBDA_LOG_GROUP_NAME = `/aws/lambda/${CSI}-pr export const PRINT_ANALYSER_LAMBDA_LOG_GROUP_NAME = `/aws/lambda/${CSI}-print-analyser`; export const MOVE_SCANNED_FILES_LAMBDA_LOG_GROUP_NAME = `/aws/lambda/${CSI}-move-scanned-files`; -// Glue -export const GLUE_DATABASE_NAME = `${CSI}-reporting`; -export const GLUE_TABLE_NAME = 'event_record'; +// Data Firehose +export const FIREHOSE_STREAM_NAME = `${CSI}-to-s3-reporting`; +export const TERRAFORM_DESTINATION_BUFFER_INTERVAL = 300; +export const TERRAFORM_PROCESSOR_BUFFER_INTERVAL = 301; +export const MINIMUM_DESTINATION_BUFFER_INTERVAL = 60; +export const MINIMUM_PROCESSOR_BUFFER_INTERVAL = 0; // Athena export const ATHENA_WORKGROUP_NAME = CSI; diff --git a/tests/playwright/digital-letters-component-tests/report-generator.component.spec.ts b/tests/playwright/digital-letters-component-tests/report-generator.component.spec.ts index 22c38326..cd9c427d 100644 --- a/tests/playwright/digital-letters-component-tests/report-generator.component.spec.ts +++ b/tests/playwright/digital-letters-component-tests/report-generator.component.spec.ts @@ -3,16 +3,71 @@ import { ATHENA_WORKGROUP_NAME, GLUE_DATABASE_NAME, GLUE_TABLE_NAME, + REPORTING_S3_BUCKET_NAME, + REPORTING_S3_KEY_PREFIX, } from 'constants/backend-constants'; +import { MESHInboxMessageDownloaded } from 'digital-letters-events'; +import messageDownloadedValidator from 'digital-letters-events/MESHInboxMessageDownloaded.js'; import { QueryExecutionState, getQueryState, triggerTableMetadataRefresh, } from 'helpers/athena-helpers'; +import eventPublisher from 'helpers/event-bus-helpers'; import expectToPassEventually from 'helpers/expectations'; +import { existsInS3 } from 'helpers/s3-helpers'; +import { v4 as uuidv4 } from 'uuid'; test.describe('Digital Letters - Report Generator', () => { - test('should refresh data in the Glue table', async () => { + test.beforeAll(async () => { + // We need to wait for events to make their way from EventBridge -> Firehose -> S3 -> Glue + test.setTimeout(300_000); + + // Use a random sender ID, so we can be sure that if there are files with this prefix + // in S3 they've been created by this test. + const senderId = `report-generator-test-${uuidv4()}`; + + const eventId = uuidv4(); + await eventPublisher.sendEvents( + [ + { + id: eventId, + specversion: '1.0', + source: + '/nhs/england/notify/production/primary/data-plane/digitalletters/mesh', + subject: + 'customer/920fca11-596a-4eca-9c47-99f624614658/recipient/769acdd4-6a47-496f-999f-76a6fd2c3959', + type: 'uk.nhs.notify.digital.letters.mesh.inbox.message.downloaded.v1', + time: '2023-06-20T12:00:00Z', + recordedtime: '2023-06-20T12:00:00.250Z', + severitynumber: 2, + traceparent: + '00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01', + datacontenttype: 'application/json', + dataschema: + 'https://notify.nhs.uk/cloudevents/schemas/digital-letters/2025-10-draft/data/digital-letters-mesh-inbox-message-downloaded-data.schema.json', + severitytext: 'INFO', + data: { + meshMessageId: '12345', + messageUri: `https://example.com/ttl/resource/${eventId}`, + messageReference: 'ref1', + senderId, + }, + }, + ], + messageDownloadedValidator, + ); + + await expectToPassEventually(async () => { + const eventsHaveBeenWrittenToS3 = await existsInS3( + REPORTING_S3_BUCKET_NAME, + `${REPORTING_S3_KEY_PREFIX}/senderid=${senderId}`, + ); + + expect(eventsHaveBeenWrittenToS3).toBeTruthy(); + }, 300_000); + + // Trigger a metadata refresh for the Glue table, which will cause it to pick up any new files in S3 const refreshQueryExecutionId = await triggerTableMetadataRefresh( GLUE_DATABASE_NAME, GLUE_TABLE_NAME, @@ -25,4 +80,6 @@ test.describe('Digital Letters - Report Generator', () => { expect(refreshQueryState).toEqual(QueryExecutionState.SUCCEEDED); }); }); + + test('should test something', async () => {}); }); diff --git a/tests/playwright/helpers/data-firehose-helpers.ts b/tests/playwright/helpers/data-firehose-helpers.ts new file mode 100644 index 00000000..384fef24 --- /dev/null +++ b/tests/playwright/helpers/data-firehose-helpers.ts @@ -0,0 +1,116 @@ +import { + DescribeDeliveryStreamCommand, + DescribeDeliveryStreamCommandOutput, + ExtendedS3DestinationUpdate, + FirehoseClient, + UpdateDestinationCommand, +} from '@aws-sdk/client-firehose'; +import { FIREHOSE_STREAM_NAME, REGION } from 'constants/backend-constants'; + +export async function alterFirehoseBufferIntervals(bufferIntervalConfig: { + expected: { + destination: number; + processor: number; + }; + update: { + destination: number; + processor: number; + }; +}) { + const client = new FirehoseClient({ region: REGION }); + + const deliveryStreamDetails: DescribeDeliveryStreamCommandOutput = + await client.send( + new DescribeDeliveryStreamCommand({ + DeliveryStreamName: FIREHOSE_STREAM_NAME, + }), + ); + + const destinations = + deliveryStreamDetails.DeliveryStreamDescription?.Destinations ?? []; + if (destinations.length !== 1) { + throw new Error('expected a single delivery destination'); + } + + const destination = destinations[0]; + + const currentDestinationBufferInterval = + destination.ExtendedS3DestinationDescription?.BufferingHints + ?.IntervalInSeconds; + + if ( + currentDestinationBufferInterval !== + bufferIntervalConfig.expected.destination + ) { + throw new Error( + `Expected destination buffer size to be ${bufferIntervalConfig.expected.destination} - got ${currentDestinationBufferInterval} - cannot safely alter, has the default value changed in code or manually?`, + ); + } + + const processors = + destination.ExtendedS3DestinationDescription?.ProcessingConfiguration + ?.Processors; + + if (processors?.length !== 1) { + throw new Error('Expected one processor to be configured'); + } + + const processor = processors[0]; + + const currentProcessorBufferInterval = processor.Parameters?.find( + (p) => p.ParameterName === 'BufferIntervalInSeconds', + )?.ParameterValue; + + const otherParams = + processor.Parameters?.filter( + (p) => p.ParameterName !== 'BufferIntervalInSeconds', + ) ?? []; + + if ( + currentProcessorBufferInterval !== + bufferIntervalConfig.expected.processor.toString() + ) { + throw new Error( + `Expected processor buffer size to be ${bufferIntervalConfig.expected.processor} - got ${currentProcessorBufferInterval} - cannot safely alter, has the default value changed in code or manually?`, + ); + } + + const destinationId = destination.DestinationId; + + if (!destinationId) { + throw new Error('Destination ID not found'); + } + + const updatedDestinationConfig: ExtendedS3DestinationUpdate = { + ...destination.ExtendedS3DestinationDescription, + BufferingHints: { + ...destination.ExtendedS3DestinationDescription?.BufferingHints, + IntervalInSeconds: bufferIntervalConfig.update.destination, + }, + ProcessingConfiguration: { + ...destination.ExtendedS3DestinationDescription?.ProcessingConfiguration, + Processors: [ + { + ...processor, + Parameters: [ + ...otherParams, + { + ParameterName: 'BufferIntervalInSeconds', + ParameterValue: bufferIntervalConfig.update.processor.toString(), + }, + ], + }, + ], + }, + }; + + await client.send( + new UpdateDestinationCommand({ + DeliveryStreamName: FIREHOSE_STREAM_NAME, + DestinationId: destinationId, + CurrentDeliveryStreamVersionId: + deliveryStreamDetails.DeliveryStreamDescription?.VersionId, + ExtendedS3DestinationUpdate: updatedDestinationConfig, + }), + ); +} diff --git a/tests/playwright/helpers/s3-helpers.ts b/tests/playwright/helpers/s3-helpers.ts index 108e90c5..b2d6eb0a 100644 --- a/tests/playwright/helpers/s3-helpers.ts +++ b/tests/playwright/helpers/s3-helpers.ts @@ -61,4 +61,12 @@ async function downloadFromS3( }; } -export { downloadFromS3, uploadToS3 }; +async function existsInS3(bucket: string, keyPrefix: string): Promise { + const objects = await s3.send( + new ListObjectsV2Command({ Bucket: bucket, Prefix: keyPrefix }), + ); + + return (objects.Contents?.length ?? 0) > 0; +} + +export { downloadFromS3, existsInS3, uploadToS3 }; diff --git a/tests/playwright/package.json b/tests/playwright/package.json index 3a26193e..f834d2df 100644 --- a/tests/playwright/package.json +++ b/tests/playwright/package.json @@ -3,6 +3,7 @@ "@aws-sdk/client-athena": "^3.900.0", "@aws-sdk/client-cloudwatch-logs": "^3.900.0", "@aws-sdk/client-dynamodb": "^3.900.0", + "@aws-sdk/client-firehose": "^3.900.0", "@aws-sdk/client-lambda": "^3.900.0", "@aws-sdk/client-s3": "^3.900.0", "@aws-sdk/client-sqs": "^3.900.0",