diff --git a/package-lock.json b/package-lock.json index 6b76b351..48342c7f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4905,6 +4905,72 @@ "node": ">=14.0.0" } }, + "node_modules/@aws-sdk/client-athena": { + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-athena/-/client-athena-3.989.0.tgz", + "integrity": "sha512-nehYJYXNjazNa8oe/GjKNyNJ96FzMDQHAhaq3Em9icUWo7ndKHTGGKFHoPo2TORayP7yxqVwQ1TSGgEBLOHwpw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "^3.973.9", + "@aws-sdk/credential-provider-node": "^3.972.8", + "@aws-sdk/middleware-host-header": "^3.972.3", + "@aws-sdk/middleware-logger": "^3.972.3", + "@aws-sdk/middleware-recursion-detection": "^3.972.3", + "@aws-sdk/middleware-user-agent": "^3.972.9", + "@aws-sdk/region-config-resolver": "^3.972.3", + "@aws-sdk/types": "^3.973.1", + "@aws-sdk/util-endpoints": "3.989.0", + "@aws-sdk/util-user-agent-browser": "^3.972.3", + "@aws-sdk/util-user-agent-node": "^3.972.7", + "@smithy/config-resolver": "^4.4.6", + "@smithy/core": "^3.23.0", + "@smithy/fetch-http-handler": "^5.3.9", + "@smithy/hash-node": "^4.2.8", + "@smithy/invalid-dependency": "^4.2.8", + "@smithy/middleware-content-length": "^4.2.8", + "@smithy/middleware-endpoint": "^4.4.14", + "@smithy/middleware-retry": "^4.4.31", + "@smithy/middleware-serde": "^4.2.9", + "@smithy/middleware-stack": "^4.2.8", + "@smithy/node-config-provider": "^4.3.8", + "@smithy/node-http-handler": "^4.4.10", + "@smithy/protocol-http": "^5.3.8", + "@smithy/smithy-client": "^4.11.3", + "@smithy/types": "^4.12.0", + "@smithy/url-parser": "^4.2.8", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-body-length-browser": "^4.2.0", + "@smithy/util-body-length-node": "^4.2.1", + "@smithy/util-defaults-mode-browser": "^4.3.30", + "@smithy/util-defaults-mode-node": "^4.2.33", + "@smithy/util-endpoints": "^3.2.8", + "@smithy/util-middleware": "^4.2.8", + "@smithy/util-retry": "^4.2.8", + "@smithy/util-utf8": "^4.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/client-athena/node_modules/@aws-sdk/util-endpoints": { + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.989.0.tgz", + "integrity": "sha512-eKmAOeQM4Qusq0jtcbZPiNWky8XaojByKC/n+THbJ8vJf7t4ys8LlcZ4PrBSHZISe9cC484mQsPVOQh6iySjqw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.973.1", + "@smithy/types": "^4.12.0", + "@smithy/url-parser": "^4.2.8", + "@smithy/util-endpoints": "^3.2.8", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@aws-sdk/client-cloudwatch-logs": { "version": "3.981.0", "license": "Apache-2.0", @@ -5139,6 +5205,72 @@ "node": ">=20.0.0" } }, + "node_modules/@aws-sdk/client-firehose": { + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-firehose/-/client-firehose-3.989.0.tgz", + "integrity": "sha512-y5zUH7XHHiZPbdSm/f98g0yBIF3+F51yzmPvbrSRHNpzmDf74haszBfEOjBKGOat3oZJngzw8XsZM72BXSfKJQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "^3.973.9", + "@aws-sdk/credential-provider-node": "^3.972.8", + "@aws-sdk/middleware-host-header": "^3.972.3", + "@aws-sdk/middleware-logger": "^3.972.3", + "@aws-sdk/middleware-recursion-detection": "^3.972.3", + "@aws-sdk/middleware-user-agent": "^3.972.9", + "@aws-sdk/region-config-resolver": "^3.972.3", + "@aws-sdk/types": "^3.973.1", + "@aws-sdk/util-endpoints": "3.989.0", + "@aws-sdk/util-user-agent-browser": "^3.972.3", + "@aws-sdk/util-user-agent-node": "^3.972.7", + "@smithy/config-resolver": "^4.4.6", + "@smithy/core": "^3.23.0", + "@smithy/fetch-http-handler": "^5.3.9", + "@smithy/hash-node": "^4.2.8", + "@smithy/invalid-dependency": "^4.2.8", + "@smithy/middleware-content-length": "^4.2.8", + "@smithy/middleware-endpoint": "^4.4.14", + "@smithy/middleware-retry": "^4.4.31", + "@smithy/middleware-serde": "^4.2.9", + "@smithy/middleware-stack": "^4.2.8", + "@smithy/node-config-provider": "^4.3.8", + "@smithy/node-http-handler": "^4.4.10", + "@smithy/protocol-http": "^5.3.8", + "@smithy/smithy-client": "^4.11.3", + "@smithy/types": "^4.12.0", + "@smithy/url-parser": "^4.2.8", + "@smithy/util-base64": "^4.3.0", + "@smithy/util-body-length-browser": "^4.2.0", + "@smithy/util-body-length-node": "^4.2.1", + "@smithy/util-defaults-mode-browser": "^4.3.30", + "@smithy/util-defaults-mode-node": "^4.2.33", + "@smithy/util-endpoints": "^3.2.8", + "@smithy/util-middleware": "^4.2.8", + "@smithy/util-retry": "^4.2.8", + "@smithy/util-utf8": "^4.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/client-firehose/node_modules/@aws-sdk/util-endpoints": { + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.989.0.tgz", + "integrity": "sha512-eKmAOeQM4Qusq0jtcbZPiNWky8XaojByKC/n+THbJ8vJf7t4ys8LlcZ4PrBSHZISe9cC484mQsPVOQh6iySjqw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.973.1", + "@smithy/types": "^4.12.0", + "@smithy/url-parser": "^4.2.8", + "@smithy/util-endpoints": "^3.2.8", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@aws-sdk/client-lambda": { "version": "3.981.0", "license": "Apache-2.0", @@ -5357,42 +5489,44 @@ } }, "node_modules/@aws-sdk/client-sso": { - "version": "3.980.0", + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.989.0.tgz", + "integrity": "sha512-3sC+J1ru5VFXLgt9KZmXto0M7mnV5RkS6FNGwRMK3XrojSjHso9DLOWjbnXhbNv4motH8vu53L1HK2VC1+Nj5w==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.973.5", + "@aws-sdk/core": "^3.973.9", "@aws-sdk/middleware-host-header": "^3.972.3", "@aws-sdk/middleware-logger": "^3.972.3", "@aws-sdk/middleware-recursion-detection": "^3.972.3", - "@aws-sdk/middleware-user-agent": "^3.972.5", + "@aws-sdk/middleware-user-agent": "^3.972.9", "@aws-sdk/region-config-resolver": "^3.972.3", "@aws-sdk/types": "^3.973.1", - "@aws-sdk/util-endpoints": "3.980.0", + "@aws-sdk/util-endpoints": "3.989.0", "@aws-sdk/util-user-agent-browser": "^3.972.3", - "@aws-sdk/util-user-agent-node": "^3.972.3", + "@aws-sdk/util-user-agent-node": "^3.972.7", "@smithy/config-resolver": "^4.4.6", - "@smithy/core": "^3.22.0", + "@smithy/core": "^3.23.0", "@smithy/fetch-http-handler": "^5.3.9", "@smithy/hash-node": "^4.2.8", "@smithy/invalid-dependency": "^4.2.8", "@smithy/middleware-content-length": "^4.2.8", - "@smithy/middleware-endpoint": "^4.4.12", - "@smithy/middleware-retry": "^4.4.29", + "@smithy/middleware-endpoint": "^4.4.14", + "@smithy/middleware-retry": "^4.4.31", "@smithy/middleware-serde": "^4.2.9", "@smithy/middleware-stack": "^4.2.8", "@smithy/node-config-provider": "^4.3.8", - "@smithy/node-http-handler": "^4.4.8", + "@smithy/node-http-handler": "^4.4.10", "@smithy/protocol-http": "^5.3.8", - "@smithy/smithy-client": "^4.11.1", + "@smithy/smithy-client": "^4.11.3", "@smithy/types": "^4.12.0", "@smithy/url-parser": "^4.2.8", "@smithy/util-base64": "^4.3.0", "@smithy/util-body-length-browser": "^4.2.0", "@smithy/util-body-length-node": "^4.2.1", - "@smithy/util-defaults-mode-browser": "^4.3.28", - "@smithy/util-defaults-mode-node": "^4.2.31", + "@smithy/util-defaults-mode-browser": "^4.3.30", + "@smithy/util-defaults-mode-node": "^4.2.33", "@smithy/util-endpoints": "^3.2.8", "@smithy/util-middleware": "^4.2.8", "@smithy/util-retry": "^4.2.8", @@ -5404,7 +5538,9 @@ } }, "node_modules/@aws-sdk/client-sso/node_modules/@aws-sdk/util-endpoints": { - "version": "3.980.0", + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.989.0.tgz", + "integrity": "sha512-eKmAOeQM4Qusq0jtcbZPiNWky8XaojByKC/n+THbJ8vJf7t4ys8LlcZ4PrBSHZISe9cC484mQsPVOQh6iySjqw==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/types": "^3.973.1", @@ -5418,17 +5554,19 @@ } }, "node_modules/@aws-sdk/core": { - "version": "3.973.5", + "version": "3.973.9", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.973.9.tgz", + "integrity": "sha512-cyUOfJSizn8da7XrBEFBf4UMI4A6JQNX6ZFcKtYmh/CrwfzsDcabv3k/z0bNwQ3pX5aeq5sg/8Bs/ASiL0bJaA==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/types": "^3.973.1", - "@aws-sdk/xml-builder": "^3.972.2", - "@smithy/core": "^3.22.0", + "@aws-sdk/xml-builder": "^3.972.4", + "@smithy/core": "^3.23.0", "@smithy/node-config-provider": "^4.3.8", "@smithy/property-provider": "^4.2.8", "@smithy/protocol-http": "^5.3.8", "@smithy/signature-v4": "^5.3.8", - "@smithy/smithy-client": "^4.11.1", + "@smithy/smithy-client": "^4.11.3", "@smithy/types": "^4.12.0", "@smithy/util-base64": "^4.3.0", "@smithy/util-middleware": "^4.2.8", @@ -5451,10 +5589,12 @@ } }, "node_modules/@aws-sdk/credential-provider-env": { - "version": "3.972.3", + "version": "3.972.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.7.tgz", + "integrity": "sha512-r8kBtglvLjGxBT87l6Lqkh9fL8yJJ6O4CYQPjKlj3AkCuL4/4784x3rxxXWw9LTKXOo114VB6mjxAuy5pI7XIg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.5", + "@aws-sdk/core": "^3.973.9", "@aws-sdk/types": "^3.973.1", "@smithy/property-provider": "^4.2.8", "@smithy/types": "^4.12.0", @@ -5465,18 +5605,20 @@ } }, "node_modules/@aws-sdk/credential-provider-http": { - "version": "3.972.5", + "version": "3.972.9", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.9.tgz", + "integrity": "sha512-40caFblEg/TPrp9EpvyMxp4xlJ5TuTI+A8H6g8FhHn2hfH2PObFAPLF9d5AljK/G69E1YtTklkuQeAwPlV3w8Q==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.5", + "@aws-sdk/core": "^3.973.9", "@aws-sdk/types": "^3.973.1", "@smithy/fetch-http-handler": "^5.3.9", - "@smithy/node-http-handler": "^4.4.8", + "@smithy/node-http-handler": "^4.4.10", "@smithy/property-provider": "^4.2.8", "@smithy/protocol-http": "^5.3.8", - "@smithy/smithy-client": "^4.11.1", + "@smithy/smithy-client": "^4.11.3", "@smithy/types": "^4.12.0", - "@smithy/util-stream": "^4.5.10", + "@smithy/util-stream": "^4.5.12", "tslib": "^2.6.2" }, "engines": { @@ -5484,17 +5626,19 @@ } }, "node_modules/@aws-sdk/credential-provider-ini": { - "version": "3.972.3", + "version": "3.972.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.972.7.tgz", + "integrity": "sha512-zeYKrMwM5bCkHFho/x3+1OL0vcZQ0OhTR7k35tLq74+GP5ieV3juHXTZfa2LVE0Bg75cHIIerpX0gomVOhzo/w==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.5", - "@aws-sdk/credential-provider-env": "^3.972.3", - "@aws-sdk/credential-provider-http": "^3.972.5", - "@aws-sdk/credential-provider-login": "^3.972.3", - "@aws-sdk/credential-provider-process": "^3.972.3", - "@aws-sdk/credential-provider-sso": "^3.972.3", - "@aws-sdk/credential-provider-web-identity": "^3.972.3", - "@aws-sdk/nested-clients": "3.980.0", + "@aws-sdk/core": "^3.973.9", + "@aws-sdk/credential-provider-env": "^3.972.7", + "@aws-sdk/credential-provider-http": "^3.972.9", + "@aws-sdk/credential-provider-login": "^3.972.7", + "@aws-sdk/credential-provider-process": "^3.972.7", + "@aws-sdk/credential-provider-sso": "^3.972.7", + "@aws-sdk/credential-provider-web-identity": "^3.972.7", + "@aws-sdk/nested-clients": "3.989.0", "@aws-sdk/types": "^3.973.1", "@smithy/credential-provider-imds": "^4.2.8", "@smithy/property-provider": "^4.2.8", @@ -5507,11 +5651,13 @@ } }, "node_modules/@aws-sdk/credential-provider-login": { - "version": "3.972.3", + "version": "3.972.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.7.tgz", + "integrity": "sha512-Q103cLU6OjAllYjX7+V+PKQw654jjvZUkD+lbUUiFbqut6gR5zwl1DrelvJPM5hnzIty7BCaxaRB3KMuz3M/ug==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.5", - "@aws-sdk/nested-clients": "3.980.0", + "@aws-sdk/core": "^3.973.9", + "@aws-sdk/nested-clients": "3.989.0", "@aws-sdk/types": "^3.973.1", "@smithy/property-provider": "^4.2.8", "@smithy/protocol-http": "^5.3.8", @@ -5524,15 +5670,17 @@ } }, "node_modules/@aws-sdk/credential-provider-node": { - "version": "3.972.4", + "version": "3.972.8", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.8.tgz", + "integrity": "sha512-AaDVOT7iNJyLjc3j91VlucPZ4J8Bw+eu9sllRDugJqhHWYyR3Iyp2huBUW8A3+DfHoh70sxGkY92cThAicSzlQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/credential-provider-env": "^3.972.3", - "@aws-sdk/credential-provider-http": "^3.972.5", - "@aws-sdk/credential-provider-ini": "^3.972.3", - "@aws-sdk/credential-provider-process": "^3.972.3", - "@aws-sdk/credential-provider-sso": "^3.972.3", - "@aws-sdk/credential-provider-web-identity": "^3.972.3", + "@aws-sdk/credential-provider-env": "^3.972.7", + "@aws-sdk/credential-provider-http": "^3.972.9", + "@aws-sdk/credential-provider-ini": "^3.972.7", + "@aws-sdk/credential-provider-process": "^3.972.7", + "@aws-sdk/credential-provider-sso": "^3.972.7", + "@aws-sdk/credential-provider-web-identity": "^3.972.7", "@aws-sdk/types": "^3.973.1", "@smithy/credential-provider-imds": "^4.2.8", "@smithy/property-provider": "^4.2.8", @@ -5545,10 +5693,12 @@ } }, "node_modules/@aws-sdk/credential-provider-process": { - "version": "3.972.3", + "version": "3.972.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.7.tgz", + "integrity": "sha512-hxMo1V3ujWWrQSONxQJAElnjredkRpB6p8SDjnvRq70IwYY38R/CZSys0IbhRPxdgWZ5j12yDRk2OXhxw4Gj3g==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.5", + "@aws-sdk/core": "^3.973.9", "@aws-sdk/types": "^3.973.1", "@smithy/property-provider": "^4.2.8", "@smithy/shared-ini-file-loader": "^4.4.3", @@ -5560,12 +5710,14 @@ } }, "node_modules/@aws-sdk/credential-provider-sso": { - "version": "3.972.3", + "version": "3.972.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.972.7.tgz", + "integrity": "sha512-ZGKBOHEj8Ap15jhG2XMncQmKLTqA++2DVU2eZfLu3T/pkwDyhCp5eZv5c/acFxbZcA/6mtxke+vzO/n+aeHs4A==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/client-sso": "3.980.0", - "@aws-sdk/core": "^3.973.5", - "@aws-sdk/token-providers": "3.980.0", + "@aws-sdk/client-sso": "3.989.0", + "@aws-sdk/core": "^3.973.9", + "@aws-sdk/token-providers": "3.989.0", "@aws-sdk/types": "^3.973.1", "@smithy/property-provider": "^4.2.8", "@smithy/shared-ini-file-loader": "^4.4.3", @@ -5577,11 +5729,13 @@ } }, "node_modules/@aws-sdk/credential-provider-web-identity": { - "version": "3.972.3", + "version": "3.972.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.7.tgz", + "integrity": "sha512-AbYupBIoSJoVMlbMqBhNvPhqj+CdGtzW7Uk4ZIMBm2br18pc3rkG1VaKVFV85H87QCvLHEnni1idJjaX1wOmIw==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.5", - "@aws-sdk/nested-clients": "3.980.0", + "@aws-sdk/core": "^3.973.9", + "@aws-sdk/nested-clients": "3.989.0", "@aws-sdk/types": "^3.973.1", "@smithy/property-provider": "^4.2.8", "@smithy/shared-ini-file-loader": "^4.4.3", @@ -5809,13 +5963,15 @@ } }, "node_modules/@aws-sdk/middleware-user-agent": { - "version": "3.972.5", + "version": "3.972.9", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.972.9.tgz", + "integrity": "sha512-1g1B7yf7KzessB0mKNiV9gAHEwbM662xgU+VE4LxyGe6kVGZ8LqYsngjhE+Stna09CJ7Pxkjr6Uq1OtbGwJJJg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.5", + "@aws-sdk/core": "^3.973.9", "@aws-sdk/types": "^3.973.1", - "@aws-sdk/util-endpoints": "3.980.0", - "@smithy/core": "^3.22.0", + "@aws-sdk/util-endpoints": "3.989.0", + "@smithy/core": "^3.23.0", "@smithy/protocol-http": "^5.3.8", "@smithy/types": "^4.12.0", "tslib": "^2.6.2" @@ -5825,7 +5981,9 @@ } }, "node_modules/@aws-sdk/middleware-user-agent/node_modules/@aws-sdk/util-endpoints": { - "version": "3.980.0", + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.989.0.tgz", + "integrity": "sha512-eKmAOeQM4Qusq0jtcbZPiNWky8XaojByKC/n+THbJ8vJf7t4ys8LlcZ4PrBSHZISe9cC484mQsPVOQh6iySjqw==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/types": "^3.973.1", @@ -5839,42 +5997,44 @@ } }, "node_modules/@aws-sdk/nested-clients": { - "version": "3.980.0", + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.989.0.tgz", + "integrity": "sha512-Dbk2HMPU3mb6RrSRzgf0WCaWSbgtZG258maCpuN2/ONcAQNpOTw99V5fU5CA1qVK6Vkm4Fwj2cnOnw7wbGVlOw==", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.973.5", + "@aws-sdk/core": "^3.973.9", "@aws-sdk/middleware-host-header": "^3.972.3", "@aws-sdk/middleware-logger": "^3.972.3", "@aws-sdk/middleware-recursion-detection": "^3.972.3", - "@aws-sdk/middleware-user-agent": "^3.972.5", + "@aws-sdk/middleware-user-agent": "^3.972.9", "@aws-sdk/region-config-resolver": "^3.972.3", "@aws-sdk/types": "^3.973.1", - "@aws-sdk/util-endpoints": "3.980.0", + "@aws-sdk/util-endpoints": "3.989.0", "@aws-sdk/util-user-agent-browser": "^3.972.3", - "@aws-sdk/util-user-agent-node": "^3.972.3", + "@aws-sdk/util-user-agent-node": "^3.972.7", "@smithy/config-resolver": "^4.4.6", - "@smithy/core": "^3.22.0", + "@smithy/core": "^3.23.0", "@smithy/fetch-http-handler": "^5.3.9", "@smithy/hash-node": "^4.2.8", "@smithy/invalid-dependency": "^4.2.8", "@smithy/middleware-content-length": "^4.2.8", - "@smithy/middleware-endpoint": "^4.4.12", - "@smithy/middleware-retry": "^4.4.29", + "@smithy/middleware-endpoint": "^4.4.14", + "@smithy/middleware-retry": "^4.4.31", "@smithy/middleware-serde": "^4.2.9", "@smithy/middleware-stack": "^4.2.8", "@smithy/node-config-provider": "^4.3.8", - "@smithy/node-http-handler": "^4.4.8", + "@smithy/node-http-handler": "^4.4.10", "@smithy/protocol-http": "^5.3.8", - "@smithy/smithy-client": "^4.11.1", + "@smithy/smithy-client": "^4.11.3", "@smithy/types": "^4.12.0", "@smithy/url-parser": "^4.2.8", "@smithy/util-base64": "^4.3.0", "@smithy/util-body-length-browser": "^4.2.0", "@smithy/util-body-length-node": "^4.2.1", - "@smithy/util-defaults-mode-browser": "^4.3.28", - "@smithy/util-defaults-mode-node": "^4.2.31", + "@smithy/util-defaults-mode-browser": "^4.3.30", + "@smithy/util-defaults-mode-node": "^4.2.33", "@smithy/util-endpoints": "^3.2.8", "@smithy/util-middleware": "^4.2.8", "@smithy/util-retry": "^4.2.8", @@ -5886,7 +6046,9 @@ } }, "node_modules/@aws-sdk/nested-clients/node_modules/@aws-sdk/util-endpoints": { - "version": "3.980.0", + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.989.0.tgz", + "integrity": "sha512-eKmAOeQM4Qusq0jtcbZPiNWky8XaojByKC/n+THbJ8vJf7t4ys8LlcZ4PrBSHZISe9cC484mQsPVOQh6iySjqw==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/types": "^3.973.1", @@ -5929,11 +6091,13 @@ } }, "node_modules/@aws-sdk/token-providers": { - "version": "3.980.0", + "version": "3.989.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.989.0.tgz", + "integrity": "sha512-OdBByMv+OjOZoekrk4THPFpLuND5aIQbDHCGh3n2rvifAbm31+6e0OLhxSeCF1UMPm+nKq12bXYYEoCIx5SQBg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.973.5", - "@aws-sdk/nested-clients": "3.980.0", + "@aws-sdk/core": "^3.973.9", + "@aws-sdk/nested-clients": "3.989.0", "@aws-sdk/types": "^3.973.1", "@smithy/property-provider": "^4.2.8", "@smithy/shared-ini-file-loader": "^4.4.3", @@ -6013,10 +6177,12 @@ } }, "node_modules/@aws-sdk/util-user-agent-node": { - "version": "3.972.3", + "version": "3.972.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.972.7.tgz", + "integrity": "sha512-oyhv+FjrgHjP+F16cmsrJzNP4qaRJzkV1n9Lvv4uyh3kLqo3rIe9NSBSBa35f2TedczfG2dD+kaQhHBB47D6Og==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/middleware-user-agent": "^3.972.5", + "@aws-sdk/middleware-user-agent": "^3.972.9", "@aws-sdk/types": "^3.973.1", "@smithy/node-config-provider": "^4.3.8", "@smithy/types": "^4.12.0", @@ -6035,7 +6201,9 @@ } }, "node_modules/@aws-sdk/xml-builder": { - "version": "3.972.3", + "version": "3.972.4", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.4.tgz", + "integrity": "sha512-0zJ05ANfYqI6+rGqj8samZBFod0dPPousBjLEqg8WdxSgbMAkRgLyn81lP215Do0rFJ/17LIXwr7q0yK24mP6Q==", "license": "Apache-2.0", "dependencies": { "@smithy/types": "^4.12.0", @@ -8614,7 +8782,9 @@ } }, "node_modules/@smithy/core": { - "version": "3.22.1", + "version": "3.23.0", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.23.0.tgz", + "integrity": "sha512-Yq4UPVoQICM9zHnByLmG8632t2M0+yap4T7ANVw482J0W7HW0pOuxwVmeOwzJqX2Q89fkXz0Vybz55Wj2Xzrsg==", "license": "Apache-2.0", "dependencies": { "@smithy/middleware-serde": "^4.2.9", @@ -8623,7 +8793,7 @@ "@smithy/util-base64": "^4.3.0", "@smithy/util-body-length-browser": "^4.2.0", "@smithy/util-middleware": "^4.2.8", - "@smithy/util-stream": "^4.5.11", + "@smithy/util-stream": "^4.5.12", "@smithy/util-utf8": "^4.2.0", "@smithy/uuid": "^1.1.0", "tslib": "^2.6.2" @@ -8804,10 +8974,12 @@ } }, "node_modules/@smithy/middleware-endpoint": { - "version": "4.4.13", + "version": "4.4.14", + "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-4.4.14.tgz", + "integrity": "sha512-FUFNE5KVeaY6U/GL0nzAAHkaCHzXLZcY1EhtQnsAqhD8Du13oPKtMB9/0WK4/LK6a/T5OZ24wPoSShff5iI6Ag==", "license": "Apache-2.0", "dependencies": { - "@smithy/core": "^3.22.1", + "@smithy/core": "^3.23.0", "@smithy/middleware-serde": "^4.2.9", "@smithy/node-config-provider": "^4.3.8", "@smithy/shared-ini-file-loader": "^4.4.3", @@ -8821,13 +8993,15 @@ } }, "node_modules/@smithy/middleware-retry": { - "version": "4.4.30", + "version": "4.4.31", + "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-4.4.31.tgz", + "integrity": "sha512-RXBzLpMkIrxBPe4C8OmEOHvS8aH9RUuCOH++Acb5jZDEblxDjyg6un72X9IcbrGTJoiUwmI7hLypNfuDACypbg==", "license": "Apache-2.0", "dependencies": { "@smithy/node-config-provider": "^4.3.8", "@smithy/protocol-http": "^5.3.8", "@smithy/service-error-classification": "^4.2.8", - "@smithy/smithy-client": "^4.11.2", + "@smithy/smithy-client": "^4.11.3", "@smithy/types": "^4.12.0", "@smithy/util-middleware": "^4.2.8", "@smithy/util-retry": "^4.2.8", @@ -8875,7 +9049,9 @@ } }, "node_modules/@smithy/node-http-handler": { - "version": "4.4.9", + "version": "4.4.10", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.4.10.tgz", + "integrity": "sha512-u4YeUwOWRZaHbWaebvrs3UhwQwj+2VNmcVCwXcYTvPIuVyM7Ex1ftAj+fdbG/P4AkBwLq/+SKn+ydOI4ZJE9PA==", "license": "Apache-2.0", "dependencies": { "@smithy/abort-controller": "^4.2.8", @@ -8972,15 +9148,17 @@ } }, "node_modules/@smithy/smithy-client": { - "version": "4.11.2", + "version": "4.11.3", + "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-4.11.3.tgz", + "integrity": "sha512-Q7kY5sDau8OoE6Y9zJoRGgje8P4/UY0WzH8R2ok0PDh+iJ+ZnEKowhjEqYafVcubkbYxQVaqwm3iufktzhprGg==", "license": "Apache-2.0", "dependencies": { - "@smithy/core": "^3.22.1", - "@smithy/middleware-endpoint": "^4.4.13", + "@smithy/core": "^3.23.0", + "@smithy/middleware-endpoint": "^4.4.14", "@smithy/middleware-stack": "^4.2.8", "@smithy/protocol-http": "^5.3.8", "@smithy/types": "^4.12.0", - "@smithy/util-stream": "^4.5.11", + "@smithy/util-stream": "^4.5.12", "tslib": "^2.6.2" }, "engines": { @@ -9063,11 +9241,13 @@ } }, "node_modules/@smithy/util-defaults-mode-browser": { - "version": "4.3.29", + "version": "4.3.30", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-4.3.30.tgz", + "integrity": "sha512-cMni0uVU27zxOiU8TuC8pQLC1pYeZ/xEMxvchSK/ILwleRd1ugobOcIRr5vXtcRqKd4aBLWlpeBoDPJJ91LQng==", "license": "Apache-2.0", "dependencies": { "@smithy/property-provider": "^4.2.8", - "@smithy/smithy-client": "^4.11.2", + "@smithy/smithy-client": "^4.11.3", "@smithy/types": "^4.12.0", "tslib": "^2.6.2" }, @@ -9076,14 +9256,16 @@ } }, "node_modules/@smithy/util-defaults-mode-node": { - "version": "4.2.32", + "version": "4.2.33", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-4.2.33.tgz", + "integrity": "sha512-LEb2aq5F4oZUSzWBG7S53d4UytZSkOEJPXcBq/xbG2/TmK9EW5naUZ8lKu1BEyWMzdHIzEVN16M3k8oxDq+DJA==", "license": "Apache-2.0", "dependencies": { "@smithy/config-resolver": "^4.4.6", "@smithy/credential-provider-imds": "^4.2.8", "@smithy/node-config-provider": "^4.3.8", "@smithy/property-provider": "^4.2.8", - "@smithy/smithy-client": "^4.11.2", + "@smithy/smithy-client": "^4.11.3", "@smithy/types": "^4.12.0", "tslib": "^2.6.2" }, @@ -9137,11 +9319,13 @@ } }, "node_modules/@smithy/util-stream": { - "version": "4.5.11", + "version": "4.5.12", + "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-4.5.12.tgz", + "integrity": "sha512-D8tgkrmhAX/UNeCZbqbEO3uqyghUnEmmoO9YEvRuwxjlkKKUE7FOgCJnqpTlQPe9MApdWPky58mNQQHbnCzoNg==", "license": "Apache-2.0", "dependencies": { "@smithy/fetch-http-handler": "^5.3.9", - "@smithy/node-http-handler": "^4.4.9", + "@smithy/node-http-handler": "^4.4.10", "@smithy/types": "^4.12.0", "@smithy/util-base64": "^4.3.0", "@smithy/util-buffer-from": "^4.2.0", @@ -12610,6 +12794,8 @@ }, "node_modules/fast-xml-parser": { "version": "5.3.4", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.3.4.tgz", + "integrity": "sha512-EFd6afGmXlCx8H8WTZHhAoDaWaGyuIBoZJ2mknrNxug+aZKjkp0a0dlars9Izl+jF+7Gu1/5f/2h68cQpe0IiA==", "funding": [ { "type": "github", @@ -19217,6 +19403,8 @@ }, "node_modules/strnum": { "version": "2.1.2", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.1.2.tgz", + "integrity": "sha512-l63NF9y/cLROq/yqKXSLtcMeeyOfnSQlfMSlzFt/K73oIaD8DGaQWd7Z34X9GPiKqP5rbSh84Hl4bOlLcjiSrQ==", "funding": [ { "type": "github", @@ -21003,8 +21191,10 @@ "name": "nhs-notify-digital-letters-integration-tests", "version": "0.0.1", "dependencies": { + "@aws-sdk/client-athena": "^3.900.0", "@aws-sdk/client-cloudwatch-logs": "^3.900.0", "@aws-sdk/client-dynamodb": "^3.900.0", + "@aws-sdk/client-firehose": "^3.900.0", "@aws-sdk/client-lambda": "^3.900.0", "@aws-sdk/client-s3": "^3.900.0", "@aws-sdk/client-sqs": "^3.900.0", diff --git a/tests/playwright/config/component/component.config.ts b/tests/playwright/config/component/component.config.ts index 340e7bb3..87ca74fa 100644 --- a/tests/playwright/config/component/component.config.ts +++ b/tests/playwright/config/component/component.config.ts @@ -13,10 +13,19 @@ export default defineConfig({ name: 'senders:setup', testMatch: 'senders.setup.ts', }, + { + name: 'firehose:setup', + testMatch: 'firehose.setup.ts', + teardown: 'firehose:teardown', + }, + { + name: 'firehose:teardown', + testMatch: 'firehose.teardown.ts', + }, { name: 'component', testMatch: '*.component.spec.ts', - dependencies: ['senders:setup'], + dependencies: ['senders:setup', 'firehose:setup'], teardown: 'component:teardown', }, { diff --git a/tests/playwright/config/component/firehose.setup.ts b/tests/playwright/config/component/firehose.setup.ts new file mode 100644 index 00000000..f7ecf537 --- /dev/null +++ b/tests/playwright/config/component/firehose.setup.ts @@ -0,0 +1,21 @@ +import { test as setup } from '@playwright/test'; +import { + MINIMUM_DESTINATION_BUFFER_INTERVAL, + MINIMUM_PROCESSOR_BUFFER_INTERVAL, + TERRAFORM_DESTINATION_BUFFER_INTERVAL, + TERRAFORM_PROCESSOR_BUFFER_INTERVAL, +} from 'constants/backend-constants'; +import { alterFirehoseBufferIntervals } from 'helpers/data-firehose-helpers'; + +setup('Reduce Firehose buffer intervals', async () => { + await alterFirehoseBufferIntervals({ + expected: { + destination: TERRAFORM_DESTINATION_BUFFER_INTERVAL, + processor: TERRAFORM_PROCESSOR_BUFFER_INTERVAL, + }, + update: { + destination: MINIMUM_DESTINATION_BUFFER_INTERVAL, + processor: MINIMUM_PROCESSOR_BUFFER_INTERVAL, + }, + }); +}); diff --git a/tests/playwright/config/component/firehose.teardown.ts b/tests/playwright/config/component/firehose.teardown.ts new file mode 100644 index 00000000..11844458 --- /dev/null +++ b/tests/playwright/config/component/firehose.teardown.ts @@ -0,0 +1,21 @@ +import { test as teardown } from '@playwright/test'; +import { + MINIMUM_DESTINATION_BUFFER_INTERVAL, + MINIMUM_PROCESSOR_BUFFER_INTERVAL, + TERRAFORM_DESTINATION_BUFFER_INTERVAL, + TERRAFORM_PROCESSOR_BUFFER_INTERVAL, +} from 'constants/backend-constants'; +import { alterFirehoseBufferIntervals } from 'helpers/data-firehose-helpers'; + +teardown('Restore Firehose buffer intervals', async () => { + await alterFirehoseBufferIntervals({ + expected: { + destination: MINIMUM_DESTINATION_BUFFER_INTERVAL, + processor: MINIMUM_PROCESSOR_BUFFER_INTERVAL, + }, + update: { + destination: TERRAFORM_DESTINATION_BUFFER_INTERVAL, + processor: TERRAFORM_PROCESSOR_BUFFER_INTERVAL, + }, + }); +}); diff --git a/tests/playwright/constants/backend-constants.ts b/tests/playwright/constants/backend-constants.ts index 9c98f58b..986060a9 100644 --- a/tests/playwright/constants/backend-constants.ts +++ b/tests/playwright/constants/backend-constants.ts @@ -42,10 +42,16 @@ export const EVENT_BUS_LOG_GROUP_NAME = `/aws/vendedlogs/events/event-bus/${CSI} // DynamoDB export const TTL_TABLE_NAME = `${CSI}-ttl`; +// Glue +export const GLUE_DATABASE_NAME = `${CSI}-reporting`; +export const GLUE_TABLE_NAME = 'event_record'; + // S3 export const LETTERS_S3_BUCKET_NAME = `nhs-${process.env.AWS_ACCOUNT_ID}-${REGION}-${ENV}-dl-letters`; export const NON_PII_S3_BUCKET_NAME = `nhs-${process.env.AWS_ACCOUNT_ID}-${REGION}-${ENV}-dl-non-pii-data`; export const PII_S3_BUCKET_NAME = `nhs-${process.env.AWS_ACCOUNT_ID}-${REGION}-${ENV}-dl-pii-data`; +export const REPORTING_S3_BUCKET_NAME = `nhs-${process.env.AWS_ACCOUNT_ID}-${REGION}-${ENV}-dl-reporting`; +export const REPORTING_S3_KEY_PREFIX = `kinesis-firehose-output/reporting/parquet/${GLUE_TABLE_NAME}`; export const FILE_SAFE_S3_BUCKET_NAME = `nhs-${process.env.AWS_ACCOUNT_ID}-${REGION}-${ENV}-dl-file-safe`; export const UNSCANNED_FILES_S3_BUCKET_NAME = `nhs-${process.env.AWS_ACCOUNT_ID}-${REGION}-main-acct-digi-unscanned-files`; export const FILE_QUARANTINE_S3_BUCKET_NAME = `nhs-${process.env.AWS_ACCOUNT_ID}-${REGION}-${ENV}-dl-file-quarantine`; @@ -61,3 +67,13 @@ export const PRINT_STATUS_HANDLER_LAMBDA_LOG_GROUP_NAME = `/aws/lambda/${CSI}-pr export const PRINT_ANALYSER_LAMBDA_LOG_GROUP_NAME = `/aws/lambda/${CSI}-print-analyser`; export const PRINT_SENDER_LAMBDA_LOG_GROUP_NAME = `/aws/lambda/${CSI}-print-sender`; export const MOVE_SCANNED_FILES_LAMBDA_LOG_GROUP_NAME = `/aws/lambda/${CSI}-move-scanned-files`; + +// Data Firehose +export const FIREHOSE_STREAM_NAME = `${CSI}-to-s3-reporting`; +export const TERRAFORM_DESTINATION_BUFFER_INTERVAL = 300; +export const TERRAFORM_PROCESSOR_BUFFER_INTERVAL = 301; +export const MINIMUM_DESTINATION_BUFFER_INTERVAL = 60; +export const MINIMUM_PROCESSOR_BUFFER_INTERVAL = 0; + +// Athena +export const ATHENA_WORKGROUP_NAME = CSI; diff --git a/tests/playwright/digital-letters-component-tests/report-generator.component.spec.ts b/tests/playwright/digital-letters-component-tests/report-generator.component.spec.ts new file mode 100644 index 00000000..cd9c427d --- /dev/null +++ b/tests/playwright/digital-letters-component-tests/report-generator.component.spec.ts @@ -0,0 +1,85 @@ +import { expect, test } from '@playwright/test'; +import { + ATHENA_WORKGROUP_NAME, + GLUE_DATABASE_NAME, + GLUE_TABLE_NAME, + REPORTING_S3_BUCKET_NAME, + REPORTING_S3_KEY_PREFIX, +} from 'constants/backend-constants'; +import { MESHInboxMessageDownloaded } from 'digital-letters-events'; +import messageDownloadedValidator from 'digital-letters-events/MESHInboxMessageDownloaded.js'; +import { + QueryExecutionState, + getQueryState, + triggerTableMetadataRefresh, +} from 'helpers/athena-helpers'; +import eventPublisher from 'helpers/event-bus-helpers'; +import expectToPassEventually from 'helpers/expectations'; +import { existsInS3 } from 'helpers/s3-helpers'; +import { v4 as uuidv4 } from 'uuid'; + +test.describe('Digital Letters - Report Generator', () => { + test.beforeAll(async () => { + // We need to wait for events to make their way from EventBridge -> Firehose -> S3 -> Glue + test.setTimeout(300_000); + + // Use a random sender ID, so we can be sure that if there are files with this prefix + // in S3 they've been created by this test. + const senderId = `report-generator-test-${uuidv4()}`; + + const eventId = uuidv4(); + await eventPublisher.sendEvents( + [ + { + id: eventId, + specversion: '1.0', + source: + '/nhs/england/notify/production/primary/data-plane/digitalletters/mesh', + subject: + 'customer/920fca11-596a-4eca-9c47-99f624614658/recipient/769acdd4-6a47-496f-999f-76a6fd2c3959', + type: 'uk.nhs.notify.digital.letters.mesh.inbox.message.downloaded.v1', + time: '2023-06-20T12:00:00Z', + recordedtime: '2023-06-20T12:00:00.250Z', + severitynumber: 2, + traceparent: + '00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01', + datacontenttype: 'application/json', + dataschema: + 'https://notify.nhs.uk/cloudevents/schemas/digital-letters/2025-10-draft/data/digital-letters-mesh-inbox-message-downloaded-data.schema.json', + severitytext: 'INFO', + data: { + meshMessageId: '12345', + messageUri: `https://example.com/ttl/resource/${eventId}`, + messageReference: 'ref1', + senderId, + }, + }, + ], + messageDownloadedValidator, + ); + + await expectToPassEventually(async () => { + const eventsHaveBeenWrittenToS3 = await existsInS3( + REPORTING_S3_BUCKET_NAME, + `${REPORTING_S3_KEY_PREFIX}/senderid=${senderId}`, + ); + + expect(eventsHaveBeenWrittenToS3).toBeTruthy(); + }, 300_000); + + // Trigger a metadata refresh for the Glue table, which will cause it to pick up any new files in S3 + const refreshQueryExecutionId = await triggerTableMetadataRefresh( + GLUE_DATABASE_NAME, + GLUE_TABLE_NAME, + ATHENA_WORKGROUP_NAME, + ); + + await expectToPassEventually(async () => { + const refreshQueryState = await getQueryState(refreshQueryExecutionId); + + expect(refreshQueryState).toEqual(QueryExecutionState.SUCCEEDED); + }); + }); + + test('should test something', async () => {}); +}); diff --git a/tests/playwright/helpers/athena-helpers.ts b/tests/playwright/helpers/athena-helpers.ts new file mode 100644 index 00000000..2d15ff24 --- /dev/null +++ b/tests/playwright/helpers/athena-helpers.ts @@ -0,0 +1,61 @@ +import { + AthenaClient, + GetQueryExecutionCommand, + QueryExecutionState, + StartQueryExecutionCommand, +} from '@aws-sdk/client-athena'; + +export { QueryExecutionState } from '@aws-sdk/client-athena'; + +const client = new AthenaClient(); + +/** + * Triggers a metadata refresh for an Athena table using the MSCK REPAIR TABLE command. + * + * This will cause any new files in S3 to be picked up. + * + * @param database - The name of the Athena database + * @param tableName - The name of the table to repair + * @param workgroup - The Athena workgroup to run the query in + * @returns The query execution ID + */ +export async function triggerTableMetadataRefresh( + database: string, + tableName: string, + workgroup: string, +): Promise { + const command = new StartQueryExecutionCommand({ + QueryString: `MSCK REPAIR TABLE ${tableName};`, + QueryExecutionContext: { + Database: database, + Catalog: 'AwsDataCatalog', + }, + WorkGroup: workgroup, + }); + + const response = await client.send(command); + + if (!response.QueryExecutionId) { + throw new Error( + 'Failed to start query execution - no query execution ID returned', + ); + } + + return response.QueryExecutionId; +} + +export async function getQueryState( + queryExecutionId: string, +): Promise { + const queryExecutionInfo = await client.send( + new GetQueryExecutionCommand({ + QueryExecutionId: queryExecutionId, + }), + ); + + if (!queryExecutionInfo.QueryExecution?.Status?.State) { + throw new Error('Failed to get query execution state'); + } + + return queryExecutionInfo.QueryExecution?.Status?.State; +} diff --git a/tests/playwright/helpers/data-firehose-helpers.ts b/tests/playwright/helpers/data-firehose-helpers.ts new file mode 100644 index 00000000..384fef24 --- /dev/null +++ b/tests/playwright/helpers/data-firehose-helpers.ts @@ -0,0 +1,116 @@ +import { + DescribeDeliveryStreamCommand, + DescribeDeliveryStreamCommandOutput, + ExtendedS3DestinationUpdate, + FirehoseClient, + UpdateDestinationCommand, +} from '@aws-sdk/client-firehose'; +import { FIREHOSE_STREAM_NAME, REGION } from 'constants/backend-constants'; + +export async function alterFirehoseBufferIntervals(bufferIntervalConfig: { + expected: { + destination: number; + processor: number; + }; + update: { + destination: number; + processor: number; + }; +}) { + const client = new FirehoseClient({ region: REGION }); + + const deliveryStreamDetails: DescribeDeliveryStreamCommandOutput = + await client.send( + new DescribeDeliveryStreamCommand({ + DeliveryStreamName: FIREHOSE_STREAM_NAME, + }), + ); + + const destinations = + deliveryStreamDetails.DeliveryStreamDescription?.Destinations ?? []; + if (destinations.length !== 1) { + throw new Error('expected a single delivery destination'); + } + + const destination = destinations[0]; + + const currentDestinationBufferInterval = + destination.ExtendedS3DestinationDescription?.BufferingHints + ?.IntervalInSeconds; + + if ( + currentDestinationBufferInterval !== + bufferIntervalConfig.expected.destination + ) { + throw new Error( + `Expected destination buffer size to be ${bufferIntervalConfig.expected.destination} - got ${currentDestinationBufferInterval} - cannot safely alter, has the default value changed in code or manually?`, + ); + } + + const processors = + destination.ExtendedS3DestinationDescription?.ProcessingConfiguration + ?.Processors; + + if (processors?.length !== 1) { + throw new Error('Expected one processor to be configured'); + } + + const processor = processors[0]; + + const currentProcessorBufferInterval = processor.Parameters?.find( + (p) => p.ParameterName === 'BufferIntervalInSeconds', + )?.ParameterValue; + + const otherParams = + processor.Parameters?.filter( + (p) => p.ParameterName !== 'BufferIntervalInSeconds', + ) ?? []; + + if ( + currentProcessorBufferInterval !== + bufferIntervalConfig.expected.processor.toString() + ) { + throw new Error( + `Expected processor buffer size to be ${bufferIntervalConfig.expected.processor} - got ${currentProcessorBufferInterval} - cannot safely alter, has the default value changed in code or manually?`, + ); + } + + const destinationId = destination.DestinationId; + + if (!destinationId) { + throw new Error('Destination ID not found'); + } + + const updatedDestinationConfig: ExtendedS3DestinationUpdate = { + ...destination.ExtendedS3DestinationDescription, + BufferingHints: { + ...destination.ExtendedS3DestinationDescription?.BufferingHints, + IntervalInSeconds: bufferIntervalConfig.update.destination, + }, + ProcessingConfiguration: { + ...destination.ExtendedS3DestinationDescription?.ProcessingConfiguration, + Processors: [ + { + ...processor, + Parameters: [ + ...otherParams, + { + ParameterName: 'BufferIntervalInSeconds', + ParameterValue: bufferIntervalConfig.update.processor.toString(), + }, + ], + }, + ], + }, + }; + + await client.send( + new UpdateDestinationCommand({ + DeliveryStreamName: FIREHOSE_STREAM_NAME, + DestinationId: destinationId, + CurrentDeliveryStreamVersionId: + deliveryStreamDetails.DeliveryStreamDescription?.VersionId, + ExtendedS3DestinationUpdate: updatedDestinationConfig, + }), + ); +} diff --git a/tests/playwright/helpers/s3-helpers.ts b/tests/playwright/helpers/s3-helpers.ts index 108e90c5..b2d6eb0a 100644 --- a/tests/playwright/helpers/s3-helpers.ts +++ b/tests/playwright/helpers/s3-helpers.ts @@ -61,4 +61,12 @@ async function downloadFromS3( }; } -export { downloadFromS3, uploadToS3 }; +async function existsInS3(bucket: string, keyPrefix: string): Promise { + const objects = await s3.send( + new ListObjectsV2Command({ Bucket: bucket, Prefix: keyPrefix }), + ); + + return (objects.Contents?.length ?? 0) > 0; +} + +export { downloadFromS3, existsInS3, uploadToS3 }; diff --git a/tests/playwright/package.json b/tests/playwright/package.json index 156975af..f834d2df 100644 --- a/tests/playwright/package.json +++ b/tests/playwright/package.json @@ -1,7 +1,9 @@ { "dependencies": { + "@aws-sdk/client-athena": "^3.900.0", "@aws-sdk/client-cloudwatch-logs": "^3.900.0", "@aws-sdk/client-dynamodb": "^3.900.0", + "@aws-sdk/client-firehose": "^3.900.0", "@aws-sdk/client-lambda": "^3.900.0", "@aws-sdk/client-s3": "^3.900.0", "@aws-sdk/client-sqs": "^3.900.0",