Improvement/cldsrv 848 add missing checksum algorithms

lib/Config.js

@@ -556,11 +556,15 @@ function parseIntegrityChecks(config) {
         'bucketPutReplication': true,
         'bucketPutVersioning': true,
         'bucketPutWebsite': true,
+        'bucketPutLogging': true,
+        'bucketPutTagging': true,
         'multiObjectDelete': true,
         'objectPutACL': true,
         'objectPutLegalHold': true,
         'objectPutTagging': true,
         'objectPutRetention': true,
+        'objectRestore': true,
+        'completeMultipartUpload': true,
     };
 
     if (config && config.integrityChecks) {

lib/api/api.js

@@ -300,15 +300,19 @@ const api = {
                     }
 
                     const buff = Buffer.concat(post, bodyLength);
+                    return validateMethodChecksumNoChunking(request, buff, log)
+                        .then(error => {
+                            if (error) {
+                                return next(error);
+                            }
 
-                    const err = validateMethodChecksumNoChunking(request, buff, log);
-                    if (err) {
-                        return next(err);
-                    }
-
-                    // Convert array of post buffers into one string
-                    request.post = buff.toString();
-                    return next(null, userInfo, authorizationResults, streamingV4Params, infos);
+                            // Convert array of post buffers into one string
+                            request.post = buff.toString();
+                            return next(null, userInfo, authorizationResults, streamingV4Params, infos);
+                        })
+                        .catch(error => {
+                            next(error);
+                        });
                 });
                 return undefined;
             },

lib/api/apiUtils/integrity/validateChecksums.js

@@ -1,42 +1,178 @@
 const crypto = require('crypto');
+const { Crc32 } = require('@aws-crypto/crc32');
+const { Crc32c } = require('@aws-crypto/crc32c');
+const { CrtCrc64Nvme } = require('@aws-sdk/crc64-nvme-crt');
 const { errors: ArsenalErrors } = require('arsenal');
 const { config } = require('../../../Config');
 
 const ChecksumError = Object.freeze({
     MD5Mismatch: 'MD5Mismatch',
+    XAmzMismatch: 'XAmzMismatch',
     MissingChecksum: 'MissingChecksum',
+    AlgoNotSupported: 'AlgoNotSupported',
+    MultipleChecksumTypes: 'MultipleChecksumTypes',
+    MissingCorresponding: 'MissingCorresponding'
 });
 
+// TEMP
+// https://github.com/aws/aws-sdk-js-v3/issues/6744
+// https://stackoverflow.com/questions/77663519/
+// does-aws-s3-allow-specifying-multiple-checksum-values-crc32-crc32c-sha1-and-s
+// Expecting a single x-amz-checksum- header. Multiple checksum Types are not allowed.
+// XAmzContentChecksumMismatch: The provided 'x-amz-checksum' header does not match what was computed.
+
+// TODO:
+// x-amz-checksum-algorithm x2 different
+// x-amz-checksum-algorithm x2 equal
+// x-amz-checksum-algorithm x2 + x-amz-checksum- valid x2
+// x-amz-checksum-algorithm x2 + x-amz-checksum- invalid x2
+// x-amz-checksum-algorithm x2 + x-amz-checksum- mismatch x2
+
+// What to do about the security vuln package?
+// Should we push only to 9.3???
+
+const algorithms = {
+    'crc64nvme': async data => {
+        const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
+        const crc = new CrtCrc64Nvme();
+        crc.update(input);
+        const result = await crc.digest();
+        return Buffer.from(result).toString('base64');
+    },
+    'crc32': data => {
+        const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
+        return uint32ToBase64(new Crc32().update(input).digest() >>> 0);
+    },
+    'crc32c': data => {
+        const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
+        return uint32ToBase64(new Crc32c().update(input).digest() >>> 0);
+    },
+    'sha1': data => {
+        const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
+       return crypto.createHash('sha1').update(input).digest('base64');
+    },
+    'sha256': data => {
+        const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
+        return crypto.createHash('sha256').update(input).digest('base64');
+    }
+};
+
+function uint32ToBase64(num) {
+    const buf = Buffer.alloc(4);
+    buf.writeUInt32BE(num, 0);
+    return buf.toString('base64');
+}
+
+async function validateXAmzChecksums(headers, body) {
+    const checksumHeaders = Object.keys(headers).filter(header => header.startsWith('x-amz-checksum-'));
+    const xAmzCheckumCnt = checksumHeaders.length;
+    if (xAmzCheckumCnt > 1) {
+        return { error: ChecksumError.MultipleChecksumTypes, details: { algorithms: checksumHeaders } };
+    }
+
+    if ('x-amz-sdk-checksum-algorithm' in headers) {
+        const algo = headers['x-amz-sdk-checksum-algorithm'];
+        if (typeof algo !== 'string') {
+            return { error: ChecksumError.AlgoNotSupported, details: { algorithm: algo } };
+        }
+
+        const lowerAlgo = algo.toLowerCase();
+        if (lowerAlgo in algorithms === false) {
+            return { error: ChecksumError.AlgoNotSupported, details: { algorithm: algo } };
+        }
+
+        if (`x-amz-checksum-${lowerAlgo}` in headers === false) {
+            return { error: ChecksumError.MissingCorresponding, details: { expected: `x-amz-checksum-${lowerAlgo}` } };
+        }
+
+        const expected = headers[`x-amz-checksum-${lowerAlgo}`];
+        const calculated = await algorithms[lowerAlgo](body);
+        // console.log('EXPECTED:', expected, calculated);
+        // console.log('==========', body, calculated, expected, expected !== calculated)
+        if (expected !== calculated) {
+            return { error: ChecksumError.XAmzMismatch, details: { calculated, expected } };
+        }
+
+        return null;
+    }
+
+    if (xAmzCheckumCnt === 0) {
+        return { error: ChecksumError.MissingChecksum, details: null };
+    }
+
+    // No x-amz-sdk-checksum-algorithm we expect one x-amz-checksum-[crc64nvme, crc32, crc32C, sha1, sha256].
+    const algo = checksumHeaders[0].split('-')[3];
+    if (typeof algo !== 'string') {
+        return { error: ChecksumError.AlgoNotSupported, details: { algorithm: algo } };
+    }
+
+    const lowerAlgo = algo.toLowerCase();
+    if (lowerAlgo in algorithms === false) {
+        return { error: ChecksumError.AlgoNotSupported, details: { algorithm: algo } };;
+    }
+
+    const expected = headers[`x-amz-checksum-${lowerAlgo}`];
+    const calculated = await algorithms[lowerAlgo](body);
+    // console.log('==========', body, calculated, expected, expected !== calculated )
+    if (expected !== calculated) {
+        return { error: ChecksumError.XAmzMismatch, details: { calculated, expected } };
+    }
+
+    return null;
+}
+
 /**
  * validateChecksumsNoChunking - Validate the checksums of a request.
  * @param {object} headers - http headers
  * @param {Buffer} body - http request body
  * @return {object} - error
  */
-function validateChecksumsNoChunking(headers, body) {
-    if (headers && 'content-md5' in headers) {
+async function validateChecksumsNoChunking(headers, body) {
+    if (!headers) {
+        return { error: ChecksumError.MissingChecksum, details: null };
+    }
+
+    let md5Present = false;
+    if ('content-md5' in headers) {
         const md5 = crypto.createHash('md5').update(body).digest('base64');
         if (md5 !== headers['content-md5']) {
             return { error: ChecksumError.MD5Mismatch, details: { calculated: md5, expected: headers['content-md5'] } };
         }
 
+        md5Present = true;
+    }
+
+    const err = await validateXAmzChecksums(headers, body);
+    if (err && err.error === ChecksumError.MissingChecksum && md5Present) {
+        // Don't return MissingChecksum if MD5 is present.
         return null;
     }
 
-    return { error: ChecksumError.MissingChecksum, details: null };
+    return err;
 }
 
-function defaultValidationFunc(request, body, log) {
-    const err = validateChecksumsNoChunking(request.headers, body);
+// async function defaultValidationFunc2(request, body, log) { // Rename
+//     const err = await validateChecksumsNoChunking(request.headers, body);
+//     if (err) {
+//         log.debug('failed checksum validation', { method: request.apiMethod, error: err });
+//         return ArsenalErrors.BadDigest; // FIXME: InvalidDigest vs BadDigest
+//     }
+
+//     return null;
+// }
+
+async function defaultValidationFunc(request, body, log) { // Rename
+    const err = await validateChecksumsNoChunking(request.headers, body);
     if (err && err.error !== ChecksumError.MissingChecksum) {
         log.debug('failed checksum validation', { method: request.apiMethod }, err);
-        return ArsenalErrors.BadDigest;
+        return ArsenalErrors.BadDigest; // FIXME: InvalidDigest vs BadDigest
     }
 
     return null;
 }
 
 const methodValidationFunc = Object.freeze({
+    'completeMultipartUpload': defaultValidationFunc,
     'bucketPutACL': defaultValidationFunc,
     'bucketPutCors': defaultValidationFunc,
     'bucketPutEncryption': defaultValidationFunc,
@@ -47,12 +183,15 @@ const methodValidationFunc = Object.freeze({
     'bucketPutReplication': defaultValidationFunc,
     'bucketPutVersioning': defaultValidationFunc,
     'bucketPutWebsite': defaultValidationFunc,
+    'bucketPutLogging': defaultValidationFunc,
+    'bucketPutTagging': defaultValidationFunc,
     // TODO: DeleteObjects requires a checksum. Should return an error if ChecksumError.MissingChecksum.
     'multiObjectDelete': defaultValidationFunc,
     'objectPutACL': defaultValidationFunc,
     'objectPutLegalHold': defaultValidationFunc,
     'objectPutTagging': defaultValidationFunc,
     'objectPutRetention': defaultValidationFunc,
+    'objectRestore': defaultValidationFunc,
 });
 
 /**
@@ -62,14 +201,13 @@ const methodValidationFunc = Object.freeze({
  * @param {object} log - logger
  * @return {object} - error
  */
-function validateMethodChecksumNoChunking(request, body, log) {
+async function validateMethodChecksumNoChunking(request, body, log) {
     if (config.integrityChecks[request.apiMethod]) {
         const validationFunc = methodValidationFunc[request.apiMethod];
         if (!validationFunc) {
-            return null;
+            return null; //await defaultValidationFunc2(request, body, log);
         }
-
-        return validationFunc(request, body, log);
+        return await validationFunc(request, body, log);
     }
 
     return null;

package.json

@@ -19,6 +19,9 @@
   },
   "homepage": "https://github.com/scality/S3#readme",
   "dependencies": {
+    "@aws-crypto/crc32": "^5.2.0",
+    "@aws-crypto/crc32c": "^5.2.0",
+    "@aws-sdk/crc64-nvme-crt": "^3.989.0",
     "@azure/storage-blob": "^12.28.0",
     "@hapi/joi": "^17.1.1",
     "arsenal": "git+https://github.com/scality/arsenal#8.2.46",