INTEGRATION [PR#6043 > development/9.2] ✨(CLDSRV-813) update CloudServer XML ListObjectsV2 to support optional attributes

lib/api/bucketGet.js

@@ -138,6 +138,7 @@ function processVersions(bucketName, listParams, list) {
         const objectKey = escapeXmlFn(item.key);
         const isLatest = lastKey !== objectKey;
         lastKey = objectKey;
+
         xml.push(
             v.IsDeleteMarker ? '<DeleteMarker>' : '<Version>',
             `<Key>${objectKey}</Key>`,
@@ -153,14 +154,17 @@ function processVersions(bucketName, listParams, list) {
             `<ID>${v.Owner.ID}</ID>`,
             `<DisplayName>${v.Owner.DisplayName}</DisplayName>`,
             '</Owner>',
+            ...processOptionalAttributes(v, listParams.optionalAttributes),
             `<StorageClass>${v.StorageClass}</StorageClass>`,
             v.IsDeleteMarker ? '</DeleteMarker>' : '</Version>'
         );
     });
+
     list.CommonPrefixes.forEach(item => {
         const val = escapeXmlFn(item);
         xml.push(`<CommonPrefixes><Prefix>${val}</Prefix></CommonPrefixes>`);
     });
+
     xml.push('</ListVersionsResult>');
     return xml.join('');
 }
@@ -224,6 +228,7 @@ function processMasterVersions(bucketName, listParams, list) {
         if (v.isDeleteMarker) {
             return null;
         }
+
         const objectKey = escapeXmlFn(item.key);
         xml.push(
             '<Contents>',
@@ -232,6 +237,7 @@ function processMasterVersions(bucketName, listParams, list) {
             `<ETag>&quot;${v.ETag}&quot;</ETag>`,
             `<Size>${v.Size}</Size>`
         );
+
         if (!listParams.v2 || listParams.fetchOwner) {
             xml.push(
                 '<Owner>',
@@ -240,6 +246,9 @@ function processMasterVersions(bucketName, listParams, list) {
                 '</Owner>'
             );
         }
+
+        xml.push(...processOptionalAttributes(v, listParams.optionalAttributes));
+
         return xml.push(
             `<StorageClass>${v.StorageClass}</StorageClass>`,
             '</Contents>'
@@ -253,20 +262,58 @@ function processMasterVersions(bucketName, listParams, list) {
     return xml.join('');
 }
 
+function processOptionalAttributes(item, optionalAttributes) {
+    const xml = [];
+    const userMetadata = new Set();
+
+    for (const attribute of optionalAttributes) {
+        switch (attribute) {
+            case 'RestoreStatus':
+                xml.push('<RestoreStatus>');
+                xml.push(`<IsRestoreInProgress>${!!item.restoreStatus?.inProgress}</IsRestoreInProgress>`);
+
+                if (item.restoreStatus?.expiryDate) {
+                    xml.push(`<RestoreExpiryDate>${item.restoreStatus?.expiryDate}</RestoreExpiryDate>`);
+                }
+
+                xml.push('</RestoreStatus>');
+                break;
+            case 'x-amz-meta-*':
+                for (const key of Object.keys(item.userMetadata)) {
+                    userMetadata.add(key);
+                }
+                break;
+            default:
+                if (item.userMetadata?.[attribute]) {
+                    userMetadata.add(attribute);
+                }
+        }
+    }
+
+    for (const key of userMetadata) {
+        xml.push(`<${key}>${item.userMetadata[key]}</${key}>`);
+    }
+
+    return xml;
+}
+
 function handleResult(listParams, requestMaxKeys, encoding, authInfo,
     bucketName, list, corsHeaders, log, callback) {
     // eslint-disable-next-line no-param-reassign
     listParams.maxKeys = requestMaxKeys;
     // eslint-disable-next-line no-param-reassign
     listParams.encoding = encoding;
+
     let res;
     if (listParams.listingType === 'DelimiterVersions') {
         res = processVersions(bucketName, listParams, list);
     } else {
         res = processMasterVersions(bucketName, listParams, list);
     }
+
     pushMetric('listBucket', log, { authInfo, bucket: bucketName });
     monitoring.promMetrics('GET', bucketName, '200', 'listBucket');
+
     return callback(null, res, corsHeaders);
 }
 
@@ -286,7 +333,11 @@ function bucketGet(authInfo, request, log, callback) {
     const v2 = params['list-type'];
 
     const optionalAttributes =
-        request.headers['x-amz-optional-object-attributes']?.split(',').map(attr => attr.trim()) ?? [];
+        request.headers['x-amz-optional-object-attributes']
+            ?.split(',')
+            .map(attr => attr.trim())
+            .map(attr => attr !== 'RestoreStatus' ? attr.toLowerCase() : attr)
+        ?? [];
     if (optionalAttributes.some(attr => !attr.startsWith('x-amz-meta-') && attr != 'RestoreStatus')) {
         return callback(
             errorInstances.InvalidArgument.customizeDescription('Invalid attribute name specified')
@@ -344,6 +395,7 @@ function bucketGet(authInfo, request, log, callback) {
         listingType: 'DelimiterMaster',
         maxKeys: actualMaxKeys,
         prefix: params.prefix,
+        optionalAttributes,
     };
 
     if (params.delimiter) {

lib/api/metadataSearch.js

@@ -19,6 +19,8 @@ function handleResult(listParams, requestMaxKeys, encoding, authInfo,
     listParams.maxKeys = requestMaxKeys;
     // eslint-disable-next-line no-param-reassign
     listParams.encoding = encoding;
+    // eslint-disable-next-line no-param-reassign
+    listParams.optionalAttributes = [];
     let res;
     if (listParams.listingType === 'DelimiterVersions') {
         res = processVersions(bucketName, listParams, list);

lib/routes/veeam/list.js

@@ -25,6 +25,7 @@ function buildXMLResponse(request, arrayOfFiles, versioned = false) {
         prefix: validPath,
         maxKeys: parsedQs['max-keys'] || 1000,
         delimiter: '/',
+        optionalAttributes: [],
     };
     const list = {
         IsTruncated: false,

tests/functional/aws-node-sdk/test/bucket/get.js

@@ -1,12 +1,20 @@
 const assert = require('assert');
 const tv4 = require('tv4');
+const { parseString } = require('xml2js');
 
 const withV4 = require('../support/withV4');
 const BucketUtility = require('../../lib/utility/bucket-util');
 const bucketSchema = require('../../schema/bucket');
 const bucketSchemaV2 = require('../../schema/bucketV2');
-const { generateToken, decryptToken } =
-    require('../../../../../lib/api/apiUtils/object/continueToken');
+const { generateToken, decryptToken } = require('../../../../../lib/api/apiUtils/object/continueToken');
+const AWS = require('aws-sdk');
+const { IAM } = AWS;
+const getConfig = require('../support/config');
+const { config } = require('../../../../../lib/Config');
+
+const isVaultScality = config.backends.auth !== 'mem';
+const internalPortBypassBP = config.internalPort;
+const vaultHost = config.vaultd?.host || 'localhost';
 
 const tests = [
     {
@@ -490,5 +498,176 @@ describe('GET Bucket - AWS.S3.listObjects', () => {
                     decryptToken(data.NextContinuationToken), k);
             });
         });
+
+        const describeBypass = isVaultScality && internalPortBypassBP ? describe : describe.skip;
+        describeBypass('x-amz-optional-attributes header', () => {
+            let policyWithoutPermission;
+            let userWithoutPermission;
+            let s3ClientWithoutPermission;
+
+            const iamConfig = getConfig('default', { region: 'us-east-1' });
+            iamConfig.endpoint = `http://${vaultHost}:8600`;
+            const iamClient = new IAM(iamConfig);
+
+            before(async () => {
+                const policyRes = await iamClient
+                    .createPolicy({
+                        PolicyName: 'bp-bypass-policy',
+                        PolicyDocument: JSON.stringify({
+                            Version: '2012-10-17',
+                            Statement: [{
+                                Sid: 'AllowS3ListBucket',
+                                Effect: 'Allow',
+                                Action: [
+                                    's3:ListBucket',
+                                ],
+                                Resource: ['*'],
+                            }],
+                        }),
+                    })
+                    .promise();
+                policyWithoutPermission = policyRes.Policy;
+                const userRes = await iamClient.createUser({ UserName: 'user-without-permission' }).promise();
+                userWithoutPermission = userRes.User;
+                await iamClient
+                    .attachUserPolicy({
+                        UserName: userWithoutPermission.UserName,
+                        PolicyArn: policyWithoutPermission.Arn,
+                    })
+                    .promise();
+
+                const accessKeyRes = await iamClient.createAccessKey({
+                    UserName: userWithoutPermission.UserName,
+                }).promise();
+                const accessKey = accessKeyRes.AccessKey;
+                const s3Config = getConfig('default', {
+                    credentials: new AWS.Credentials(accessKey.AccessKeyId, accessKey.SecretAccessKey),
+                });
+                s3ClientWithoutPermission = new AWS.S3(s3Config);
+            });
+
+            after(async () => {
+                await iamClient
+                    .detachUserPolicy({
+                        UserName: userWithoutPermission.UserName,
+                        PolicyArn: policyWithoutPermission.Arn,
+                    })
+                    .promise();
+                await iamClient.deletePolicy({ PolicyArn: policyWithoutPermission.Arn }).promise();
+                await iamClient.deleteUser({ UserName: userWithoutPermission.UserName }).promise();
+            });
+
+            // eslint-disable-next-line max-len
+            const listObjectsV2WithOptionalAttributes = async (s3, bucket, headerValue) => await new Promise((resolve, reject) => {
+                let rawXml = '';
+                const req = s3.listObjectsV2({ Bucket: bucket });
+
+                req.on('build', () => {
+                    req.httpRequest.headers['x-amz-optional-object-attributes'] = headerValue;
+                });
+                req.on('httpData', chunk => { rawXml += chunk; });
+                req.on('error', err => reject(err));
+                req.on('success', response => {
+                    parseString(rawXml, (err, parsedXml) => {
+                        if (err) {
+                            return reject(err);
+                        }
+
+                        const contents = response.data.Contents;
+                        const parsedContents = parsedXml.ListBucketResult.Contents;
+
+                        if (!contents || !parsedContents) {
+                            return resolve(response.data);
+                        }
+
+                        if (parsedContents[0]?.['x-amz-meta-department']) {
+                            contents[0]['x-amz-meta-department'] = parsedContents[0]['x-amz-meta-department'][0];
+                        }
+
+                        if (parsedContents[0]?.['x-amz-meta-hr']) {
+                            contents[0]['x-amz-meta-hr'] = parsedContents[0]['x-amz-meta-hr'][0];
+                        }
+
+                        return resolve(response.data);
+                    });
+                });
+
+                req.send();
+            });
+
+            it('should return an XML if the header is set', async () => {
+                const s3 = bucketUtil.s3;
+                const Bucket = bucketName;
+
+                await s3.putObject({
+                    Bucket,
+                    Key: 'super-power-object',
+                    Metadata: {
+                        Department: 'sales',
+                        HR: 'true',
+                    },
+                }).promise();
+                const result = await listObjectsV2WithOptionalAttributes(
+                    s3,
+                    Bucket,
+                    'x-amz-meta-*,RestoreStatus,x-amz-meta-department',
+                );
+
+                assert.strictEqual(result.Contents.length, 1);
+                assert.strictEqual(result.Contents[0].Key, 'super-power-object');
+                assert.strictEqual(result.Contents[0]['x-amz-meta-department'], 'sales');
+                assert.strictEqual(result.Contents[0]['x-amz-meta-hr'], 'true');
+            });
+
+            it('should reject the request if the user does not have the permission', async () => {
+                const s3 = bucketUtil.s3;
+                const Bucket = bucketName;
+
+                await s3.putObject({
+                    Bucket,
+                    Key: 'super-power-object',
+                    Metadata: {
+                        Department: 'sales',
+                        HR: 'true',
+                    },
+                }).promise();
+
+                try {
+                    await listObjectsV2WithOptionalAttributes(
+                        s3ClientWithoutPermission,
+                        Bucket,
+                        'x-amz-meta-*,RestoreStatus,x-amz-meta-department',
+                    );
+                    throw new Error('Request should have been rejected');
+                } catch (err) {
+                    assert.strictEqual(err.statusCode, 403);
+                    assert.strictEqual(err.code, 'AccessDenied');
+                }
+            });
+
+            it('should always (ignore permission) return an XML when the header is RestoreStatus', async () => {
+                const s3 = bucketUtil.s3;
+                const Bucket = bucketName;
+
+                await s3.putObject({
+                    Bucket,
+                    Key: 'super-power-object',
+                    Metadata: {
+                        Department: 'sales',
+                        HR: 'true',
+                    },
+                }).promise();
+                const result = await listObjectsV2WithOptionalAttributes(
+                    s3ClientWithoutPermission,
+                    Bucket,
+                    'RestoreStatus',
+                );
+
+                assert.strictEqual(result.Contents.length, 1);
+                assert.strictEqual(result.Contents[0].Key, 'super-power-object');
+                assert.strictEqual(result.Contents[0]['x-amz-meta-department'], undefined);
+                assert.strictEqual(result.Contents[0]['x-amz-meta-hr'], undefined);
+            });
+        });
     });
 });