diff --git a/lib/api/bucketGet.js b/lib/api/bucketGet.js index 64b5322fdd..3f5e352b75 100644 --- a/lib/api/bucketGet.js +++ b/lib/api/bucketGet.js @@ -138,6 +138,7 @@ function processVersions(bucketName, listParams, list) { const objectKey = escapeXmlFn(item.key); const isLatest = lastKey !== objectKey; lastKey = objectKey; + xml.push( v.IsDeleteMarker ? '' : '', `${objectKey}`, @@ -153,14 +154,17 @@ function processVersions(bucketName, listParams, list) { `${v.Owner.ID}`, `${v.Owner.DisplayName}`, '', + ...processOptionalAttributes(v, listParams.optionalAttributes), `${v.StorageClass}`, v.IsDeleteMarker ? '' : '' ); }); + list.CommonPrefixes.forEach(item => { const val = escapeXmlFn(item); xml.push(`${val}`); }); + xml.push(''); return xml.join(''); } @@ -224,6 +228,7 @@ function processMasterVersions(bucketName, listParams, list) { if (v.isDeleteMarker) { return null; } + const objectKey = escapeXmlFn(item.key); xml.push( '', @@ -232,6 +237,7 @@ function processMasterVersions(bucketName, listParams, list) { `"${v.ETag}"`, `${v.Size}` ); + if (!listParams.v2 || listParams.fetchOwner) { xml.push( '', @@ -240,6 +246,9 @@ function processMasterVersions(bucketName, listParams, list) { '' ); } + + xml.push(...processOptionalAttributes(v, listParams.optionalAttributes)); + return xml.push( `${v.StorageClass}`, '' @@ -253,20 +262,58 @@ function processMasterVersions(bucketName, listParams, list) { return xml.join(''); } +function processOptionalAttributes(item, optionalAttributes) { + const xml = []; + const userMetadata = new Set(); + + for (const attribute of optionalAttributes) { + switch (attribute) { + case 'RestoreStatus': + xml.push(''); + xml.push(`${!!item.restoreStatus?.inProgress}`); + + if (item.restoreStatus?.expiryDate) { + xml.push(`${item.restoreStatus?.expiryDate}`); + } + + xml.push(''); + break; + case 'x-amz-meta-*': + for (const key of Object.keys(item.userMetadata)) { + userMetadata.add(key); + } + break; + default: + if (item.userMetadata?.[attribute]) { + userMetadata.add(attribute); + } + } + } + + for (const key of userMetadata) { + xml.push(`<${key}>${item.userMetadata[key]}`); + } + + return xml; +} + function handleResult(listParams, requestMaxKeys, encoding, authInfo, bucketName, list, corsHeaders, log, callback) { // eslint-disable-next-line no-param-reassign listParams.maxKeys = requestMaxKeys; // eslint-disable-next-line no-param-reassign listParams.encoding = encoding; + let res; if (listParams.listingType === 'DelimiterVersions') { res = processVersions(bucketName, listParams, list); } else { res = processMasterVersions(bucketName, listParams, list); } + pushMetric('listBucket', log, { authInfo, bucket: bucketName }); monitoring.promMetrics('GET', bucketName, '200', 'listBucket'); + return callback(null, res, corsHeaders); } @@ -286,7 +333,11 @@ function bucketGet(authInfo, request, log, callback) { const v2 = params['list-type']; const optionalAttributes = - request.headers['x-amz-optional-object-attributes']?.split(',').map(attr => attr.trim()) ?? []; + request.headers['x-amz-optional-object-attributes'] + ?.split(',') + .map(attr => attr.trim()) + .map(attr => attr !== 'RestoreStatus' ? attr.toLowerCase() : attr) + ?? []; if (optionalAttributes.some(attr => !attr.startsWith('x-amz-meta-') && attr != 'RestoreStatus')) { return callback( errorInstances.InvalidArgument.customizeDescription('Invalid attribute name specified') @@ -336,6 +387,7 @@ function bucketGet(authInfo, request, log, callback) { listingType: 'DelimiterMaster', maxKeys: actualMaxKeys, prefix: params.prefix, + optionalAttributes, }; if (params.delimiter) { diff --git a/lib/api/metadataSearch.js b/lib/api/metadataSearch.js index 959068eee0..900b0cceef 100644 --- a/lib/api/metadataSearch.js +++ b/lib/api/metadataSearch.js @@ -19,6 +19,8 @@ function handleResult(listParams, requestMaxKeys, encoding, authInfo, listParams.maxKeys = requestMaxKeys; // eslint-disable-next-line no-param-reassign listParams.encoding = encoding; + // eslint-disable-next-line no-param-reassign + listParams.optionalAttributes = []; let res; if (listParams.listingType === 'DelimiterVersions') { res = processVersions(bucketName, listParams, list); diff --git a/lib/routes/veeam/list.js b/lib/routes/veeam/list.js index 1fcb2e0158..5fd8585bfb 100644 --- a/lib/routes/veeam/list.js +++ b/lib/routes/veeam/list.js @@ -25,6 +25,7 @@ function buildXMLResponse(request, arrayOfFiles, versioned = false) { prefix: validPath, maxKeys: parsedQs['max-keys'] || 1000, delimiter: '/', + optionalAttributes: [], }; const list = { IsTruncated: false, diff --git a/package.json b/package.json index df84ef014b..5eeedc211d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@zenko/cloudserver", - "version": "9.1.10", + "version": "9.1.11", "description": "Zenko CloudServer, an open-source Node.js implementation of a server handling the Amazon S3 protocol", "main": "index.js", "engines": { diff --git a/tests/functional/aws-node-sdk/test/bucket/get.js b/tests/functional/aws-node-sdk/test/bucket/get.js index 0267ad7a65..3f17a7ede4 100644 --- a/tests/functional/aws-node-sdk/test/bucket/get.js +++ b/tests/functional/aws-node-sdk/test/bucket/get.js @@ -1,12 +1,20 @@ const assert = require('assert'); const tv4 = require('tv4'); +const { parseString } = require('xml2js'); const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); const bucketSchema = require('../../schema/bucket'); const bucketSchemaV2 = require('../../schema/bucketV2'); -const { generateToken, decryptToken } = - require('../../../../../lib/api/apiUtils/object/continueToken'); +const { generateToken, decryptToken } = require('../../../../../lib/api/apiUtils/object/continueToken'); +const AWS = require('aws-sdk'); +const { IAM } = AWS; +const getConfig = require('../support/config'); +const { config } = require('../../../../../lib/Config'); + +const isVaultScality = config.backends.auth !== 'mem'; +const internalPortBypassBP = config.internalPort; +const vaultHost = config.vaultd?.host || 'localhost'; const tests = [ { @@ -490,5 +498,176 @@ describe('GET Bucket - AWS.S3.listObjects', () => { decryptToken(data.NextContinuationToken), k); }); }); + + const describeBypass = isVaultScality && internalPortBypassBP ? describe : describe.skip; + describeBypass('x-amz-optional-attributes header', () => { + let policyWithoutPermission; + let userWithoutPermission; + let s3ClientWithoutPermission; + + const iamConfig = getConfig('default', { region: 'us-east-1' }); + iamConfig.endpoint = `http://${vaultHost}:8600`; + const iamClient = new IAM(iamConfig); + + before(async () => { + const policyRes = await iamClient + .createPolicy({ + PolicyName: 'bp-bypass-policy', + PolicyDocument: JSON.stringify({ + Version: '2012-10-17', + Statement: [{ + Sid: 'AllowS3ListBucket', + Effect: 'Allow', + Action: [ + 's3:ListBucket', + ], + Resource: ['*'], + }], + }), + }) + .promise(); + policyWithoutPermission = policyRes.Policy; + const userRes = await iamClient.createUser({ UserName: 'user-without-permission' }).promise(); + userWithoutPermission = userRes.User; + await iamClient + .attachUserPolicy({ + UserName: userWithoutPermission.UserName, + PolicyArn: policyWithoutPermission.Arn, + }) + .promise(); + + const accessKeyRes = await iamClient.createAccessKey({ + UserName: userWithoutPermission.UserName, + }).promise(); + const accessKey = accessKeyRes.AccessKey; + const s3Config = getConfig('default', { + credentials: new AWS.Credentials(accessKey.AccessKeyId, accessKey.SecretAccessKey), + }); + s3ClientWithoutPermission = new AWS.S3(s3Config); + }); + + after(async () => { + await iamClient + .detachUserPolicy({ + UserName: userWithoutPermission.UserName, + PolicyArn: policyWithoutPermission.Arn, + }) + .promise(); + await iamClient.deletePolicy({ PolicyArn: policyWithoutPermission.Arn }).promise(); + await iamClient.deleteUser({ UserName: userWithoutPermission.UserName }).promise(); + }); + + // eslint-disable-next-line max-len + const listObjectsV2WithOptionalAttributes = async (s3, bucket, headerValue) => await new Promise((resolve, reject) => { + let rawXml = ''; + const req = s3.listObjectsV2({ Bucket: bucket }); + + req.on('build', () => { + req.httpRequest.headers['x-amz-optional-object-attributes'] = headerValue; + }); + req.on('httpData', chunk => { rawXml += chunk; }); + req.on('error', err => reject(err)); + req.on('success', response => { + parseString(rawXml, (err, parsedXml) => { + if (err) { + return reject(err); + } + + const contents = response.data.Contents; + const parsedContents = parsedXml.ListBucketResult.Contents; + + if (!contents || !parsedContents) { + return resolve(response.data); + } + + if (parsedContents[0]?.['x-amz-meta-department']) { + contents[0]['x-amz-meta-department'] = parsedContents[0]['x-amz-meta-department'][0]; + } + + if (parsedContents[0]?.['x-amz-meta-hr']) { + contents[0]['x-amz-meta-hr'] = parsedContents[0]['x-amz-meta-hr'][0]; + } + + return resolve(response.data); + }); + }); + + req.send(); + }); + + it('should return an XML if the header is set', async () => { + const s3 = bucketUtil.s3; + const Bucket = bucketName; + + await s3.putObject({ + Bucket, + Key: 'super-power-object', + Metadata: { + Department: 'sales', + HR: 'true', + }, + }).promise(); + const result = await listObjectsV2WithOptionalAttributes( + s3, + Bucket, + 'x-amz-meta-*,RestoreStatus,x-amz-meta-department', + ); + + assert.strictEqual(result.Contents.length, 1); + assert.strictEqual(result.Contents[0].Key, 'super-power-object'); + assert.strictEqual(result.Contents[0]['x-amz-meta-department'], 'sales'); + assert.strictEqual(result.Contents[0]['x-amz-meta-hr'], 'true'); + }); + + it('should reject the request if the user does not have the permission', async () => { + const s3 = bucketUtil.s3; + const Bucket = bucketName; + + await s3.putObject({ + Bucket, + Key: 'super-power-object', + Metadata: { + Department: 'sales', + HR: 'true', + }, + }).promise(); + + try { + await listObjectsV2WithOptionalAttributes( + s3ClientWithoutPermission, + Bucket, + 'x-amz-meta-*,RestoreStatus,x-amz-meta-department', + ); + throw new Error('Request should have been rejected'); + } catch (err) { + assert.strictEqual(err.statusCode, 403); + assert.strictEqual(err.code, 'AccessDenied'); + } + }); + + it('should always (ignore permission) return an XML when the header is RestoreStatus', async () => { + const s3 = bucketUtil.s3; + const Bucket = bucketName; + + await s3.putObject({ + Bucket, + Key: 'super-power-object', + Metadata: { + Department: 'sales', + HR: 'true', + }, + }).promise(); + const result = await listObjectsV2WithOptionalAttributes( + s3ClientWithoutPermission, + Bucket, + 'RestoreStatus', + ); + + assert.strictEqual(result.Contents.length, 1); + assert.strictEqual(result.Contents[0].Key, 'super-power-object'); + assert.strictEqual(result.Contents[0]['x-amz-meta-department'], undefined); + assert.strictEqual(result.Contents[0]['x-amz-meta-hr'], undefined); + }); + }); }); }); diff --git a/tests/unit/api/bucketGet.js b/tests/unit/api/bucketGet.js index 48befeafef..2ed7026c5f 100644 --- a/tests/unit/api/bucketGet.js +++ b/tests/unit/api/bucketGet.js @@ -439,5 +439,130 @@ describe('bucketGet API V2', () => { done(); }); }); + + it('should return user metadata if requested and present', done => { + const objectNameMeta = 'objectWithMeta'; + const putRequest = new DummyRequest({ + bucketName, + headers: { 'x-amz-meta-color': 'red' }, + url: `/${bucketName}/${objectNameMeta}`, + namespace, + objectKey: objectNameMeta, + }, postBody); + + const testGetRequest = Object.assign({ + query: {}, + url: baseUrl, + }, baseGetRequest); + testGetRequest.headers['x-amz-optional-object-attributes'] = 'x-amz-meta-color'; + + async.waterfall([ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (_, next) => objectPut(authInfo, putRequest, undefined, log, next), + (_, next) => bucketGet(authInfo, testGetRequest, log, next), + (result, _, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual(err, null); + const content = result.ListBucketResult.Contents[0]; + assert.strictEqual(content.Key[0], objectNameMeta); + assert.strictEqual(content['x-amz-meta-color'][0], 'red'); + done(); + }); + }); + + it('should return all user metadata if wildcard requested', done => { + const objectNameMeta = 'objectWithMetaWildcard'; + const putRequest = new DummyRequest({ + bucketName, + headers: { 'x-amz-meta-color': 'red', 'x-amz-meta-size': 'large' }, + url: `/${bucketName}/${objectNameMeta}`, + namespace, + objectKey: objectNameMeta, + }, postBody); + + const testGetRequest = Object.assign({ + query: {}, + url: baseUrl, + }, baseGetRequest); + testGetRequest.headers['x-amz-optional-object-attributes'] = 'x-amz-meta-*'; + + async.waterfall([ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (_, next) => objectPut(authInfo, putRequest, undefined, log, next), + (_, next) => bucketGet(authInfo, testGetRequest, log, next), + (result, _, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual(err, null); + const content = result.ListBucketResult.Contents[0]; + assert.strictEqual(content.Key[0], objectNameMeta); + assert.strictEqual(content['x-amz-meta-color'][0], 'red'); + assert.strictEqual(content['x-amz-meta-size'][0], 'large'); + done(); + }); + }); + + it('should return user metadata in version listing if requested', done => { + const objectNameMeta = 'objectWithMetaVersion'; + const putRequest = new DummyRequest({ + bucketName, + headers: { 'x-amz-meta-ver': '1' }, + url: `/${bucketName}/${objectNameMeta}`, + namespace, + objectKey: objectNameMeta, + }, postBody); + + const testGetRequest = Object.assign({ + query: { versions: '' }, + url: `${baseUrl}?versions`, + }, baseGetRequest); + testGetRequest.headers['x-amz-optional-object-attributes'] = 'x-amz-meta-ver'; + + async.waterfall([ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (_, next) => objectPut(authInfo, putRequest, undefined, log, next), + (_, next) => bucketGet(authInfo, testGetRequest, log, next), + (result, _, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual(err, null); + const version = result.ListVersionsResult.Version[0]; + assert.strictEqual(version.Key[0], objectNameMeta); + assert.strictEqual(version['x-amz-meta-ver'][0], '1'); + done(); + }); + }); + + it('should return user metadata as case insentive (lowercase header)', done => { + const objectNameMeta = 'objectWithMeta'; + const putRequest = new DummyRequest({ + bucketName, + headers: { 'x-amz-meta-color': 'yellow' }, + url: `/${bucketName}/${objectNameMeta}`, + namespace, + objectKey: objectNameMeta, + }, postBody); + + const testGetRequest = Object.assign({ + query: {}, + url: baseUrl, + }, baseGetRequest); + testGetRequest.headers['x-amz-optional-object-attributes'] = 'x-amz-meta-coLor'; + + async.waterfall([ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (_, next) => objectPut(authInfo, putRequest, undefined, log, next), + (_, next) => bucketGet(authInfo, testGetRequest, log, next), + (result, _, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual(err, null); + const content = result.ListBucketResult.Contents[0]; + assert.strictEqual(content.Key[0], objectNameMeta); + assert.strictEqual(content['x-amz-meta-color'][0], 'yellow'); + done(); + }); + }); }); });