CLDSRV-632 ✨ put metadata edge cases

[DarkIsDude]

Description

Reactivate some tests that were disable because some edge case were not covered.

Motivation and context

Manage 2 use case under the put metadata route:

• Manage null version when an object is created before bucket versioning and put metadata called after bucket versioning enabled

/* eslint-disable no-console */
const util = require('util');

const { makeBackbeatRequest: makeBackbeatRequestBase } = require('./tests/functional/raw-node/utils/makeRequest');
const { models } = require('arsenal');
const { ObjectMD } = models;
const BucketUtility = require('./tests/functional/aws-node-sdk/lib/utility/bucket-util');

const makeBackbeatRequest = util.promisify(makeBackbeatRequestBase);

function objectMDFromRequestBody(data) {
    const bodyStr = JSON.parse(data.body).Body;
    return new ObjectMD(JSON.parse(bodyStr));
}

async function main() {
    const OBJECT_KEY = 'test-object';
    const BUCKET_NAME = 'test-bucket';
    const TEST_DATA = 'This is a test object for replication';

    const authCredentials = {
        accessKey: 'BJ8Q0L35PRJ92ABA2K0B',
        secretKey: 'kTgcfEaLjxvrLN5EKVcTnb4Ac046FU1m=33/baf1',
    };


    const bucketUtil = new BucketUtility('local-test-vault-s3', { signatureVersion: 'v4' });
    const s3 = bucketUtil.s3;

    console.info('Starting test for object metadata replication...');
    console.info('Checking if bucket exists...');
    await bucketUtil.emptyIfExists(BUCKET_NAME);

    if (await bucketUtil.bucketExists(BUCKET_NAME)) {
        console.info('Deleting bucket...');
        await s3.deleteBucket({ Bucket: BUCKET_NAME }).promise();
    }

    console.info('Creating bucket...');
    await s3.createBucket({ Bucket: BUCKET_NAME }).promise();
 
    console.info('Putting object with versioning disabled...');
    await s3.putObject({ Bucket: BUCKET_NAME, Key: OBJECT_KEY, Body: Buffer.from(TEST_DATA) }).promise();

    console.info('Enabling versioning on bucket...');
    await s3.putBucketVersioning({ Bucket: BUCKET_NAME, VersioningConfiguration: { Status: 'Enabled' } }).promise();
    const versionId = null;
    console.info('Retrieve metadata for the object with versioning enabled...');
    const data = await makeBackbeatRequest({
        method: 'GET',
        resourceType: 'metadata',
        bucket: BUCKET_NAME,
        objectKey: OBJECT_KEY,
        queryObj: {
            versionId,
        },
        authCredentials,
    });

    const objMD = objectMDFromRequestBody(data)
        .setContentLanguage('fr-FR')
        .getSerialized();

    console.info('Object metadata retrieved successfully:', objMD);
    console.info('Updating object metadata...');

    objMD.tags = {
        'fuck': 'test-value',
    };

    await makeBackbeatRequest({
        method: 'PUT',
        resourceType: 'metadata',
        bucket: BUCKET_NAME,
        objectKey: OBJECT_KEY,
        queryObj: {
            versionId,
        },
        authCredentials,
        requestBody: objMD,
    });

    console.info('Object metadata updated successfully.');

    const versions = await s3.listObjectVersions({
        Bucket: BUCKET_NAME,
        Prefix: OBJECT_KEY,
    }).promise();

    console.info({ versions });
}

main();

• Add missing metadata

/* eslint-disable no-console */
const util = require('util');

const { makeBackbeatRequest: makeBackbeatRequestBase } = require('./tests/functional/raw-node/utils/makeRequest');
const { models } = require('arsenal');
const { ObjectMD } = models;
const BucketUtility = require('./tests/functional/aws-node-sdk/lib/utility/bucket-util');

const makeBackbeatRequest = util.promisify(makeBackbeatRequestBase);

/**
 * The final result should be to have two versions of the object:
 * 1. The original version created when versioning was disabled (null versionId)
 * 2. A new version created when versioning was re-enabled (with a VersionId)
 * The new version should have the updated metadata (ContentLanguage set to 'fr-FR')
 * while the original version should remain unchanged.
 **/

const OBJECT_KEY_ROOT = 'test-object-root';
const OBJECT_KEY = 'test-object';
const BUCKET_NAME = 'test-bucket';
const TEST_DATA = 'This is a test object for replication';

const authCredentials = {
    accessKey: 'BJ8Q0L35PRJ92ABA2K0B',
    secretKey: 'kTgcfEaLjxvrLN5EKVcTnb4Ac046FU1m=33/baf1',
};

function objectMDFromRequestBody(data) {
    const bodyStr = JSON.parse(data.body).Body;
    return new ObjectMD(JSON.parse(bodyStr));
}

async function main() {
    const bucketUtil = new BucketUtility('local-test-vault-s3', { signatureVersion: 'v4' });
    const s3 = bucketUtil.s3;

    // await initTest(s3, bucketUtil);
    await updateObjectMetadata(s3);
}

async function initTest(s3, bucketUtil) {
    if (await bucketUtil.bucketExists(BUCKET_NAME)) {
        await bucketUtil.emptyIfExists(BUCKET_NAME);
        await s3.deleteBucket({ Bucket: BUCKET_NAME }).promise();
    }

    await s3.createBucket({ Bucket: BUCKET_NAME }).promise();
    await s3.putBucketVersioning({ Bucket: BUCKET_NAME, VersioningConfiguration: { Status: 'Enabled' } }).promise();
    await s3.putObject({ Bucket: BUCKET_NAME, Key: OBJECT_KEY_ROOT, Body: Buffer.from(TEST_DATA) }).promise();
    await s3.putBucketVersioning({ Bucket: BUCKET_NAME, VersioningConfiguration: { Status: 'Suspended' } }).promise();
    await s3.putObject({ Bucket: BUCKET_NAME, Key: OBJECT_KEY, Body: Buffer.from(TEST_DATA) }).promise();
    await s3.putBucketVersioning({ Bucket: BUCKET_NAME, VersioningConfiguration: { Status: 'Enabled' } }).promise();
}

async function updateObjectMetadata(s3) {
        const data = await makeBackbeatRequest({
        method: 'GET',
        resourceType: 'metadata',
        bucket: BUCKET_NAME,
        objectKey: OBJECT_KEY,
        queryObj: {
            versionId: null,
        },
        authCredentials,
    });

    const objMD = objectMDFromRequestBody(data)
        .setContentLanguage('fr-FR')
        .getSerialized();

    await makeBackbeatRequest({
        method: 'PUT',
        resourceType: 'metadata',
        bucket: BUCKET_NAME,
        objectKey: OBJECT_KEY,
        queryObj: {
            versionId: '393832343336313437373734363139393939393952473030312020333061396264',
        },
        authCredentials,
        requestBody: objMD,
    });

    const versions = await s3.listObjectVersions({
        Bucket: BUCKET_NAME,
        Prefix: OBJECT_KEY,
    }).promise();

    console.info({ Versions: versions.Versions });
 }

async function run() {
    try {
        await main();
    } catch (e) {
        console.error('Error running test:', e);
        process.exit(1);
    }
}

run();

Related issues

https://scality.atlassian.net/browse/CLDSRV-632
scality/Arsenal#2490

[bert-e]

Hello darkisdude,

My role is to assist you with the merge of this
pull request. Please type @bert-e help to get information
on this process, or consult the user documentation.

Available options

name	description	privileged	authored
/after_pull_request	Wait for the given pull request id to be merged before continuing with the current one.
/bypass_author_approval	Bypass the pull request author's approval	⭐
/bypass_build_status	Bypass the build and test status	⭐
/bypass_commit_size	Bypass the check on the size of the changeset TBA	⭐
/bypass_incompatible_branch	Bypass the check on the source branch prefix	⭐
/bypass_jira_check	Bypass the Jira issue check	⭐
/bypass_peer_approval	Bypass the pull request peers' approval	⭐
/bypass_leader_approval	Bypass the pull request leaders' approval	⭐
/approve	Instruct Bert-E that the author has approved the pull request.		✍️
/create_pull_requests	Allow the creation of integration pull requests.
/create_integration_branches	Allow the creation of integration branches.
/no_octopus	Prevent Wall-E from doing any octopus merge and use multiple consecutive merge instead
/unanimity	Change review acceptance criteria from one reviewer at least to all reviewers
/wait	Instruct Bert-E not to run until further notice.

Available commands

name	description	privileged
/help	Print Bert-E's manual in the pull request.
/status	Print Bert-E's current status in the pull request TBA
/clear	Remove all comments from Bert-E from the history TBA
/retry	Re-start a fresh build TBA
/build	Re-start a fresh build TBA
/force_reset	Delete integration branches & pull requests, and restart merge process from the beginning.
/reset	Try to remove integration branches unless there are commits on them which do not appear on the source branch.

Status report is not available.

[bert-e]

Incorrect fix version

The Fix Version/s in issue CLDSRV-632 contains:

• None

Considering where you are trying to merge, I ignored possible hotfix versions and I expected to find:

• 9.0.23

• 9.1.0

Please check the Fix Version/s of CLDSRV-632, or the target
branch of this pull request.

[bert-e]

Incorrect fix version

The Fix Version/s in issue CLDSRV-632 contains:

• None

Considering where you are trying to merge, I ignored possible hotfix versions and I expected to find:

• 9.1.0

Please check the Fix Version/s of CLDSRV-632, or the target
branch of this pull request.

[codecov]

Codecov Report

❌ Patch coverage is 92.00000% with 2 lines in your changes missing coverage. Please review.
✅ Project coverage is 83.83%. Comparing base (d013306) to head (3d7d86a).
⚠️ Report is 21 commits behind head on development/9.1.
✅ All tests successful. No failed tests found.

Files with missing lines	Patch %	Lines
lib/routes/routeBackbeat.js	91.66%	2 Missing ⚠️

Additional details and impacted files

Files with missing lines	Coverage Δ
lib/api/apiUtils/object/createAndStoreObject.js	95.65% <ø> (ø)
lib/api/apiUtils/object/versioning.js	97.10% <100.00%> (+0.01%)	⬆️
lib/routes/routeBackbeat.js	76.33% <91.66%> (+0.53%)	⬆️

@@                 Coverage Diff                 @@
##           development/9.1    #5913      +/-   ##
===================================================
+ Coverage            83.82%   83.83%   +0.01%     
===================================================
  Files                  193      193              
  Lines                12306    12331      +25     
===================================================
+ Hits                 10315    10338      +23     
- Misses                1991     1993       +2     

Flag	Coverage Δ
ceph-backend-test	64.85% <92.00%> (+0.05%)	⬆️
file-ft-tests	67.04% <36.00%> (-0.07%)	⬇️
kmip-ft-tests	27.14% <20.00%> (-0.02%)	⬇️
mongo-v0-ft-tests	68.33% <36.00%> (-0.10%)	⬇️
mongo-v1-ft-tests	68.35% <36.00%> (-0.07%)	⬇️
multiple-backend	34.33% <92.00%> (+0.15%)	⬆️
sur-tests	34.71% <20.00%> (+<0.01%)	⬆️
sur-tests-inflights	36.59% <20.00%> (-0.04%)	⬇️
unit	68.54% <68.00%> (-0.01%)	⬇️
utapi-v2-tests	33.47% <20.00%> (-0.03%)	⬇️

Flags with carried forward coverage won't be shown. Click here to find out more.

🚀 New features to boost your workflow:

• 📦 JS Bundle Analysis: Save yourself from yourself by tracking and limiting bundle sizes in JS merges.

[bert-e]

Incorrect fix version

The Fix Version/s in issue CLDSRV-632 contains:

• None

Considering where you are trying to merge, I ignored possible hotfix versions and I expected to find:

• 9.1.1

Please check the Fix Version/s of CLDSRV-632, or the target
branch of this pull request.

[bert-e]

Waiting for approval

The following approvals are needed before I can proceed with the merge:

• the author

• 2 peers

[bert-e]

Conflict

There is a conflict between your branch feature/CLDSRV-632/put-metadata-edge-cases and the
destination branch development/9.1.

Please resolve the conflict on the feature branch (feature/CLDSRV-632/put-metadata-edge-cases).

git fetch && \
git checkout origin/feature/CLDSRV-632/put-metadata-edge-cases && \
git merge origin/development/9.1

Resolve merge conflicts and commit

git push origin HEAD:feature/CLDSRV-632/put-metadata-edge-cases

[bert-e]

Waiting for approval

The following approvals are needed before I can proceed with the merge:

• the author

• 2 peers

[DarkIsDude]

/approve

[bert-e]

I have successfully merged the changeset of this pull request
into targetted development branches:

• ✔️ development/9.1

The following branches have NOT changed:

• development/7.10
• development/7.4
• development/7.70
• development/8.8
• development/9.0

Please check the status of the associated issue CLDSRV-632.

Goodbye darkisdude.

The following options are set: approve