Skip to content

Commit

Permalink
Merge branch 'w/8.7/feature/CLDSRV-420/backport' into tmp/octopus/w/8…
Browse files Browse the repository at this point in the history 
….8/feature/CLDSRV-420/backport
  • Loading branch information
@bert-e
bert-e committed Aug 18, 2023
2 parents 19d3e0b + 858c31a commit 18bf6b8
Show file tree
Hide file tree
Showing 12 changed files with 294 additions and 41 deletions.
6 changes: 3 additions & 3 deletions lib/api/apiUtils/object/lifecycle.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ function processCurrents(bucketName, listParams, isBucketVersioned, list) {
Size: v.Size,
Owner: {
ID: v.Owner.ID,
DisplayName: v.Owner.DisplayName
DisplayName: v.Owner.DisplayName,
},
StorageClass: v.StorageClass,
TagSet: _makeTags(v.tags),
Expand Down Expand Up @@ -98,7 +98,7 @@ function processNonCurrents(bucketName, listParams, list) {
Size: v.Size,
Owner: {
ID: v.Owner.ID,
DisplayName: v.Owner.DisplayName
DisplayName: v.Owner.DisplayName,
},
StorageClass: v.StorageClass,
TagSet: _makeTags(v.tags),
Expand Down Expand Up @@ -135,7 +135,7 @@ function processOrphans(bucketName, listParams, list) {
LastModified: v.LastModified,
Owner: {
ID: v.Owner.ID,
DisplayName: v.Owner.DisplayName
DisplayName: v.Owner.DisplayName,
},
VersionId: versionId,
IsLatest: true, // for compatibility with AWS ListObjectVersions.
Expand Down
2 changes: 1 addition & 1 deletion lib/api/backbeat/listLifecycleCurrents.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ function listLifecycleCurrents(authInfo, locationConstraints, request, log, call
return services.getLifecycleListing(bucketName, listParams, log,
(err, list) => {
if (err) {
log.debug('error processing request', { method: 'services.getLifecycleListing', error: err });
log.debug('error processing request', { method: 'services.getLifecycleListing', error: err });
monitoring.promMetrics(
'GET', bucketName, err.code, 'listLifecycleCurrents');
return callback(err, null);
Expand Down
2 changes: 1 addition & 1 deletion lib/api/backbeat/listLifecycleNonCurrents.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ function listLifecycleNonCurrents(authInfo, locationConstraints, request, log, c

return metadataValidateBucket(metadataValParams, log, (err, bucket) => {
if (err) {
log.debug('error processing request', { method: 'metadataValidateBucket', error: err });
log.debug('error processing request', { method: 'metadataValidateBucket', error: err });
monitoring.promMetrics(
'GET', bucketName, err.code, 'listLifecycleNonCurrents');
return callback(err, null);
Expand Down
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
"dependencies": {
"@azure/storage-blob": "^12.12.0",
"@hapi/joi": "^17.1.0",
"arsenal": "git+https://github.com/scality/arsenal#8.1.105",
"arsenal": "git+https://github.com/scality/arsenal#8.1.108",
"async": "~2.5.0",
"aws-sdk": "2.905.0",
"bucketclient": "scality/bucketclient#8.1.9",
Expand Down
6 changes: 3 additions & 3 deletions tests/functional/backbeat/excludedDataStoreName.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ const assert = require('assert');
const async = require('async');
const BucketUtility = require('../aws-node-sdk/lib/utility/bucket-util');
const { removeAllVersions } = require('../aws-node-sdk/lib/utility/versioning-util');
const { makeBackbeatRequest, runIfMongoV1, updateMetadata } = require('./utils');
const { makeBackbeatRequest, updateMetadata } = require('./utils');

const testBucket = 'bucket-for-list-lifecycle-current-tests';
const location1 = 'us-east-1';
Expand All @@ -13,7 +13,7 @@ const credentials = {
secretKey: 'verySecretKey1',
};

runIfMongoV1('excludedDataStoreName', () => {
describe('excludedDataStoreName', () => {
let bucketUtil;
let s3;
const expectedVersions = [];
Expand Down Expand Up @@ -145,7 +145,7 @@ runIfMongoV1('excludedDataStoreName', () => {
assert.strictEqual(contents[0].Key, 'key0');
assert.strictEqual(contents[0].DataStoreName, location1);

return makeBackbeatRequest({
return makeBackbeatRequest({
method: 'GET',
bucket: testBucket,
queryObj: {
Expand Down
6 changes: 3 additions & 3 deletions tests/functional/backbeat/listDeleteMarker.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,14 @@ const assert = require('assert');
const async = require('async');
const BucketUtility = require('../aws-node-sdk/lib/utility/bucket-util');
const { removeAllVersions } = require('../aws-node-sdk/lib/utility/versioning-util');
const { makeBackbeatRequest, runIfMongoV1 } = require('./utils');
const { makeBackbeatRequest } = require('./utils');

const credentials = {
accessKey: 'accessKey1',
secretKey: 'verySecretKey1',
};

runIfMongoV1('listLifecycle with non-current delete marker', () => {
describe('listLifecycle with non-current delete marker', () => {
let bucketUtil;
let s3;
let expectedVersionId;
Expand Down Expand Up @@ -113,7 +113,7 @@ runIfMongoV1('listLifecycle with non-current delete marker', () => {
});
});

runIfMongoV1('listLifecycle with current delete marker version', () => {
describe('listLifecycle with current delete marker version', () => {
let bucketUtil;
let s3;
let expectedVersionId;
Expand Down
261 changes: 255 additions & 6 deletions tests/functional/backbeat/listLifecycleCurrents.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,7 @@ const assert = require('assert');
const async = require('async');
const BucketUtility = require('../aws-node-sdk/lib/utility/bucket-util');
const { removeAllVersions } = require('../aws-node-sdk/lib/utility/versioning-util');
const { makeBackbeatRequest, runIfMongoV1 } = require('./utils');

const testBucket = 'bucket-for-list-lifecycle-current-tests';
const emptyBucket = 'empty-bucket-for-list-lifecycle-current-tests';
const { makeBackbeatRequest } = require('./utils');

const credentials = {
accessKey: 'accessKey1',
Expand Down Expand Up @@ -39,7 +36,10 @@ function checkContents(contents, expectedKeyVersions) {
}

['Enabled', 'Disabled'].forEach(versioning => {
runIfMongoV1(`listLifecycleCurrents with bucket versioning ${versioning}`, () => {
describe(`listLifecycleCurrents with bucket versioning ${versioning}`, () => {
const testBucket = `bucket-for-list-lifecycle-current-tests-${versioning.toLowerCase()}`;
const emptyBucket = `empty-bucket-for-list-lifecycle-current-tests-${versioning.toLowerCase()}`;

let bucketUtil;
let s3;
let date;
Expand Down Expand Up @@ -127,7 +127,7 @@ function checkContents(contents, expectedKeyVersions) {
makeBackbeatRequest({
method: 'GET',
bucket: testBucket,
queryObj: { 'list-type': 'current', prefix: 'unknown' },
queryObj: { 'list-type': 'current', 'prefix': 'unknown' },
authCredentials: credentials,
}, (err, response) => {
assert.ifError(err);
Expand Down Expand Up @@ -335,3 +335,252 @@ function checkContents(contents, expectedKeyVersions) {
});
});
});

describe('listLifecycleCurrents with bucket versioning enabled and maxKeys', () => {
const testBucket = 'bucket-for-list-lifecycle-current-tests-truncated';
let bucketUtil;
let s3;
const expectedKeyVersions = {};

before(done => {
bucketUtil = new BucketUtility('account1', { signatureVersion: 'v4' });
s3 = bucketUtil.s3;

return async.series([
next => s3.createBucket({ Bucket: testBucket }, next),
next => s3.putBucketVersioning({
Bucket: testBucket,
VersioningConfiguration: { Status: 'Enabled' },
}, next),
next => async.times(3, (n, cb) => {
const keyName = 'key0';
s3.putObject({ Bucket: testBucket, Key: keyName, Body: '123', Tagging: 'mykey=myvalue' },
(err, data) => {
if (err) {
cb(err);
}
expectedKeyVersions[keyName] = data.VersionId;
return cb();
});
}, next),
next => async.times(5, (n, cb) => {
const keyName = 'key1';
s3.putObject({ Bucket: testBucket, Key: keyName, Body: '123', Tagging: 'mykey=myvalue' },
(err, data) => {
if (err) {
cb(err);
}
expectedKeyVersions[keyName] = data.VersionId;
return cb();
});
}, next),
next => async.times(3, (n, cb) => {
const keyName = 'key2';
s3.putObject({ Bucket: testBucket, Key: keyName, Body: '123', Tagging: 'mykey=myvalue' },
(err, data) => {
if (err) {
cb(err);
}
expectedKeyVersions[keyName] = data.VersionId;
return cb();
});
}, next),
], done);
});

after(done => async.series([
next => removeAllVersions({ Bucket: testBucket }, next),
next => s3.deleteBucket({ Bucket: testBucket }, next),
], done));

it('should return truncated lists - part 1', done => {
makeBackbeatRequest({
method: 'GET',
bucket: testBucket,
queryObj: { 'list-type': 'current', 'max-keys': '1' },
authCredentials: credentials,
}, (err, response) => {
assert.ifError(err);
assert.strictEqual(response.statusCode, 200);
const data = JSON.parse(response.body);

assert.strictEqual(data.IsTruncated, true);
assert.strictEqual(data.NextMarker, 'key0');
assert.strictEqual(data.MaxKeys, 1);
assert.strictEqual(data.Contents.length, 1);

const contents = data.Contents;
assert.strictEqual(contents.length, 1);
checkContents(contents, expectedKeyVersions);
assert.strictEqual(contents[0].Key, 'key0');
return done();
});
});

it('should return truncated lists - part 2', done => {
makeBackbeatRequest({
method: 'GET',
bucket: testBucket,
queryObj: {
'list-type': 'current',
'max-keys': '1',
'marker': 'key0',
},
authCredentials: credentials,
}, (err, response) => {
assert.ifError(err);
assert.strictEqual(response.statusCode, 200);
const data = JSON.parse(response.body);

assert.strictEqual(data.Marker, 'key0');
assert.strictEqual(data.IsTruncated, true);
assert.strictEqual(data.NextMarker, 'key1');
assert.strictEqual(data.MaxKeys, 1);
assert.strictEqual(data.Contents.length, 1);

const contents = data.Contents;
assert.strictEqual(contents.length, 1);
checkContents(contents, expectedKeyVersions);
assert.strictEqual(contents[0].Key, 'key1');
return done();
});
});

it('should return truncated lists - part 3', done => {
makeBackbeatRequest({
method: 'GET',
bucket: testBucket,
queryObj: {
'list-type': 'current',
'max-keys': '1',
'marker': 'key1',
},
authCredentials: credentials,
}, (err, response) => {
assert.ifError(err);
assert.strictEqual(response.statusCode, 200);
const data = JSON.parse(response.body);

assert(!data.NextMarker);
assert.strictEqual(data.IsTruncated, false);
assert.strictEqual(data.Marker, 'key1');
assert.strictEqual(data.MaxKeys, 1);
assert.strictEqual(data.Contents.length, 1);

const contents = data.Contents;
assert.strictEqual(contents.length, 1);
checkContents(contents, expectedKeyVersions);
assert.strictEqual(contents[0].Key, 'key2');
return done();
});
});
});


describe('listLifecycleCurrents with bucket versioning enabled and delete object', () => {
const testBucket = 'bucket-for-list-lifecycle-current-tests-truncated';
const keyName0 = 'key0';
const keyName1 = 'key1';
const keyName2 = 'key2';
let bucketUtil;
let s3;
const expectedKeyVersions = {};

before(done => {
bucketUtil = new BucketUtility('account1', { signatureVersion: 'v4' });
s3 = bucketUtil.s3;

return async.series([
next => s3.createBucket({ Bucket: testBucket }, next),
next => s3.putBucketVersioning({
Bucket: testBucket,
VersioningConfiguration: { Status: 'Enabled' },
}, next),
next => {
s3.putObject({ Bucket: testBucket, Key: keyName0, Body: '123', Tagging: 'mykey=myvalue' },
(err, data) => {
if (err) {
next(err);
}
expectedKeyVersions[keyName0] = data.VersionId;
return next();
});
},
next => s3.putObject({ Bucket: testBucket, Key: keyName1, Body: '123', Tagging: 'mykey=myvalue' }, next),
next => s3.deleteObject({ Bucket: testBucket, Key: keyName1 }, next),
next => s3.putObject({ Bucket: testBucket, Key: keyName2, Body: '123', Tagging: 'mykey=myvalue' },
(err, data) => {
if (err) {
next(err);
}
expectedKeyVersions[keyName2] = data.VersionId;
return next();
}),
next => s3.putObject({ Bucket: testBucket, Key: keyName2, Body: '123', Tagging: 'mykey=myvalue' },
(err, data) => {
if (err) {
return next(err);
}
return s3.deleteObject({ Bucket: testBucket, Key: keyName2, VersionId: data.VersionId }, next);
}),
], done);
});

after(done => async.series([
next => removeAllVersions({ Bucket: testBucket }, next),
next => s3.deleteBucket({ Bucket: testBucket }, next),
], done));

it('should return truncated lists - part 1', done => {
makeBackbeatRequest({
method: 'GET',
bucket: testBucket,
queryObj: { 'list-type': 'current', 'max-keys': '1' },
authCredentials: credentials,
}, (err, response) => {
assert.ifError(err);
assert.strictEqual(response.statusCode, 200);
const data = JSON.parse(response.body);

assert.strictEqual(data.IsTruncated, true);
assert.strictEqual(data.NextMarker, keyName0);
assert.strictEqual(data.MaxKeys, 1);
assert.strictEqual(data.Contents.length, 1);

const contents = data.Contents;
assert.strictEqual(contents.length, 1);
checkContents(contents, expectedKeyVersions);
assert.strictEqual(contents[0].Key, keyName0);
return done();
});
});

it('should return truncated lists - part 2', done => {
makeBackbeatRequest({
method: 'GET',
bucket: testBucket,
queryObj: {
'list-type': 'current',
'max-keys': '1',
'marker': keyName0,
},
authCredentials: credentials,
}, (err, response) => {
assert.ifError(err);
assert.strictEqual(response.statusCode, 200);
const data = JSON.parse(response.body);

assert(!data.NextMarker);
assert.strictEqual(data.IsTruncated, false);
assert.strictEqual(data.Marker, keyName0);
assert.strictEqual(data.MaxKeys, 1);
assert.strictEqual(data.Contents.length, 1);

const contents = data.Contents;
assert.strictEqual(contents.length, 1);
checkContents(contents, expectedKeyVersions);
assert.strictEqual(contents[0].Key, keyName2);
return done();
});
});
});
Loading

0 comments on commit 18bf6b8

Please sign in to comment.