diff --git a/constants.js b/constants.js index 61f2ba5b58..fb30210aa9 100644 --- a/constants.js +++ b/constants.js @@ -202,10 +202,14 @@ const constants = { allowedUtapiEventFilterFields: ['operationId', 'location', 'account', 'user', 'bucket'], arrayOfAllowed: ['objectPutTagging', 'objectPutLegalHold', 'objectPutRetention'], allowedUtapiEventFilterStates: ['allow', 'deny'], - allowedRestoreObjectRequestTierValues: ['Standard'], + // All tiers are accepted but restored the same way: the tier is not used + allowedRestoreObjectRequestTierValues: ['Standard', 'Bulk', 'Expedited'], // Only STANDARD class is supported, but keep the option to override supported values for now. // This should be removed in CLDSRV-639. validStorageClasses: process.env.VALID_STORAGE_CLASSES?.split(',') || ['STANDARD'], + // AWS archive storage classes, mapped to a cold location of the same (lower case) name: + // standard S3 clients (AWS CLI, Veeam...) only accept the official storage class names. + awsArchiveStorageClasses: new Map(['GLACIER', 'DEEP_ARCHIVE'].map(name => [name.toLowerCase(), name])), lifecycleListing: { CURRENT_TYPE: 'current', NON_CURRENT_TYPE: 'noncurrent', diff --git a/lib/api/apiUtils/object/objectAttributes.js b/lib/api/apiUtils/object/objectAttributes.js index c461d8734c..45d9ce2490 100644 --- a/lib/api/apiUtils/object/objectAttributes.js +++ b/lib/api/apiUtils/object/objectAttributes.js @@ -1,6 +1,7 @@ const { errorInstances } = require('@scality/arsenal'); const { getPartCountFromMd5 } = require('./partInfo'); const { algorithms } = require('../integrity/validateChecksums'); +const { formatStorageClass } = require('./storageClass'); /** * Parse and validate attribute headers from a request. @@ -74,7 +75,7 @@ function buildAttributesXml(objectMD, userMetadata, requestedAttrs, xml, log) { break; } case 'StorageClass': - xml.push(`${objectMD['x-amz-storage-class']}`); + xml.push(`${formatStorageClass(objectMD['x-amz-storage-class'])}`); break; case 'ObjectSize': xml.push(`${objectMD['content-length']}`); diff --git a/lib/api/apiUtils/object/storageClass.js b/lib/api/apiUtils/object/storageClass.js index 37238d748e..b2fdf8f323 100644 --- a/lib/api/apiUtils/object/storageClass.js +++ b/lib/api/apiUtils/object/storageClass.js @@ -3,6 +3,28 @@ const { errors } = require('@scality/arsenal'); const constants = require('../../../../constants'); const { config } = require('../../../Config'); +const { awsArchiveStorageClasses } = constants; + +/** + * Map an AWS archive storage class to the matching location name + * @param {string} storageClass - value of the x-amz-storage-class header + * @returns {string} the storage class as used internally + */ +function normalizeStorageClass(storageClass) { + const location = storageClass.toLowerCase(); + return awsArchiveStorageClasses.get(location) === storageClass ? location : storageClass; +} + +/** + * Map a stored storage class to the value reported to clients: a location named after an + * AWS archive storage class is reported with the AWS name. + * @param {string} storageClass - storage class from the object metadata + * @returns {string} the storage class as reported in responses + */ +function formatStorageClass(storageClass) { + return awsArchiveStorageClasses.get(storageClass) || storageClass; +} + /** * Whether the given storage class names a cold location of this deployment * @param {string} storageClass - value of the x-amz-storage-class header @@ -12,16 +34,32 @@ function isColdStorageClass(storageClass) { return config.locationConstraints[storageClass]?.isCold; } +/** + * Whether the Veeam on-prem archive tier can be used with this deployment + * + * Veeam writes to the archive tier with the GLACIER storage class, which needs direct-to-cold + * writes and a cold location of that name + * @returns {boolean} true if the archive tier is supported + */ +function supportsOnPremArchiveTier() { + return !!(config.enableDirectToCold && isColdStorageClass('glacier')); +} + /** * Validate the x-amz-storage-class header of an object write request. * * Accepted values are the regular S3 storage classes, and, with - * `enableDirectToCold`, the name of a cold location of this deployment. + * `enableDirectToCold`, the name of a cold location of this deployment. An AWS + * archive storage class names the location of the same (lower case) name: the + * header is rewritten accordingly, so that the API only ever sees the location name. * * Restricting which identity may use a given storage class is done through the - * `s3:x-amz-storage-class` IAM policy condition key, and so is not handled here. + * `s3:x-amz-storage-class` policy condition key, and so is not handled here. IAM + * policies are evaluated before this (by Vault, on the signed header), so they see + * the storage class as sent by the client; bucket policies are evaluated afterwards, + * so they see the location name. * - * @param {object} headers - request headers + * @param {object} headers - request headers, rewritten in place * @returns {ArsenalError|null} InvalidStorageClass if the value is not supported */ function validateStorageClass(headers) { @@ -38,13 +76,19 @@ function validateStorageClass(headers) { if (putVersionId || putVersionId === '') { return errors.InvalidStorageClass; } - if (config.enableDirectToCold && isColdStorageClass(storageClass)) { + const location = normalizeStorageClass(storageClass); + if (config.enableDirectToCold && isColdStorageClass(location)) { + // eslint-disable-next-line no-param-reassign + headers['x-amz-storage-class'] = location; return null; } return errors.InvalidStorageClass; } module.exports = { + normalizeStorageClass, + formatStorageClass, isColdStorageClass, + supportsOnPremArchiveTier, validateStorageClass, }; diff --git a/lib/api/bucketGet.js b/lib/api/bucketGet.js index db097cffe3..a9b2beafdb 100644 --- a/lib/api/bucketGet.js +++ b/lib/api/bucketGet.js @@ -11,6 +11,7 @@ const versionIdUtils = versioning.VersionID; const monitoring = require('../utilities/monitoringHandler'); const { generateToken, decryptToken } = require('../api/apiUtils/object/continueToken'); const { parseAttributesHeaders, buildAttributesXml } = require('./apiUtils/object/objectAttributes'); +const { formatStorageClass } = require('./apiUtils/object/storageClass'); const OPTIONAL_ATTRIBUTES = new Set(['RestoreStatus']); @@ -158,7 +159,10 @@ function processVersions(bucketName, listParams, list) { buildAttributesXml(v, v.userMetadata, listParams.optionalAttributes, xml); } - xml.push(`${v.StorageClass}`, v.IsDeleteMarker ? '' : ''); + xml.push( + `${formatStorageClass(v.StorageClass)}`, + v.IsDeleteMarker ? '' : '', + ); }); list.CommonPrefixes.forEach(item => { @@ -242,7 +246,7 @@ function processMasterVersions(bucketName, listParams, list) { buildAttributesXml(v, v.userMetadata, listParams.optionalAttributes, xml); - return xml.push(`${v.StorageClass}`, ''); + return xml.push(`${formatStorageClass(v.StorageClass)}`, ''); }); list.CommonPrefixes.forEach(item => { xml.push(`${escapeXmlFn(item)}`); diff --git a/lib/api/listMultipartUploads.js b/lib/api/listMultipartUploads.js index 9024231678..e20de6ef9e 100644 --- a/lib/api/listMultipartUploads.js +++ b/lib/api/listMultipartUploads.js @@ -7,6 +7,7 @@ const constants = require('../../constants'); const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const services = require('../services'); const { standardMetadataValidateBucket } = require('../metadata/metadataUtils'); +const { formatStorageClass } = require('./apiUtils/object/storageClass'); const { pushMetric } = require('../utapi/utilities'); const monitoring = require('../utilities/monitoringHandler'); @@ -151,6 +152,10 @@ function listMultipartUploads(authInfo, request, log, callback) { monitoring.promMetrics('GET', bucketName, err.code, 'listMultipartUploads'); return callback(err, null, corsHeaders); } + list.Uploads.forEach(upload => { + // eslint-disable-next-line no-param-reassign + upload.value.StorageClass = formatStorageClass(upload.value.StorageClass); + }); const xmlParams = { bucketName, encoding, diff --git a/lib/api/listParts.js b/lib/api/listParts.js index 0e29302ce3..479b1e55f4 100644 --- a/lib/api/listParts.js +++ b/lib/api/listParts.js @@ -13,6 +13,7 @@ const { pushMetric } = require('../utapi/utilities'); const monitoring = require('../utilities/monitoringHandler'); const { data } = require('../data/wrapper'); const { setExpirationHeaders } = require('./apiUtils/object/expirationHeaders'); +const { formatStorageClass } = require('./apiUtils/object/storageClass'); const { algorithms } = require('./apiUtils/integrity/validateChecksums'); /* @@ -303,7 +304,7 @@ function listParts(authInfo, request, log, callback) { xml.push(''); buildXML( [ - { tag: 'StorageClass', value: mpuOverviewObj.storageClass }, + { tag: 'StorageClass', value: formatStorageClass(mpuOverviewObj.storageClass) }, { tag: 'PartNumberMarker', value: partNumberMarker || undefined }, // print only if it's truncated { tag: 'NextPartNumberMarker', value: isTruncated ? parseInt(lastPartShown, 10) : undefined }, diff --git a/lib/routes/veeam/put.js b/lib/routes/veeam/put.js index 0e91810be3..0171389eab 100644 --- a/lib/routes/veeam/put.js +++ b/lib/routes/veeam/put.js @@ -52,7 +52,8 @@ function putVeeamFile(request, response, bucketMd, log) { validatedData = validateFn(parsedXML); } catch (err) { log.error('xml file did not pass validation', { err }); - return next(errors.MalformedXML); + // Forward arsenal errors as is, anything else is a malformed file + return next(err.is ? err : errors.MalformedXML); } const file = getFileToBuild(request, validatedData, true); if (file.error) { diff --git a/lib/routes/veeam/schemas/system.js b/lib/routes/veeam/schemas/system.js index feeafbacf5..620f73bcac 100644 --- a/lib/routes/veeam/schemas/system.js +++ b/lib/routes/veeam/schemas/system.js @@ -1,5 +1,6 @@ const joi = require('joi'); const { errors, errorInstances } = require('@scality/arsenal'); +const { supportsOnPremArchiveTier } = require('../../../api/apiUtils/object/storageClass'); // Allow supporting any version of the protocol const systemSchemasPerVersion = { @@ -14,6 +15,7 @@ const systemSchemasPerVersion = { CapacityInfo: joi.boolean().required(), UploadSessions: joi.boolean().required(), IAMSTS: joi.boolean().default(false), + EnableOnPremArchiveTier: joi.boolean(), }) .required(), APIEndpoints: joi.object({ @@ -43,6 +45,7 @@ const systemSchemasPerVersion = { * true * true * true + * true * * * https://storage.acme.local/iam/endpoint @@ -89,6 +92,16 @@ function validateSystemSchema(parsedXML) { ) { throw new Error(errors.MalformedXML); } + // The archive tier can only be enabled when the deployment supports it + if ( + validatedData.value.SystemInfo.ProtocolCapabilities.EnableOnPremArchiveTier && + !supportsOnPremArchiveTier() + ) { + throw errorInstances.InvalidRequest.customizeDescription( + 'EnableOnPremArchiveTier is not supported: direct-to-cold writes ' + + 'and a cold location named glacier are required', + ); + } break; default: break; diff --git a/lib/routes/veeam/utils.js b/lib/routes/veeam/utils.js index cd3375b1b9..1e1c06ca06 100644 --- a/lib/routes/veeam/utils.js +++ b/lib/routes/veeam/utils.js @@ -6,6 +6,7 @@ const collectResponseHeaders = require('../../utilities/collectResponseHeaders') const collectCorsHeaders = require('../../utilities/collectCorsHeaders'); const crypto = require('crypto'); const { prepareStream } = require('../../api/apiUtils/object/prepareStream'); +const { supportsOnPremArchiveTier } = require('../../api/apiUtils/object/storageClass'); const { getChecksumDataFromHeaders, arsenalErrorFromChecksumError, @@ -319,6 +320,14 @@ async function buildVeeamFileData(request, bucketMd, log) { } } else { bucketMetrics = { date: new Date() }; + // The archive tier capability depends on the current deployment, not on the + // (possibly stale) value stored at PUT time: it is reported only when supported, + // and defaults to the supported state when the client did not set it. + const capabilities = fileToBuild.value.SystemInfo.ProtocolCapabilities; + if (capabilities) { + const supported = supportsOnPremArchiveTier(); + capabilities.EnableOnPremArchiveTier = (capabilities.EnableOnPremArchiveTier ?? supported) && supported; + } } const modified = bucketMetrics.date; diff --git a/lib/utilities/collectResponseHeaders.js b/lib/utilities/collectResponseHeaders.js index c5eaf7ea5c..5521778114 100644 --- a/lib/utilities/collectResponseHeaders.js +++ b/lib/utilities/collectResponseHeaders.js @@ -1,6 +1,7 @@ const { getVersionIdResHeader } = require('../api/apiUtils/object/versioning'); const checkUserMetadataSize = require('../api/apiUtils/object/checkUserMetadataSize'); const { getAmzRestoreResHeader } = require('../api/apiUtils/object/coldStorage'); +const { formatStorageClass } = require('../api/apiUtils/object/storageClass'); const { config } = require('../Config'); const { getKeyIdFromArn } = require('@scality/arsenal/build/lib/network/KMSInterface'); @@ -35,7 +36,7 @@ function collectResponseHeaders(objectMD, corsHeaders, versioningCfg, returnTagC responseMetaHeaders['x-amz-website-redirect-location'] = objectMD['x-amz-website-redirect-location']; } if (objectMD['x-amz-storage-class'] !== 'STANDARD') { - responseMetaHeaders['x-amz-storage-class'] = objectMD['x-amz-storage-class']; + responseMetaHeaders['x-amz-storage-class'] = formatStorageClass(objectMD['x-amz-storage-class']); } if (objectMD['x-amz-server-side-encryption']) { responseMetaHeaders['x-amz-server-side-encryption'] = objectMD['x-amz-server-side-encryption']; diff --git a/package.json b/package.json index 49f608959e..51f97381c5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@scality/cloudserver", - "version": "9.5.0-preview.5", + "version": "9.5.0", "description": "Zenko CloudServer, an open-source Node.js implementation of a server handling the Amazon S3 protocol", "main": "index.js", "engines": { diff --git a/tests/sur/routeVeeam.js b/tests/sur/routeVeeam.js index 7dd3420dee..bed81be417 100644 --- a/tests/sur/routeVeeam.js +++ b/tests/sur/routeVeeam.js @@ -75,6 +75,14 @@ const testSystem = ` const testSystemMd5 = crypto.createHash('md5').update(testSystem, 'utf-8').digest('hex'); +// The served system.xml reports the archive tier capability of the deployment, which +// the server under test does not support (no direct-to-cold) +const systemOut = testSystem.replace( + 'true\n', + 'true\n false\n', +); +const systemOutMd5 = crypto.createHash('md5').update(systemOut, 'utf-8').digest('hex'); + const invalidTestSystem = ` "1.0" @@ -237,6 +245,38 @@ function makeVeeamRequest(params, callback) { )); }); + it('PUT system.xml should reject EnableOnPremArchiveTier when not supported', done => { + // The server under test has no direct-to-cold, so the archive tier cannot be enabled + const systemWithArchiveTier = testSystem.replace( + 'true\n', + 'true\n true\n', + ); + const systemWithArchiveTierMd5 = crypto + .createHash('md5') + .update(systemWithArchiveTier, 'utf-8') + .digest('hex'); + makeVeeamRequest( + { + method: 'PUT', + bucket: TEST_BUCKET, + objectKey: '.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml', + headers: { + 'content-length': systemWithArchiveTier.length, + 'content-md5': systemWithArchiveTierMd5, + 'x-scal-canonical-id': testArn, + }, + authCredentials: veeamAuthCredentials, + requestBody: systemWithArchiveTier, + }, + err => { + assert.strictEqual(err.statusCode, 400); + assert.strictEqual(err.code, 'InvalidRequest'); + assert.match(err.message, /EnableOnPremArchiveTier is not supported/); + return done(); + }, + ); + }); + [ ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml', testSystem, testSystemMd5], ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/capacity.xml', testCapacity, testCapacityMd5], @@ -367,8 +407,10 @@ function makeVeeamRequest(params, callback) { }); [ - ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml', testSystem, testSystemMd5], - ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/capacity.xml', testCapacity, testCapacityMd5], + ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml', testSystem, testSystemMd5, systemOut], + // archive tier explicitly disabled: reported as is + ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml', systemOut, systemOutMd5, systemOut], + ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/capacity.xml', testCapacity, testCapacityMd5, testCapacity], ].forEach(key => { it(`GET ${key[0]} should return the expected XML file`, done => { scuba.incrementBytesForBucket(TEST_BUCKET, 0); @@ -412,7 +454,7 @@ function makeVeeamRequest(params, callback) { return done(err); } assert.strictEqual(response.statusCode, 200); - assert.strictEqual(response.body.replaceAll(' ', ''), key[1].replaceAll(' ', '')); + assert.strictEqual(response.body.replaceAll(' ', ''), key[3].replaceAll(' ', '')); return next(); }, ), @@ -426,8 +468,8 @@ function makeVeeamRequest(params, callback) { }); [ - ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml', testSystem, testSystemMd5], - ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/capacity.xml', testCapacity, testCapacityMd5], + ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml', testSystem, testSystemMd5, systemOut], + ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/capacity.xml', testCapacity, testCapacityMd5, testCapacity], ].forEach(key => { it(`GET ${key[0]} should return the expected XML file for cors requests`, done => { async.waterfall( @@ -472,7 +514,7 @@ function makeVeeamRequest(params, callback) { return done(err); } assert.strictEqual(response.statusCode, 200); - assert.strictEqual(response.body.replaceAll(' ', ''), key[1].replaceAll(' ', '')); + assert.strictEqual(response.body.replaceAll(' ', ''), key[3].replaceAll(' ', '')); return next(); }, ), @@ -627,7 +669,7 @@ function makeVeeamRequest(params, callback) { 200, 'system.xml should always return 200 even when scuba is down', ); - assert.strictEqual(response.body.replaceAll(' ', ''), testSystem.replaceAll(' ', '')); + assert.strictEqual(response.body.replaceAll(' ', ''), systemOut.replaceAll(' ', '')); return next(); }, ), @@ -662,8 +704,8 @@ function makeVeeamRequest(params, callback) { }); [ - ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml', testSystem, testSystemMd5], - ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/capacity.xml', testCapacity, testCapacityMd5], + ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml', testSystem, testSystemMd5, systemOut], + ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/capacity.xml', testCapacity, testCapacityMd5, testCapacity], ].forEach(key => { it(`DELETE ${key[0]} should delete the XML file`, done => { async.waterfall( @@ -706,7 +748,7 @@ function makeVeeamRequest(params, callback) { return done(err); } assert.strictEqual(response.statusCode, 200); - assert.strictEqual(response.body.replaceAll(' ', ''), key[1].replaceAll(' ', '')); + assert.strictEqual(response.body.replaceAll(' ', ''), key[3].replaceAll(' ', '')); return next(); }, ), diff --git a/tests/unit/api/apiUtils/object/objectAttributes.js b/tests/unit/api/apiUtils/object/objectAttributes.js index 2f9acdc553..6c59563982 100644 --- a/tests/unit/api/apiUtils/object/objectAttributes.js +++ b/tests/unit/api/apiUtils/object/objectAttributes.js @@ -91,6 +91,15 @@ describe('buildXmlAttributes', () => { assert.strictEqual(result[0], 'STANDARD'); }); + it('should report an archive location with the AWS storage class name', () => { + const result = []; + const archivedMD = { ...objectMD, 'x-amz-storage-class': 'deep_archive' }; + buildAttributesXml(archivedMD, userMetadata, ['StorageClass'], result, log); + + assert.strictEqual(result.length, 1); + assert.strictEqual(result[0], 'DEEP_ARCHIVE'); + }); + it('should generate ObjectSize XML', () => { const result = []; buildAttributesXml(objectMD, userMetadata, ['ObjectSize'], result, log); diff --git a/tests/unit/api/apiUtils/storageClass.js b/tests/unit/api/apiUtils/storageClass.js index 04c5b12be3..bba47b47d1 100644 --- a/tests/unit/api/apiUtils/storageClass.js +++ b/tests/unit/api/apiUtils/storageClass.js @@ -1,6 +1,12 @@ const assert = require('assert'); -const { isColdStorageClass, validateStorageClass } = require('../../../../lib/api/apiUtils/object/storageClass'); +const { + normalizeStorageClass, + formatStorageClass, + isColdStorageClass, + supportsOnPremArchiveTier, + validateStorageClass, +} = require('../../../../lib/api/apiUtils/object/storageClass'); const { config } = require('../../../../lib/Config'); const coldLocation = 'location-dmf-v1'; @@ -17,6 +23,44 @@ describe('storage class helpers', () => { config.enableDirectToCold = originalEnableDirectToCold; }); + describe('normalizeStorageClass', () => { + [ + ['GLACIER', 'glacier'], + ['DEEP_ARCHIVE', 'deep_archive'], + ].forEach(([awsName, location]) => { + it(`should map ${awsName} to the ${location} location`, () => { + assert.strictEqual(normalizeStorageClass(awsName), location); + }); + }); + + ['STANDARD', 'GLACIER_IR', 'Glacier', coldLocation, 'Some-Location'].forEach(storageClass => { + it(`should keep ${storageClass} as is`, () => { + assert.strictEqual(normalizeStorageClass(storageClass), storageClass); + }); + }); + }); + + describe('formatStorageClass', () => { + [ + ['glacier', 'GLACIER'], + ['deep_archive', 'DEEP_ARCHIVE'], + ].forEach(([location, awsName]) => { + it(`should report the ${location} location as ${awsName}`, () => { + assert.strictEqual(formatStorageClass(location), awsName); + }); + }); + + ['STANDARD', 'GLACIER', coldLocation, 'Some-Location'].forEach(storageClass => { + it(`should keep ${storageClass} as is`, () => { + assert.strictEqual(formatStorageClass(storageClass), storageClass); + }); + }); + + it('should keep an undefined storage class', () => { + assert.strictEqual(formatStorageClass(undefined), undefined); + }); + }); + describe('isColdStorageClass', () => { it('should return true for a cold location', () => { assert.strictEqual(isColdStorageClass(coldLocation), true); @@ -39,6 +83,35 @@ describe('storage class helpers', () => { }); }); + describe('supportsOnPremArchiveTier', () => { + afterEach(() => { + delete config.locationConstraints.glacier; + }); + + it('should return true with direct-to-cold and a cold glacier location', () => { + config.enableDirectToCold = true; + config.locationConstraints.glacier = { ...config.locationConstraints[coldLocation] }; + assert.strictEqual(supportsOnPremArchiveTier(), true); + }); + + it('should return false without direct-to-cold', () => { + config.enableDirectToCold = false; + config.locationConstraints.glacier = { ...config.locationConstraints[coldLocation] }; + assert.strictEqual(supportsOnPremArchiveTier(), false); + }); + + it('should return false without a glacier location', () => { + config.enableDirectToCold = true; + assert.strictEqual(supportsOnPremArchiveTier(), false); + }); + + it('should return false when the glacier location is not cold', () => { + config.enableDirectToCold = true; + config.locationConstraints.glacier = { ...config.locationConstraints[hotLocation] }; + assert.strictEqual(supportsOnPremArchiveTier(), false); + }); + }); + describe('validateStorageClass', () => { it('should accept a request without a storage class header', () => { assert.strictEqual(validateStorageClass({}), null); @@ -94,5 +167,48 @@ describe('storage class helpers', () => { const headers = { 'x-amz-storage-class': 'STANDARD', 'x-scal-s3-version-id': '' }; assert.strictEqual(validateStorageClass(headers), null); }); + + describe('with a location named after an AWS archive storage class', () => { + beforeEach(() => { + config.locationConstraints.glacier = { ...config.locationConstraints[coldLocation] }; + }); + + afterEach(() => { + delete config.locationConstraints.glacier; + }); + + it('should rewrite the header to the location name', () => { + config.enableDirectToCold = true; + const headers = { 'x-amz-storage-class': 'GLACIER' }; + assert.strictEqual(validateStorageClass(headers), null); + assert.strictEqual(headers['x-amz-storage-class'], 'glacier'); + }); + + it('should reject the AWS storage class when the option is disabled', () => { + config.enableDirectToCold = false; + const err = validateStorageClass({ 'x-amz-storage-class': 'GLACIER' }); + assert.strictEqual(err.message, 'InvalidStorageClass'); + }); + + it('should reject the AWS storage class on a restore', () => { + config.enableDirectToCold = true; + const headers = { 'x-amz-storage-class': 'GLACIER', 'x-scal-s3-version-id': '' }; + const err = validateStorageClass(headers); + assert.strictEqual(err.message, 'InvalidStorageClass'); + }); + + it('should not rewrite a regular storage class', () => { + const headers = { 'x-amz-storage-class': 'STANDARD' }; + assert.strictEqual(validateStorageClass(headers), null); + assert.strictEqual(headers['x-amz-storage-class'], 'STANDARD'); + }); + + it('should not rewrite a location name', () => { + config.enableDirectToCold = true; + const headers = { 'x-amz-storage-class': coldLocation }; + assert.strictEqual(validateStorageClass(headers), null); + assert.strictEqual(headers['x-amz-storage-class'], coldLocation); + }); + }); }); }); diff --git a/tests/unit/api/directToCold.js b/tests/unit/api/directToCold.js index 2b37ee4b1a..ff49cb9865 100644 --- a/tests/unit/api/directToCold.js +++ b/tests/unit/api/directToCold.js @@ -1,6 +1,8 @@ const assert = require('assert'); const async = require('async'); +const { parseString } = require('xml2js'); +const { bucketGet } = require('../../../lib/api/bucketGet'); const { bucketPut } = require('../../../lib/api/bucketPut'); const objectPut = require('../../../lib/api/objectPut'); const objectCopy = require('../../../lib/api/objectCopy'); @@ -8,6 +10,8 @@ const objectGet = require('../../../lib/api/objectGet'); const objectHead = require('../../../lib/api/objectHead'); const objectRestore = require('../../../lib/api/objectRestore'); const initiateMultipartUpload = require('../../../lib/api/initiateMultipartUpload'); +const listMultipartUploads = require('../../../lib/api/listMultipartUploads'); +const listParts = require('../../../lib/api/listParts'); const DummyRequest = require('../DummyRequest'); const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const metadata = require('../metadataswitch'); @@ -96,8 +100,8 @@ function putDirectToColdObject(cb) { ); } -function assertDirectToCold(md) { - assert.strictEqual(md['x-amz-storage-class'], coldLocation); +function assertDirectToCold(md, location = coldLocation) { + assert.strictEqual(md['x-amz-storage-class'], location); // the data itself stays in the hot location assert.strictEqual(md.dataStoreName, hotLocation); // the transition has not happened yet, so there is no archive info @@ -505,4 +509,176 @@ describe('direct to cold', () => { }); }); }); + + describe('with a location named after an AWS archive storage class', () => { + const awsStorageClass = 'GLACIER'; + const archiveLocation = 'glacier'; + const listObjectsRequest = { + bucketName, + namespace, + headers: { host: '/' }, + url: `/${bucketName}`, + query: {}, + actionImplicitDenies: false, + }; + + before(() => { + config.locationConstraints[archiveLocation] = { ...config.locationConstraints[coldLocation] }; + }); + + after(() => { + delete config.locationConstraints[archiveLocation]; + }); + + beforeEach(done => { + config.enableDirectToCold = true; + objectPut(authInfo, putObjectRequest({ 'x-amz-storage-class': awsStorageClass }), undefined, log, err => + done(err), + ); + }); + + it('should store the location name as the storage class of a PUT object', done => { + getObjectMD(objectKey, (err, md) => { + assert.ifError(err); + assertDirectToCold(md, archiveLocation); + done(); + }); + }); + + it('should store the location name as the storage class of a copied object', done => { + async.waterfall( + [ + next => + objectCopy( + authInfo, + copyObjectRequest({ 'x-amz-storage-class': awsStorageClass }), + bucketName, + objectKey, + undefined, + log, + err => next(err), + ), + next => getObjectMD('copiedObject', next), + ], + (err, md) => { + assert.ifError(err); + assertDirectToCold(md, archiveLocation); + done(); + }, + ); + }); + + it('should store the location name as the storage class of a multipart upload', done => { + const mpuKey = 'mpuObject'; + async.waterfall( + [ + next => + mpuUtils + .initiateMpuP(bucketName, namespace, mpuKey, log, { + 'x-amz-storage-class': awsStorageClass, + }) + .then(uploadId => next(null, uploadId), next), + (uploadId, next) => + mpuUtils + .uploadPartP(bucketName, namespace, mpuKey, uploadId, log) + .then(() => next(null, uploadId), next), + (uploadId, next) => + mpuUtils.completeMpuP(bucketName, namespace, mpuKey, uploadId, log).then(() => next(), next), + next => getObjectMD(mpuKey, next), + ], + (err, md) => { + assert.ifError(err); + assertDirectToCold(md, archiveLocation); + done(); + }, + ); + }); + + it('should report the AWS storage class name on HeadObject', done => { + objectHead(authInfo, getObjectRequest(), log, (err, headers) => { + assert.ifError(err); + assert.strictEqual(headers['x-amz-storage-class'], 'GLACIER'); + done(); + }); + }); + + it('should report the AWS storage class name on ListObjects', done => { + async.waterfall( + [ + next => bucketGet(authInfo, listObjectsRequest, log, (err, xml) => next(err, xml)), + (xml, next) => parseString(xml, next), + ], + (err, result) => { + assert.ifError(err); + assert.strictEqual(result.ListBucketResult.Contents[0].StorageClass[0], 'GLACIER'); + done(); + }, + ); + }); + + it('should report the AWS storage class name on ListParts', done => { + const mpuKey = 'mpuObject'; + async.waterfall( + [ + next => + mpuUtils + .initiateMpuP(bucketName, namespace, mpuKey, log, { + 'x-amz-storage-class': awsStorageClass, + }) + .then(uploadId => next(null, uploadId), next), + (uploadId, next) => + mpuUtils + .uploadPartP(bucketName, namespace, mpuKey, uploadId, log) + .then(() => next(null, uploadId), next), + (uploadId, next) => { + const listPartsRequest = { + bucketName, + namespace, + objectKey: mpuKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${mpuKey}?uploadId=${uploadId}`, + query: { uploadId }, + actionImplicitDenies: false, + }; + listParts(authInfo, listPartsRequest, log, (err, xml) => next(err, xml)); + }, + (xml, next) => parseString(xml, next), + ], + (err, result) => { + assert.ifError(err); + assert.strictEqual(result.ListPartsResult.StorageClass[0], 'GLACIER'); + done(); + }, + ); + }); + + it('should report the AWS storage class name on ListMultipartUploads', done => { + const mpuKey = 'mpuObject'; + const listMpuRequest = { + bucketName, + namespace, + headers: { host: '/' }, + url: `/${bucketName}?uploads`, + query: {}, + actionImplicitDenies: false, + }; + async.waterfall( + [ + next => + mpuUtils + .initiateMpuP(bucketName, namespace, mpuKey, log, { + 'x-amz-storage-class': awsStorageClass, + }) + .then(() => next(), next), + next => listMultipartUploads(authInfo, listMpuRequest, log, (err, xml) => next(err, xml)), + (xml, next) => parseString(xml, next), + ], + (err, result) => { + assert.ifError(err); + assert.strictEqual(result.ListMultipartUploadsResult.Upload[0].StorageClass[0], 'GLACIER'); + done(); + }, + ); + }); + }); }); diff --git a/tests/unit/api/objectRestore.js b/tests/unit/api/objectRestore.js index c639c3b743..0dfd112e1b 100644 --- a/tests/unit/api/objectRestore.js +++ b/tests/unit/api/objectRestore.js @@ -93,23 +93,23 @@ describe('restoreObject API', () => { }); }); - it("should return NotImplemented error when object restore Tier is 'Bulk'", done => { - mdColdHelper.putBucketMock(bucketName, null, () => { - mdColdHelper.putObjectMock(bucketName, objectName, mdColdHelper.getArchivedObjectMD(), () => { - objectRestore(authInfo, objectRestoreRequest(objectRestoreXmlBulkTier), log, err => { - assert.strictEqual(err.is.NotImplemented, true); - done(); - }); - }); - }); - }); - - it("should return NotImplemented error when object restore Tier is 'Expedited'", done => { - mdColdHelper.putBucketMock(bucketName, null, () => { - mdColdHelper.putObjectMock(bucketName, objectName, mdColdHelper.getArchivedObjectMD(), () => { - objectRestore(authInfo, objectRestoreRequest(objectRestoreXmlExpeditedTier), log, err => { - assert.strictEqual(err.is.NotImplemented, true); - done(); + [ + ['Bulk', objectRestoreXmlBulkTier], + ['Expedited', objectRestoreXmlExpeditedTier], + ].forEach(([tier, xml]) => { + it(`should accept a restore with the '${tier}' Tier`, done => { + mdColdHelper.putBucketMock(bucketName, null, () => { + mdColdHelper.putObjectMock(bucketName, objectName, mdColdHelper.getArchivedObjectMD(), () => { + objectRestore(authInfo, objectRestoreRequest(xml), log, (err, statusCode) => { + assert.ifError(err); + assert.strictEqual(statusCode, 202); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert.ifError(err); + // the tier is accepted but not acted upon: the restore is a regular one + assert.strictEqual(md.archive.restoreRequestedDays, restoreDays); + done(); + }); + }); }); }); }); diff --git a/tests/unit/internal/veeam/schemas/system.js b/tests/unit/internal/veeam/schemas/system.js index c0294231e7..8931fb373b 100644 --- a/tests/unit/internal/veeam/schemas/system.js +++ b/tests/unit/internal/veeam/schemas/system.js @@ -1,6 +1,7 @@ const assert = require('assert'); const validateSystemSchema = require('../../../../../lib/routes/veeam/schemas/system'); const { errorInstances } = require('@scality/arsenal'); +const { config } = require('../../../../../lib/Config'); const modelName = '"ARTESCA"'; @@ -82,11 +83,124 @@ describe('RouteVeeam: validateSystemSchema 1.0', () => { }, }, }, + { + SystemInfo: { + ProtocolVersion: protocolVersion, + ModelName: modelName, + ProtocolCapabilities: { + CapacityInfo: true, + UploadSessions: true, + EnableOnPremArchiveTier: false, + }, + }, + }, ].forEach(test => { it(`should validate XML for ${JSON.stringify(test)}`, () => { assert.doesNotThrow(() => validateSystemSchema(test).message); }); }); + + describe('EnableOnPremArchiveTier support', () => { + const systemInfo = { + SystemInfo: { + ProtocolVersion: protocolVersion, + ModelName: modelName, + ProtocolCapabilities: { + CapacityInfo: true, + UploadSessions: true, + }, + }, + }; + const withEnableOnPremArchiveTier = value => ({ + SystemInfo: { + ...systemInfo.SystemInfo, + ProtocolCapabilities: { + ...systemInfo.SystemInfo.ProtocolCapabilities, + EnableOnPremArchiveTier: value, + }, + }, + }); + let originalEnableDirectToCold; + + const setSupported = supported => { + config.enableDirectToCold = supported; + if (supported) { + config.locationConstraints.glacier = { ...config.locationConstraints['location-dmf-v1'] }; + } + }; + + beforeEach(() => { + originalEnableDirectToCold = config.enableDirectToCold; + }); + + afterEach(() => { + config.enableDirectToCold = originalEnableDirectToCold; + delete config.locationConstraints.glacier; + }); + + [true, false].forEach(supported => { + it(`should not add EnableOnPremArchiveTier when not provided (supported: ${supported})`, () => { + setSupported(supported); + const validated = validateSystemSchema(systemInfo); + assert.strictEqual(validated.SystemInfo.ProtocolCapabilities.EnableOnPremArchiveTier, undefined); + }); + }); + + it('should accept true when supported', () => { + setSupported(true); + const validated = validateSystemSchema(withEnableOnPremArchiveTier(true)); + assert.strictEqual(validated.SystemInfo.ProtocolCapabilities.EnableOnPremArchiveTier, true); + }); + + it('should accept false when not supported', () => { + setSupported(false); + const validated = validateSystemSchema(withEnableOnPremArchiveTier(false)); + assert.strictEqual(validated.SystemInfo.ProtocolCapabilities.EnableOnPremArchiveTier, false); + }); + + it('should reject true when not supported', () => { + setSupported(false); + assert.throws( + () => validateSystemSchema(withEnableOnPremArchiveTier(true)), + err => { + assert.strictEqual(err.message, 'InvalidRequest'); + assert.match(err.description, /EnableOnPremArchiveTier is not supported/); + return true; + }, + ); + }); + }); + + it('should convert EnableOnPremArchiveTier from the XML string value', () => { + const validated = validateSystemSchema({ + SystemInfo: { + ProtocolVersion: protocolVersion, + ModelName: modelName, + ProtocolCapabilities: { + CapacityInfo: 'true', + UploadSessions: 'false', + EnableOnPremArchiveTier: 'false', + }, + }, + }); + assert.strictEqual(validated.SystemInfo.ProtocolCapabilities.EnableOnPremArchiveTier, false); + }); + + it('should reject an invalid EnableOnPremArchiveTier value', () => { + assert.throws(() => + validateSystemSchema({ + SystemInfo: { + ProtocolVersion: protocolVersion, + ModelName: modelName, + ProtocolCapabilities: { + CapacityInfo: true, + UploadSessions: true, + EnableOnPremArchiveTier: 'maybe', + }, + }, + }), + ); + }); }); describe('RouteVeeam: validateSystemSchema unknown version', () => { diff --git a/tests/unit/routes/veeam-utils.js b/tests/unit/routes/veeam-utils.js index 648a8e1503..bf3a5bccf0 100644 --- a/tests/unit/routes/veeam-utils.js +++ b/tests/unit/routes/veeam-utils.js @@ -133,18 +133,25 @@ describe('buildVeeamFileData', () => { }, }; - const bucketMdWithSystem = { + // Built per test: buildVeeamFileData adjusts the stored SystemInfo in place + const createBucketMdWithSystem = (protocolCapabilities = {}) => ({ ...bucketMd, _capabilities: { VeeamSOSApi: { SystemInfo: { ProtocolVersion: '1.0', ModelName: 'ARTESCA', + ProtocolCapabilities: { + CapacityInfo: true, + UploadSessions: true, + IAMSTS: false, + ...protocolCapabilities, + }, LastModified: '2024-01-01T00:00:00.000Z', }, }, }, - }; + }); const createRequest = objectKey => ({ bucketName: 'test-bucket', @@ -152,6 +159,15 @@ describe('buildVeeamFileData', () => { headers: { host: 'test-bucket.s3.amazonaws.com' }, }); + let originalEnableDirectToCold; + + const setArchiveTierSupported = supported => { + config.enableDirectToCold = supported; + if (supported) { + config.locationConstraints.glacier = { ...config.locationConstraints['location-dmf-v1'] }; + } + }; + beforeEach(() => { log = new DummyRequestLogger(); log.warn = sinon.stub(); @@ -159,10 +175,13 @@ describe('buildVeeamFileData', () => { metadataStub = sinon.stub(metadata, 'getBucket'); utilizationStub = sinon.stub(UtilizationService, 'getUtilizationMetrics'); + originalEnableDirectToCold = config.enableDirectToCold; }); afterEach(() => { sinon.restore(); + config.enableDirectToCold = originalEnableDirectToCold; + delete config.locationConstraints.glacier; }); it('should reject with InternalError when metadata.getBucket fails', async () => { @@ -226,6 +245,7 @@ describe('buildVeeamFileData', () => { }); it('should build system.xml without calling UtilizationService', async () => { + const bucketMdWithSystem = createBucketMdWithSystem(); metadataStub.callsArgWith(2, null, bucketMdWithSystem); const before = Date.now(); @@ -239,6 +259,49 @@ describe('buildVeeamFileData', () => { assert(Buffer.isBuffer(result.dataBuffer)); }); + describe('system.xml EnableOnPremArchiveTier', () => { + const buildSystemXML = async protocolCapabilities => { + const bucketMdWithSystem = createBucketMdWithSystem(protocolCapabilities); + metadataStub.callsArgWith(2, null, bucketMdWithSystem); + const result = await buildVeeamFileData(createRequest(systemObjectKey), bucketMdWithSystem, log); + return result.xmlContent; + }; + + it('should report false when not stored and not supported', async () => { + setArchiveTierSupported(false); + assert.match(await buildSystemXML(), /false<\/EnableOnPremArchiveTier>/); + }); + + it('should report true when not stored and supported', async () => { + setArchiveTierSupported(true); + assert.match(await buildSystemXML(), /true<\/EnableOnPremArchiveTier>/); + }); + + it('should report true when stored true and supported', async () => { + setArchiveTierSupported(true); + assert.match( + await buildSystemXML({ EnableOnPremArchiveTier: true }), + /true<\/EnableOnPremArchiveTier>/, + ); + }); + + it('should report false when stored true but not supported', async () => { + setArchiveTierSupported(false); + assert.match( + await buildSystemXML({ EnableOnPremArchiveTier: true }), + /false<\/EnableOnPremArchiveTier>/, + ); + }); + + it('should report false when stored false but supported', async () => { + setArchiveTierSupported(true); + assert.match( + await buildSystemXML({ EnableOnPremArchiveTier: false }), + /false<\/EnableOnPremArchiveTier>/, + ); + }); + }); + it('should not overwrite Used when it is already set', async () => { const bucketMdWithUsed = { ...bucketMd, diff --git a/tests/unit/utils/collectResponseHeaders.js b/tests/unit/utils/collectResponseHeaders.js index 883937fc99..5ab0f968ff 100644 --- a/tests/unit/utils/collectResponseHeaders.js +++ b/tests/unit/utils/collectResponseHeaders.js @@ -110,6 +110,21 @@ describe('Middleware: Collect Response Headers', () => { assert.strictEqual(headers['x-amz-website-redirect-location'], 'google.com'); }); + it('should not report the STANDARD storage class', () => { + const headers = collectResponseHeaders({ 'x-amz-storage-class': 'STANDARD' }); + assert.strictEqual(headers['x-amz-storage-class'], undefined); + }); + + it('should report a cold location as the storage class', () => { + const headers = collectResponseHeaders({ 'x-amz-storage-class': 'location-dmf-v1' }); + assert.strictEqual(headers['x-amz-storage-class'], 'location-dmf-v1'); + }); + + it('should report an archive location with the AWS storage class name', () => { + const headers = collectResponseHeaders({ 'x-amz-storage-class': 'glacier' }); + assert.strictEqual(headers['x-amz-storage-class'], 'GLACIER'); + }); + it('should not set flag when transition not in progress', () => { const obj = {}; const headers = collectResponseHeaders(obj);