Accept GLACIER/DEEP_ARCHIVE as cold locations, advertise the Veeam archive tier

constants.js

@@ -206,6 +206,9 @@ const constants = {
     // Only STANDARD class is supported, but keep the option to override supported values for now.
     // This should be removed in CLDSRV-639.
     validStorageClasses: process.env.VALID_STORAGE_CLASSES?.split(',') || ['STANDARD'],
+    // AWS archive storage classes, mapped to a cold location of the same (lower case) name:
+    // standard S3 clients (AWS CLI, Veeam...) only accept the official storage class names.
+    awsArchiveStorageClasses: new Map(['GLACIER', 'DEEP_ARCHIVE'].map(name => [name.toLowerCase(), name])),
     lifecycleListing: {
         CURRENT_TYPE: 'current',
         NON_CURRENT_TYPE: 'noncurrent',

lib/api/apiUtils/object/objectAttributes.js

@@ -1,6 +1,7 @@
 const { errorInstances } = require('@scality/arsenal');
 const { getPartCountFromMd5 } = require('./partInfo');
 const { algorithms } = require('../integrity/validateChecksums');
+const { formatStorageClass } = require('./storageClass');
 
 /**
  * Parse and validate attribute headers from a request.
@@ -74,7 +75,7 @@ function buildAttributesXml(objectMD, userMetadata, requestedAttrs, xml, log) {
                 break;
             }
             case 'StorageClass':
-                xml.push(`<StorageClass>${objectMD['x-amz-storage-class']}</StorageClass>`);
+                xml.push(`<StorageClass>${formatStorageClass(objectMD['x-amz-storage-class'])}</StorageClass>`);
                 break;
             case 'ObjectSize':
                 xml.push(`<ObjectSize>${objectMD['content-length']}</ObjectSize>`);

lib/api/apiUtils/object/storageClass.js

@@ -3,6 +3,28 @@ const { errors } = require('@scality/arsenal');
 const constants = require('../../../../constants');
 const { config } = require('../../../Config');
 
+const { awsArchiveStorageClasses } = constants;
+
+/**
+ * Map an AWS archive storage class to the matching location name
+ * @param {string} storageClass - value of the x-amz-storage-class header
+ * @returns {string} the storage class as used internally
+ */
+function normalizeStorageClass(storageClass) {
+    const location = storageClass.toLowerCase();
+    return awsArchiveStorageClasses.get(location) === storageClass ? location : storageClass;
+}
+
+/**
+ * Map a stored storage class to the value reported to clients: a location named after an
+ * AWS archive storage class is reported with the AWS name.
+ * @param {string} storageClass - storage class from the object metadata
+ * @returns {string} the storage class as reported in responses
+ */
+function formatStorageClass(storageClass) {
+    return awsArchiveStorageClasses.get(storageClass) || storageClass;
+}
+
 /**
  * Whether the given storage class names a cold location of this deployment
  * @param {string} storageClass - value of the x-amz-storage-class header
@@ -16,12 +38,15 @@ function isColdStorageClass(storageClass) {
  * Validate the x-amz-storage-class header of an object write request.
  *
  * Accepted values are the regular S3 storage classes, and, with
- * `enableDirectToCold`, the name of a cold location of this deployment.
+ * `enableDirectToCold`, the name of a cold location of this deployment. An AWS
+ * archive storage class names the location of the same (lower case) name: the
+ * header is rewritten accordingly, so that the API only ever sees the location name.
  *
  * Restricting which identity may use a given storage class is done through the
- * `s3:x-amz-storage-class` IAM policy condition key, and so is not handled here.
+ * `s3:x-amz-storage-class` IAM policy condition key, and so is not handled here:
+ * policies apply to the storage class as sent by the client.
  *
- * @param {object} headers - request headers
+ * @param {object} headers - request headers, rewritten in place
  * @returns {ArsenalError|null} InvalidStorageClass if the value is not supported
  */
 function validateStorageClass(headers) {
@@ -38,13 +63,18 @@ function validateStorageClass(headers) {
     if (putVersionId || putVersionId === '') {
         return errors.InvalidStorageClass;
     }
-    if (config.enableDirectToCold && isColdStorageClass(storageClass)) {
+    const location = normalizeStorageClass(storageClass);
+    if (config.enableDirectToCold && isColdStorageClass(location)) {
+        // eslint-disable-next-line no-param-reassign
+        headers['x-amz-storage-class'] = location;
         return null;
     }
     return errors.InvalidStorageClass;
 }
 
 module.exports = {
+    normalizeStorageClass,
+    formatStorageClass,
     isColdStorageClass,
     validateStorageClass,
 };

lib/api/bucketGet.js

@@ -11,6 +11,7 @@ const versionIdUtils = versioning.VersionID;
 const monitoring = require('../utilities/monitoringHandler');
 const { generateToken, decryptToken } = require('../api/apiUtils/object/continueToken');
 const { parseAttributesHeaders, buildAttributesXml } = require('./apiUtils/object/objectAttributes');
+const { formatStorageClass } = require('./apiUtils/object/storageClass');
 
 const OPTIONAL_ATTRIBUTES = new Set(['RestoreStatus']);
 
@@ -158,7 +159,10 @@ function processVersions(bucketName, listParams, list) {
             buildAttributesXml(v, v.userMetadata, listParams.optionalAttributes, xml);
         }
 
-        xml.push(`<StorageClass>${v.StorageClass}</StorageClass>`, v.IsDeleteMarker ? '</DeleteMarker>' : '</Version>');
+        xml.push(
+            `<StorageClass>${formatStorageClass(v.StorageClass)}</StorageClass>`,
+            v.IsDeleteMarker ? '</DeleteMarker>' : '</Version>',
+        );
     });
 
     list.CommonPrefixes.forEach(item => {
@@ -242,7 +246,7 @@ function processMasterVersions(bucketName, listParams, list) {
 
         buildAttributesXml(v, v.userMetadata, listParams.optionalAttributes, xml);
 
-        return xml.push(`<StorageClass>${v.StorageClass}</StorageClass>`, '</Contents>');
+        return xml.push(`<StorageClass>${formatStorageClass(v.StorageClass)}</StorageClass>`, '</Contents>');
     });
     list.CommonPrefixes.forEach(item => {
         xml.push(`<CommonPrefixes><Prefix>${escapeXmlFn(item)}</Prefix></CommonPrefixes>`);

lib/api/listMultipartUploads.js

@@ -7,6 +7,7 @@ const constants = require('../../constants');
 const collectCorsHeaders = require('../utilities/collectCorsHeaders');
 const services = require('../services');
 const { standardMetadataValidateBucket } = require('../metadata/metadataUtils');
+const { formatStorageClass } = require('./apiUtils/object/storageClass');
 const { pushMetric } = require('../utapi/utilities');
 const monitoring = require('../utilities/monitoringHandler');
 
@@ -151,6 +152,10 @@ function listMultipartUploads(authInfo, request, log, callback) {
                 monitoring.promMetrics('GET', bucketName, err.code, 'listMultipartUploads');
                 return callback(err, null, corsHeaders);
             }
+            list.Uploads.forEach(upload => {
+                // eslint-disable-next-line no-param-reassign
+                upload.value.StorageClass = formatStorageClass(upload.value.StorageClass);
+            });
             const xmlParams = {
                 bucketName,
                 encoding,

lib/api/listParts.js

@@ -13,6 +13,7 @@ const { pushMetric } = require('../utapi/utilities');
 const monitoring = require('../utilities/monitoringHandler');
 const { data } = require('../data/wrapper');
 const { setExpirationHeaders } = require('./apiUtils/object/expirationHeaders');
+const { formatStorageClass } = require('./apiUtils/object/storageClass');
 const { algorithms } = require('./apiUtils/integrity/validateChecksums');
 
 /*
@@ -303,7 +304,7 @@ function listParts(authInfo, request, log, callback) {
                 xml.push('</Owner>');
                 buildXML(
                     [
-                        { tag: 'StorageClass', value: mpuOverviewObj.storageClass },
+                        { tag: 'StorageClass', value: formatStorageClass(mpuOverviewObj.storageClass) },
                         { tag: 'PartNumberMarker', value: partNumberMarker || undefined },
                         // print only if it's truncated
                         { tag: 'NextPartNumberMarker', value: isTruncated ? parseInt(lastPartShown, 10) : undefined },

lib/routes/veeam/schemas/system.js

@@ -1,5 +1,13 @@
 const joi = require('joi');
 const { errors, errorInstances } = require('@scality/arsenal');
+const { config } = require('../../../Config');
+const { isColdStorageClass } = require('../../../api/apiUtils/object/storageClass');
+
+// Veeam writes to the archive tier with the GLACIER storage class, which needs direct-to-cold
+// writes and a cold location of that name
+function supportsOnPremArchiveTier() {
+    return !!(config.enableDirectToCold && isColdStorageClass('glacier'));
+}
 
 // Allow supporting any version of the protocol
 const systemSchemasPerVersion = {
@@ -14,6 +22,7 @@ const systemSchemasPerVersion = {
                         CapacityInfo: joi.boolean().required(),
                         UploadSessions: joi.boolean().required(),
                         IAMSTS: joi.boolean().default(false),
+                        EnableOnPremArchiveTier: joi.boolean().default(supportsOnPremArchiveTier),
                     })
                     .required(),
                 APIEndpoints: joi.object({
@@ -43,6 +52,7 @@ const systemSchemasPerVersion = {
  *         <CapacityInfo>true</CapacityInfo>
  *         <UploadSessions>true</UploadSessions>
  *         <IAMSTS>true</IAMSTS>
+ *         <EnableOnPremArchiveTier>true</EnableOnPremArchiveTier>
  *     </ProtocolCapabilities>
  *     <APIEndpoints>
  *         <IAMEndpoint>https://storage.acme.local/iam/endpoint</IAMEndpoint>

lib/utilities/collectResponseHeaders.js

@@ -1,6 +1,7 @@
 const { getVersionIdResHeader } = require('../api/apiUtils/object/versioning');
 const checkUserMetadataSize = require('../api/apiUtils/object/checkUserMetadataSize');
 const { getAmzRestoreResHeader } = require('../api/apiUtils/object/coldStorage');
+const { formatStorageClass } = require('../api/apiUtils/object/storageClass');
 const { config } = require('../Config');
 const { getKeyIdFromArn } = require('@scality/arsenal/build/lib/network/KMSInterface');
 
@@ -35,7 +36,7 @@ function collectResponseHeaders(objectMD, corsHeaders, versioningCfg, returnTagC
         responseMetaHeaders['x-amz-website-redirect-location'] = objectMD['x-amz-website-redirect-location'];
     }
     if (objectMD['x-amz-storage-class'] !== 'STANDARD') {
-        responseMetaHeaders['x-amz-storage-class'] = objectMD['x-amz-storage-class'];
+        responseMetaHeaders['x-amz-storage-class'] = formatStorageClass(objectMD['x-amz-storage-class']);
     }
     if (objectMD['x-amz-server-side-encryption']) {
         responseMetaHeaders['x-amz-server-side-encryption'] = objectMD['x-amz-server-side-encryption'];

tests/sur/routeVeeam.js

@@ -60,6 +60,7 @@ const testSystem = `<?xmlversion="1.0"encoding="UTF-8"?>
           <CapacityInfo>true</CapacityInfo>
           <UploadSessions>false</UploadSessions>
           <IAMSTS>true</IAMSTS>
+          <EnableOnPremArchiveTier>true</EnableOnPremArchiveTier>
        </ProtocolCapabilities>
        <APIEndpoints>
             <IAMEndpoint>a</IAMEndpoint>
@@ -83,6 +84,7 @@ const invalidTestSystem = `<?xmlversion="1.0"encoding="UTF-8"?>
           <CapacityInfo>true</CapacityInfo>
           <UploadSessions>false</UploadSessions>
           <IAMSTS>true</IAMSTS>
+          <EnableOnPremArchiveTier>true</EnableOnPremArchiveTier>
        </ProtocolCapabilities>
        <APIEndpoints>
             <IAMEndpoint>a</IAMEndpoint>
@@ -425,6 +427,71 @@ function makeVeeamRequest(params, callback) {
             });
         });
 
+        it('GET system.xml should report the archive tier capability when not provided', done => {
+            // The default reflects the server setup: direct-to-cold is not enabled here
+            const systemWithoutArchiveTier = testSystem.replace(
+                '<EnableOnPremArchiveTier>true</EnableOnPremArchiveTier>\n',
+                '',
+            );
+            const systemWithoutArchiveTierMd5 = crypto
+                .createHash('md5')
+                .update(systemWithoutArchiveTier, 'utf-8')
+                .digest('hex');
+            async.waterfall(
+                [
+                    next =>
+                        makeVeeamRequest(
+                            {
+                                method: 'PUT',
+                                bucket: TEST_BUCKET,
+                                objectKey: '.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml',
+                                headers: {
+                                    'content-length': systemWithoutArchiveTier.length,
+                                    'content-md5': systemWithoutArchiveTierMd5,
+                                    'x-scal-canonical-id': testArn,
+                                },
+                                authCredentials: veeamAuthCredentials,
+                                requestBody: systemWithoutArchiveTier,
+                            },
+                            (err, response) => {
+                                if (err) {
+                                    return done(err);
+                                }
+                                assert.strictEqual(response.statusCode, 200);
+                                return next();
+                            },
+                        ),
+                    next =>
+                        makeVeeamRequest(
+                            {
+                                method: 'GET',
+                                bucket: TEST_BUCKET,
+                                objectKey: '.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml',
+                                headers: {
+                                    'x-scal-canonical-id': testArn,
+                                },
+                                authCredentials: veeamAuthCredentials,
+                            },
+                            (err, response) => {
+                                if (err) {
+                                    return done(err);
+                                }
+                                assert.strictEqual(response.statusCode, 200);
+                                assert.match(
+                                    response.body,
+                                    /<EnableOnPremArchiveTier>false<\/EnableOnPremArchiveTier>/,
+                                );
+                                return next();
+                            },
+                        ),
+                ],
+                err => {
+                    assert.ifError(err);
+                    return done();
+                },
+            );
+        });
+
         [
             ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml', testSystem, testSystemMd5],
             ['.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/capacity.xml', testCapacity, testCapacityMd5],

tests/unit/api/apiUtils/object/objectAttributes.js

@@ -91,6 +91,15 @@ describe('buildXmlAttributes', () => {
             assert.strictEqual(result[0], '<StorageClass>STANDARD</StorageClass>');
         });
 
+        it('should report an archive location with the AWS storage class name', () => {
+            const result = [];
+            const archivedMD = { ...objectMD, 'x-amz-storage-class': 'deep_archive' };
+            buildAttributesXml(archivedMD, userMetadata, ['StorageClass'], result, log);
+
+            assert.strictEqual(result.length, 1);
+            assert.strictEqual(result[0], '<StorageClass>DEEP_ARCHIVE</StorageClass>');
+        });
+
         it('should generate ObjectSize XML', () => {
             const result = [];
             buildAttributesXml(objectMD, userMetadata, ['ObjectSize'], result, log);