CLDSRV-1056: Fail a self-copy when no known master key decrypts its data

lib/api/apiUtils/object/sourceChecksum.js

@@ -0,0 +1,64 @@
+const { PassThrough } = require('stream');
+const async = require('async');
+const { jsutil } = require('arsenal');
+
+const { data } = require('../../../data/wrapper');
+
+/**
+ * Sequentially GET the ordered source `dataLocator` parts into a single
+ * readable stream, reading them in order through `data.get`. Returns the
+ * PassThrough immediately; consumers should pipe it onward and observe
+ * `error` for any read failure along the way.
+ *
+ * @param {Array} dataLocator - ordered source parts
+ * @param {object} log - request logger
+ * @return {PassThrough}
+ */
+function buildSourcePartsStream(dataLocator, log) {
+    const passthrough = new PassThrough();
+    const wrapErr = (err, part) =>
+        Object.assign(err, {
+            copyPart: { key: part.key, dataStoreName: part.dataStoreName, dataStoreType: part.dataStoreType },
+        });
+    async.eachSeries(
+        dataLocator,
+        (part, cb) => {
+            const done = jsutil.once(cb);
+            if (part.dataStoreType === 'azure') {
+                // Azure's data.get writes part bytes into the provided writable
+                // instead of returning a Readable. Pipe a per-part PassThrough
+                // into the master passthrough and use its 'end' as the completion
+                // signal — same pattern arsenal's data.copyObject uses.
+                const perPart = new PassThrough();
+                perPart.once('error', err => done(wrapErr(err, part)));
+                perPart.once('end', () => done());
+                perPart.pipe(passthrough, { end: false });
+                return data.get(part, perPart, log, err => {
+                    if (err) {
+                        perPart.destroy(err);
+                        done(wrapErr(err, part));
+                    }
+                });
+            }
+            return data.get(part, null, log, (err, partStream) => {
+                if (err) {
+                    return done(wrapErr(err, part));
+                }
+                partStream.once('error', err => done(wrapErr(err, part)));
+                partStream.once('end', () => done());
+                partStream.pipe(passthrough, { end: false });
+                return undefined;
+            });
+        },
+        err => {
+            if (err) {
+                passthrough.destroy(err);
+            } else {
+                passthrough.end();
+            }
+        },
+    );
+    return passthrough;
+}
+
+module.exports = { buildSourcePartsStream };

lib/api/apiUtils/object/sourceMasterKey.js

@@ -0,0 +1,126 @@
+const async = require('async');
+const crypto = require('crypto');
+const { jsutil } = require('arsenal');
+
+const kms = require('../../../kms/wrapper');
+const { buildSourcePartsStream } = require('./sourceChecksum');
+
+// `<part number>:<MD5 of the part's plaintext>`
+const partETagPattern = /^\d+:([0-9a-f]{32})$/;
+
+/**
+ * Find the smallest encrypted part of the data that has an MD5. A part is an
+ * MPU part, or the whole object if it was not uploaded in parts.
+ *
+ * An MPU part written by UploadPartCopy can span several locations, and each
+ * of them carries the MD5 of the whole part in its dataStoreETag. So a part is
+ * the run of consecutive locations that share a dataStoreETag.
+ *
+ * An empty part decrypts to the same MD5 under any key, so it is picked only if
+ * every other part is empty too.
+ *
+ * @param {Array} dataLocator - source data locations
+ * @returns {object|null} the part's locations and MD5, or null if no
+ * encrypted part has an MD5
+ */
+function _smallestPart(dataLocator) {
+    const parts = [];
+    dataLocator.forEach(location => {
+        const part = parts[parts.length - 1];
+        if (part && location.dataStoreETag === part.dataStoreETag) {
+            part.locations.push(location);
+            part.size += Number(location.size);
+        } else {
+            parts.push({
+                dataStoreETag: location.dataStoreETag,
+                locations: [location],
+                size: Number(location.size),
+            });
+        }
+    });
+    const rank = part => (part.size === 0 ? Infinity : part.size);
+    return parts.reduce((smallest, part) => {
+        const match = partETagPattern.exec(part.dataStoreETag);
+        if (!match || !part.locations.every(location => location.cipheredDataKey)) {
+            return smallest;
+        }
+        if (smallest && rank(smallest) <= rank(part)) {
+            return smallest;
+        }
+        return { locations: part.locations, md5: match[1], size: part.size };
+    }, null);
+}
+
+/**
+ * Compute the MD5 of the locations' data, decrypted with masterKeyId.
+ *
+ * @param {Array} locations - data locations, in order
+ * @param {string} masterKeyId - master key to decrypt the data with
+ * @param {object} log - request logger
+ * @param {function} cb - cb(err, md5): md5 is the hex digest
+ * @returns {undefined}
+ */
+function _md5WithKey(locations, masterKeyId, log, cb) {
+    const done = jsutil.once(cb);
+    const hash = crypto.createHash('md5');
+    const stream = buildSourcePartsStream(
+        locations.map(location => ({ ...location, masterKeyId })),
+        log,
+    );
+    stream.on('data', chunk => hash.update(chunk));
+    stream.once('error', done);
+    stream.once('end', () => done(null, hash.digest('hex')));
+}
+
+/**
+ * Find which master key decrypts the source data correctly: read one part,
+ * decrypted with each candidate key in turn, and compare its MD5 with the
+ * MD5 recorded for the part.
+ *
+ * @param {Array} dataLocator - source data locations
+ * @param {Array} candidateKeyIds - master key IDs, in the order to try
+ * @param {object} log - request logger
+ * @param {function} cb - cb(err, masterKeyId, reason): masterKeyId is the
+ * first candidate that decrypts the data, or null with the reason when none
+ * does or the data has no MD5 to check; err is the error of the read with the
+ * first candidate, returned when no other candidate decrypts the data
+ * @returns {undefined}
+ */
+function findSourceMasterKey(dataLocator, candidateKeyIds, log, cb) {
+    const part = _smallestPart(dataLocator);
+    if (!part) {
+        return process.nextTick(cb, null, null, 'no MD5 recorded for the encrypted data');
+    }
+    const keyIds = candidateKeyIds.filter(
+        (keyId, i) => keyId && !candidateKeyIds.slice(0, i).some(other => kms.isSameMasterKey(other, keyId, log)),
+    );
+    let firstReadError = null;
+    return async.detectSeries(
+        keyIds,
+        (keyId, next) =>
+            _md5WithKey(part.locations, keyId, log, (err, md5) => {
+                if (err) {
+                    log.debug('cannot read the source data with a candidate master key', {
+                        masterKeyId: keyId,
+                        error: err,
+                    });
+                    if (keyId === keyIds[0]) {
+                        firstReadError = err;
+                    }
+                    return next(null, false);
+                }
+                return next(null, md5 === part.md5);
+            }),
+        (err, keyId) => {
+            if (keyId) {
+                return cb(null, keyId);
+            }
+            if (firstReadError) {
+                return cb(firstReadError);
+            }
+            return cb(null, null, 'no candidate master key decrypts the data');
+        },
+    );
+}
+
+module.exports = { findSourceMasterKey, _smallestPart };

lib/api/objectCopy.js

@@ -24,6 +24,7 @@
 const { setExpirationHeaders } = require('./apiUtils/object/expirationHeaders');
 const { verifyColdObjectAvailable } = require('./apiUtils/object/coldStorage');
 const { setSSEHeaders } = require('./apiUtils/object/sseHeaders');
+const { findSourceMasterKey } = require('./apiUtils/object/sourceMasterKey');
 const { updateEncryption } = require('./apiUtils/bucket/updateEncryption');
 const { initializeInternalLogRequestQueue, queueInternalLogRequest } = require('../utilities/serverAccessLogger');
 
@@ -79,6 +80,33 @@
     );
 }
 
+/**
+ * Tell whether the copy replaces its own source, without keeping the source as
+ * another version: if the copy rewrites the data, the source's data is deleted.
+ *
+ * @param {boolean} sourceIsDestination - source key == destination key
+ * @param {BucketInfo} destBucketMD - destination bucket metadata
+ * @returns {boolean} true if the copy replaces its source
+ */
+function _overwritesSource(sourceIsDestination, destBucketMD) {
+    const vcfg = destBucketMD.getVersioningConfiguration();
+    const isVersionedObj = vcfg && vcfg.Status === 'Enabled';
+    return sourceIsDestination && !isVersionedObj;
+}
+
+/**
+ * Tell whether the copy replaces its own source in the same location, without
+ * keeping the source as another version.
+ *
+ * @param {boolean} sourceIsDestination - source key == destination key
+ * @param {object} storeMetadataParams - destination metadata params
+ * @param {BucketInfo} destBucketMD - destination bucket metadata
+ * @returns {boolean} true if the copy replaces its source in place
+ */
+function _isInPlaceCopy(sourceIsDestination, storeMetadataParams, destBucketMD) {
+    return _overwritesSource(sourceIsDestination, destBucketMD) && storeMetadataParams.locationMatch;
+}
+
 /**
  * Preps metadata to be saved (based on copy or replace request header)
  * @param {object} request - request
@@ -310,6 +338,9 @@
     };
     const websiteRedirectHeader = request.headers['x-amz-website-redirect-location'];
     const responseHeaders = {};
+    // Master key recorded on the source locations, before this copy replaces it
+    // with the master key in the object metadata
+    let recordedMasterKeyId;
 
     if (
         request.headers['x-amz-storage-class'] &&
@@ -474,6 +505,7 @@
                         }
 
                         if (sourceObjMD['x-amz-server-side-encryption']) {
+                            recordedMasterKeyId = dataLocator.find(location => location.masterKeyId)?.masterKeyId;
                             for (let i = 0; i < dataLocator.length; i++) {
                                 dataLocator[i].masterKeyId = sourceObjMD['x-amz-server-side-encryption-aws-kms-key-id'];
                                 dataLocator[i].algorithm = sourceObjMD['x-amz-server-side-encryption'];
@@ -531,6 +563,86 @@
                     ),
                 );
             },
+            function checkSourceMasterKey(
+                storeMetadataParams,
+                dataLocator,
+                sourceBucketMD,
+                destBucketMD,
+                destObjMD,
+                sourceLocationConstraintName,
+                backendInfoDest,
+                serverSideEncryption,
+                next,
+            ) {
+                const forward = () =>
+                    next(
+                        null,
+                        storeMetadataParams,
+                        dataLocator,
+                        sourceBucketMD,
+                        destBucketMD,
+                        destObjMD,
+                        sourceLocationConstraintName,
+                        backendInfoDest,
+                        serverSideEncryption,
+                    );
+                // The master key in the object metadata may not be the one the
+                // data is encrypted with: until CLDSRV-910, a self-copy to another
+                // master key changed only the metadata and left the data encrypted
+                // under the old one. When this copy rewrites such data, it decrypts
+                // it with the wrong key and deletes the original. Before rewriting,
+                // decrypt one part of the data with the master key in the metadata,
+                // then the one recorded on the locations, then the destination's,
+                // and use the first whose result matches the part's MD5; fail the
+                // copy if none does.
+                // Each key tried costs a read of the part and a KMS call, so only
+                // objects that can be in this state are checked. A self-copy that
+                // does not rewrite the data is the only write that records a master
+                // key on the locations, so objects without one are skipped.
+                if (
+                    !recordedMasterKeyId ||
+                    !_overwritesSource(sourceIsDestination, destBucketMD) ||
+                    !dataLocator.some(location => location.cipheredDataKey) ||
+                    (storeMetadataParams.locationMatch && !_mustReencryptData(dataLocator, serverSideEncryption, log))
+                ) {
+                    return forward();
+                }
+                const objectMasterKeyId = dataLocator[0].masterKeyId;
+                const destMasterKeyId =
+                    serverSideEncryption?.configuredMasterKeyId || serverSideEncryption?.masterKeyId;
+                return findSourceMasterKey(
+                    dataLocator,
+                    [objectMasterKeyId, recordedMasterKeyId, destMasterKeyId],
+                    log,
+                    (err, masterKeyId, reason) => {
+                        const logFields = { bucketName: destBucketName, objectKey: destObjectKey, objectMasterKeyId };
+                        if (!err && !masterKeyId) {
+                            log.error('refusing to rewrite the data of a self-copy', {
+                                ...logFields,
+                                candidateMasterKeyIds: [recordedMasterKeyId, destMasterKeyId],
+                                reason,
+                            });
+                        }
+                        const error = err || (!masterKeyId && errors.InternalError);
+                        if (error) {
+                            // eslint-disable-next-line no-param-reassign
+                            request.sourceServerAccessLog && (request.sourceServerAccessLog.error = error);
+                            return next(error, destBucketMD);
+                        }
+                        if (masterKeyId !== objectMasterKeyId) {
+                            log.info('source data is not encrypted with the master key in its metadata', {
+                                ...logFields,
+                                masterKeyId,
+                            });
+                            for (let i = 0; i < dataLocator.length; i++) {
+                                // eslint-disable-next-line no-param-reassign
+                                dataLocator[i].masterKeyId = masterKeyId;
+                            }
+                        }
+                        return forward();
+                    },
+                );
+            },
             function goGetData(
                 storeMetadataParams,
                 dataLocator,
@@ -550,9 +662,7 @@
                 // still send along serverSideEncryption info so algo
                 // and masterKeyId stored properly in metadata
                 if (
-                    sourceIsDestination &&
-                    storeMetadataParams.locationMatch &&
-                    !isVersionedObj &&
+                    _isInPlaceCopy(sourceIsDestination, storeMetadataParams, destBucketMD) &&
                     !_mustReencryptData(dataLocator, serverSideEncryption, log)
                 ) {
                     return next(null, storeMetadataParams, dataLocator, destObjMD, serverSideEncryption, destBucketMD);

tests/unit/api/apiUtils/object/sourceMasterKey.js

@@ -0,0 +1,45 @@
+const assert = require('assert');
+
+const { _smallestPart } = require('../../../../../lib/api/apiUtils/object/sourceMasterKey');
+
+const md5A = 'a'.repeat(32);
+const md5B = 'b'.repeat(32);
+const md5C = 'c'.repeat(32);
+const md5Empty = 'd41d8cd98f00b204e9800998ecf8427e';
+const encrypted = (key, size, dataStoreETag) => ({ key, size, dataStoreETag, cipheredDataKey: 'ZGF0YWtleQ==' });
+
+describe('_smallestPart', () => {
+    it('should pick the smallest part, with every location of a part copied from several locations', () => {
+        const dataLocator = [
+            encrypted(1, 10, `1:${md5A}`),
+            encrypted(2, 4, `2:${md5B}`),
+            encrypted(3, 2, `2:${md5B}`),
+            encrypted(4, 8, `3:${md5C}`),
+        ];
+        assert.deepStrictEqual(_smallestPart(dataLocator), {
+            locations: [dataLocator[1], dataLocator[2]],
+            md5: md5B,
+            size: 6,
+        });
+    });
+
+    it('should skip parts with an unencrypted location or no MD5', () => {
+        const dataLocator = [
+            { key: 1, size: 1, dataStoreETag: `1:${md5A}` },
+            encrypted(2, 2, undefined),
+            encrypted(3, 3, `${md5B}-2`),
+            encrypted(4, 9, `4:${md5C}`),
+        ];
+        assert.deepStrictEqual(_smallestPart(dataLocator).locations, [dataLocator[3]]);
+    });
+
+    it('should pick a part with data over a smaller empty part', () => {
+        const dataLocator = [encrypted(1, 10, `1:${md5A}`), encrypted(2, 0, `2:${md5Empty}`)];
+        assert.deepStrictEqual(_smallestPart(dataLocator).locations, [dataLocator[0]]);
+    });
+
+    it('should pick an empty part when every part is empty', () => {
+        const dataLocator = [encrypted(1, 0, `1:${md5Empty}`)];
+        assert.deepStrictEqual(_smallestPart(dataLocator).locations, [dataLocator[0]]);
+    });
+});