kvm: clone encrypted roots from a dense base image, not from the sparse template

PendingReleaseNotes

@@ -39,3 +39,13 @@ example.ver.1 > example.ver.2:
     which can now be attached to Instances. This is to prevent the Secondary
     Storage to grow to enormous sizes as Linux Distributions keep growing in
     size while a stripped down Linux should fit on a 2.88MB floppy.
+
+
+4.22.0.0 > 4.23.0.0:
+  * KVM/Ceph: RBD volumes can now be encrypted at rest using native librbd
+    LUKS2 (<encryption format='luks2' engine='librbd'>), for both data disks
+    and root disks. Encryption is transparent to the guest and reuses the
+    existing CloudStack volume-encryption passphrase handling, so no
+    additional key store is required. Note: attaching an encrypted RBD volume
+    to a running Instance requires libvirt >= 10.1.0; booting an Instance from
+    an encrypted RBD root disk works on older libvirt.

api/src/main/java/com/cloud/host/Host.java

@@ -56,6 +56,7 @@ public static String[] toStrings(Host.Type... types) {
 
     String HOST_UEFI_ENABLE = "host.uefi.enable";
     String HOST_VOLUME_ENCRYPTION = "host.volume.encryption";
+    String HOST_RBD_VOLUME_ENCRYPTION = "host.volume.encryption.rbd";
     String HOST_INSTANCE_CONVERSION = "host.instance.conversion";
     String HOST_VDDK_SUPPORT = "host.vddk.support";
     String HOST_VDDK_LIB_DIR = "vddk.lib.dir";

api/src/main/java/com/cloud/storage/Storage.java

@@ -172,7 +172,7 @@ public static enum StoragePoolType {
         LVM(false, false, EncryptionSupport.Unsupported), // XenServer local LVM SR
         CLVM(true, false, EncryptionSupport.Unsupported),
         CLVM_NG(true, false, EncryptionSupport.Hypervisor),
-        RBD(true, true, EncryptionSupport.Unsupported), // http://libvirt.org/storage.html#StorageBackendRBD
+        RBD(true, true, EncryptionSupport.Hypervisor), // http://libvirt.org/storage.html#StorageBackendRBD ; encrypted natively by librbd (LUKS2, engine='librbd')
         SharedMountPoint(true, true, EncryptionSupport.Hypervisor),
         VMFS(true, true, EncryptionSupport.Unsupported), // VMware VMFS storage
         PreSetup(true, true, EncryptionSupport.Unsupported), // for XenServer, Storage Pool is set up by customers.

core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java

@@ -24,7 +24,9 @@
 import com.cloud.vm.VirtualMachine;
 import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
 
+import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 
 public class RestoreBackupCommand extends Command  {
     private String vmName;
@@ -40,11 +42,38 @@ public class RestoreBackupCommand extends Command  {
     private Boolean vmExists;
     private VirtualMachine.State vmState;
     private Integer mountTimeout;
+    /**
+     * Passphrase of every encrypted volume being restored, keyed by volume path. The backup itself is
+     * LUKS-encrypted (see TakeBackupCommand), so the agent needs these both to read the backup and to
+     * write the data back through the volume's own encryption.
+     */
+    @LogLevel(LogLevel.Log4jLevel.Off)
+    private Map<String, byte[]> volumePassphrases;
 
     protected RestoreBackupCommand() {
         super();
     }
 
+    public Map<String, byte[]> getVolumePassphrases() {
+        return volumePassphrases;
+    }
+
+    public void setVolumePassphrases(Map<String, byte[]> volumePassphrases) {
+        this.volumePassphrases = volumePassphrases;
+    }
+
+    /** Wipes the passphrases once the command has been handled. */
+    public void clearPassphrases() {
+        if (volumePassphrases == null) {
+            return;
+        }
+        for (byte[] passphrase : volumePassphrases.values()) {
+            if (passphrase != null) {
+                Arrays.fill(passphrase, (byte) 0);
+            }
+        }
+    }
+
     public String getVmName() {
         return vmName;
     }

core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java

@@ -23,7 +23,9 @@
 import com.cloud.agent.api.LogLevel;
 import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
 
+import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 
 public class TakeBackupCommand extends Command {
     private String vmName;
@@ -35,6 +37,15 @@ public class TakeBackupCommand extends Command {
     private Boolean quiesce;
     @LogLevel(LogLevel.Log4jLevel.Off)
     private String mountOptions;
+    /**
+     * Passphrase of every encrypted volume of the VM, keyed by volume path (the uuid the backup
+     * files are named after). Volumes that are not encrypted are absent. The agent uses these to
+     * make the backup itself LUKS-encrypted, so an encrypted volume is never written out in the
+     * clear. Keyed rather than positional because the running-VM path enumerates disks from
+     * libvirt instead of from {@link #volumePaths}.
+     */
+    @LogLevel(LogLevel.Log4jLevel.Off)
+    private Map<String, byte[]> volumePassphrases;
 
     // Incremental backup fields (NAS provider; null/empty for legacy full-only callers).
     private String mode;          // "full" or "incremental"; null => legacy behaviour (script default)
@@ -53,6 +64,26 @@ public TakeBackupCommand(String vmName, String backupPath) {
         this.backupPath = backupPath;
     }
 
+    public Map<String, byte[]> getVolumePassphrases() {
+        return volumePassphrases;
+    }
+
+    public void setVolumePassphrases(Map<String, byte[]> volumePassphrases) {
+        this.volumePassphrases = volumePassphrases;
+    }
+
+    /** Wipes the passphrases once the command has been handled. */
+    public void clearPassphrases() {
+        if (volumePassphrases == null) {
+            return;
+        }
+        for (byte[] passphrase : volumePassphrases.values()) {
+            if (passphrase != null) {
+                Arrays.fill(passphrase, (byte) 0);
+            }
+        }
+    }
+
     public String getVmName() {
         return vmName;
     }

plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java

@@ -36,6 +36,8 @@
 import com.cloud.storage.dao.DiskOfferingDao;
 import com.cloud.storage.dao.StoragePoolHostDao;
 import com.cloud.storage.dao.VolumeDao;
+import org.apache.cloudstack.secret.PassphraseVO;
+import org.apache.cloudstack.secret.dao.PassphraseDao;
 import com.cloud.utils.Pair;
 import com.cloud.utils.component.AdapterBase;
 import com.cloud.utils.db.GlobalLock;
@@ -71,6 +73,7 @@
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -130,6 +133,9 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co
     @Inject
     private VolumeDao volumeDao;
 
+    @Inject
+    private PassphraseDao passphraseDao;
+
     @Inject
     private StoragePoolHostDao storagePoolHostDao;
 
@@ -591,6 +597,11 @@ public Pair<Boolean, Backup> takeBackup(final VirtualMachine vm, Boolean quiesce
             command.setVolumePaths(volumePoolsAndPaths.second());
         }
 
+        // Sent for every VM state: an encrypted volume must not be written to the backup target in
+        // the clear, and the running-VM path enumerates its disks from libvirt rather than from the
+        // paths above.
+        command.setVolumePassphrases(getVolumePassphrases(volumeDao.findByInstance(vm.getId())));
+
         BackupAnswer answer;
         try {
             answer = (BackupAnswer) agentManager.send(host.getId(), command);
@@ -689,9 +700,14 @@ public boolean restoreVMFromBackup(VirtualMachine vm, Backup backup, boolean qui
     }
 
     private Pair<Boolean, String> restoreVMBackup(VirtualMachine vm, Backup backup) {
-        List<Backup.VolumeInfo> backedVolumes = backup.getBackedUpVolumes();
-        List<String> backedVolumesUUIDs = backedVolumes.stream()
+        // Sort once and derive every per-volume list from that same ordering. The UUID list and the
+        // backup file list are consumed index-by-index on the agent side, so they have to agree;
+        // sorting only one of them leaves the two out of step whenever the stored order of the
+        // backed up volumes differs from their device id order.
+        List<Backup.VolumeInfo> backedVolumes = backup.getBackedUpVolumes().stream()
                 .sorted(Comparator.comparingLong(Backup.VolumeInfo::getDeviceId))
+                .collect(Collectors.toList());
+        List<String> backedVolumesUUIDs = backedVolumes.stream()
                 .map(Backup.VolumeInfo::getUuid)
                 .collect(Collectors.toList());
 
@@ -714,6 +730,9 @@ private Pair<Boolean, String> restoreVMBackup(VirtualMachine vm, Backup backup)
         restoreCommand.setRestoreVolumePools(volumePoolsAndPaths.first());
         restoreCommand.setRestoreVolumePaths(volumePoolsAndPaths.second());
         restoreCommand.setBackupFiles(getBackupFiles(backedVolumes));
+        // The backup of an encrypted volume is itself LUKS-encrypted, so the agent needs the
+        // passphrase to read it back and to rewrite the volume through its own encryption.
+        restoreCommand.setVolumePassphrases(getVolumePassphrases(restoreVolumes));
         restoreCommand.setVmExists(vm.getRemoved() == null);
         restoreCommand.setVmState(vm.getState());
         restoreCommand.setMountTimeout(NASBackupRestoreMountTimeout.value());
@@ -742,6 +761,28 @@ private List<String> getBackupFiles(List<Backup.VolumeInfo> backedVolumes) {
         return backupFiles;
     }
 
+    /**
+     * Passphrases of the encrypted volumes among {@code volumes}, keyed by volume path (the uuid the
+     * backup files are named after). Unencrypted volumes are omitted, so an empty map means there is
+     * nothing to encrypt and the backup behaves exactly as before.
+     */
+    private Map<String, byte[]> getVolumePassphrases(List<VolumeVO> volumes) {
+        Map<String, byte[]> passphrases = new HashMap<>();
+        for (VolumeVO volume : volumes) {
+            if (volume.getPassphraseId() == null) {
+                continue;
+            }
+            PassphraseVO passphrase = passphraseDao.findById(volume.getPassphraseId());
+            if (passphrase == null || passphrase.getPassphrase() == null || passphrase.getPassphrase().length == 0) {
+                throw new CloudRuntimeException(String.format(
+                        "Volume %s is encrypted but its passphrase could not be read; refusing to back it up unencrypted",
+                        volume.getUuid()));
+            }
+            passphrases.put(volume.getPath(), passphrase.getPassphrase());
+        }
+        return passphrases;
+    }
+
     private Pair<List<PrimaryDataStoreTO>, List<String>> getVolumePoolsAndPaths(List<VolumeVO> volumes) {
         List<PrimaryDataStoreTO> volumePools = new ArrayList<>();
         List<String> volumePaths = new ArrayList<>();

plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java

@@ -22,6 +22,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
+import java.util.Map;
 import java.util.Optional;
 
 import com.cloud.vm.snapshot.dao.VMSnapshotDao;
@@ -66,6 +67,8 @@
 
 import org.apache.cloudstack.backup.dao.BackupDao;
 import org.apache.cloudstack.backup.dao.BackupDetailsDao;
+import org.apache.cloudstack.secret.PassphraseVO;
+import org.apache.cloudstack.secret.dao.PassphraseDao;
 import org.apache.cloudstack.backup.dao.BackupRepositoryDao;
 import org.apache.cloudstack.backup.dao.BackupOfferingDao;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
@@ -126,6 +129,9 @@ public class NASBackupProviderTest {
     @Mock
     private ResourceLimitService resourceLimitMgr;
 
+    @Mock
+    private PassphraseDao passphraseDao;
+
     @Test
     public void testDeleteBackup() throws OperationTimedoutException, AgentUnavailableException {
         Long hostId = 1L;
@@ -238,9 +244,12 @@ public void takeBackupSuccessfully() throws AgentUnavailableException, Operation
         VolumeVO volume1 = mock(VolumeVO.class);
         Mockito.when(volume1.getState()).thenReturn(Volume.State.Ready);
         Mockito.when(volume1.getSize()).thenReturn(100L);
+        // Unencrypted volumes: nothing for the backup to encrypt.
+        Mockito.when(volume1.getPassphraseId()).thenReturn(null);
         VolumeVO volume2 = mock(VolumeVO.class);
         Mockito.when(volume2.getState()).thenReturn(Volume.State.Ready);
         Mockito.when(volume2.getSize()).thenReturn(200L);
+        Mockito.when(volume2.getPassphraseId()).thenReturn(null);
         Mockito.when(volumeDao.findByInstance(vmId)).thenReturn(List.of(volume1, volume2));
 
         BackupAnswer answer = mock(BackupAnswer.class);
@@ -1034,4 +1043,62 @@ public void sweepContinuesPastFailedTombstoneDelete()
         Mockito.verify(backupDao, Mockito.never()).remove(51L);
         Mockito.verify(backupDao).remove(50L);
     }
+
+    /**
+     * An encrypted volume must not reach the backup target in the clear: its passphrase has to travel
+     * on the command so the agent can make QEMU write the backup already encrypted.
+     */
+    @Test
+    public void takeBackupSendsPassphraseOfEncryptedVolume() throws AgentUnavailableException, OperationTimedoutException {
+        Long vmId = 1L;
+        Long hostId = 2L;
+        Long backupOfferingId = 3L;
+        Long passphraseId = 77L;
+        byte[] secret = "volume-passphrase".getBytes(java.nio.charset.StandardCharsets.UTF_8);
+
+        VMInstanceVO vm = mock(VMInstanceVO.class);
+        Mockito.when(vm.getId()).thenReturn(vmId);
+        Mockito.when(vm.getHostId()).thenReturn(hostId);
+        Mockito.when(vm.getInstanceName()).thenReturn("test-vm");
+        Mockito.when(vm.getBackupOfferingId()).thenReturn(backupOfferingId);
+        Mockito.when(vm.getState()).thenReturn(VMInstanceVO.State.Running);
+
+        BackupRepository backupRepository = mock(BackupRepository.class);
+        Mockito.when(backupRepository.getType()).thenReturn("nfs");
+        Mockito.when(backupRepository.getAddress()).thenReturn("10.0.0.1:/export");
+        Mockito.when(backupRepositoryDao.findByBackupOfferingId(backupOfferingId)).thenReturn(backupRepository);
+
+        HostVO host = mock(HostVO.class);
+        Mockito.when(host.getId()).thenReturn(hostId);
+        Mockito.when(host.getStatus()).thenReturn(Status.Up);
+        Mockito.when(host.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM);
+        Mockito.when(hostDao.findById(hostId)).thenReturn(host);
+
+        VolumeVO encryptedVolume = mock(VolumeVO.class);
+        Mockito.when(encryptedVolume.getState()).thenReturn(Volume.State.Ready);
+        Mockito.when(encryptedVolume.getSize()).thenReturn(100L);
+        Mockito.when(encryptedVolume.getPassphraseId()).thenReturn(passphraseId);
+        Mockito.when(encryptedVolume.getPath()).thenReturn("volume-path-uuid");
+        Mockito.when(volumeDao.findByInstance(vmId)).thenReturn(List.of(encryptedVolume));
+
+        PassphraseVO passphrase = mock(PassphraseVO.class);
+        Mockito.when(passphrase.getPassphrase()).thenReturn(secret);
+        Mockito.when(passphraseDao.findById(passphraseId)).thenReturn(passphrase);
+
+        BackupAnswer answer = mock(BackupAnswer.class);
+        Mockito.when(answer.getResult()).thenReturn(true);
+        Mockito.when(answer.getSize()).thenReturn(100L);
+        Mockito.when(agentManager.send(anyLong(), Mockito.any(TakeBackupCommand.class))).thenReturn(answer);
+        Mockito.when(backupDao.persist(Mockito.any(BackupVO.class))).thenAnswer(invocation -> invocation.getArgument(0));
+        Mockito.when(backupDao.update(Mockito.anyLong(), Mockito.any(BackupVO.class))).thenReturn(true);
+
+        nasBackupProvider.takeBackup(vm, false, false);
+
+        ArgumentCaptor<TakeBackupCommand> captor = ArgumentCaptor.forClass(TakeBackupCommand.class);
+        Mockito.verify(agentManager).send(anyLong(), captor.capture());
+        Map<String, byte[]> sent = captor.getValue().getVolumePassphrases();
+        Assert.assertNotNull("the encrypted volume's passphrase must be sent to the agent", sent);
+        Assert.assertArrayEquals(secret, sent.get("volume-path-uuid"));
+    }
+
 }

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -19,6 +19,7 @@
 import static com.cloud.host.Host.HOST_CDROM_MAX_COUNT;
 import static com.cloud.host.Host.HOST_INSTANCE_CONVERSION;
 import static com.cloud.host.Host.HOST_OVFTOOL_VERSION;
+import static com.cloud.host.Host.HOST_RBD_VOLUME_ENCRYPTION;
 import static com.cloud.host.Host.HOST_VDDK_LIB_DIR;
 import static com.cloud.host.Host.HOST_VDDK_SUPPORT;
 import static com.cloud.host.Host.HOST_VDDK_VERSION;
@@ -91,6 +92,7 @@
 import org.apache.cloudstack.storage.volume.VolumeOnStorageTO;
 import org.apache.cloudstack.utils.bytescale.ByteScaleUtils;
 import org.apache.cloudstack.utils.cryptsetup.CryptSetup;
+import org.apache.cloudstack.utils.rbd.RbdEncryption;
 import org.apache.cloudstack.utils.hypervisor.HypervisorUtils;
 import org.apache.cloudstack.utils.linux.CPUStat;
 import org.apache.cloudstack.utils.linux.KVMHostInfo;
@@ -3882,7 +3884,9 @@ public int compare(final DiskTO arg0, final DiskTO arg1) {
                 if (volumeObjectTO.requiresEncryption() &&
                         pool.getType().encryptionSupportMode() == Storage.EncryptionSupport.Hypervisor ) {
                     String secretUuid = createLibvirtVolumeSecret(conn, volumeObjectTO.getPath(), volumeObjectTO.getPassphrase());
-                    DiskDef.LibvirtDiskEncryptDetails encryptDetails = new DiskDef.LibvirtDiskEncryptDetails(secretUuid, QemuObject.EncryptFormat.enumValue(volumeObjectTO.getEncryptFormat()));
+                    // RBD volumes are encrypted natively by librbd, so request the librbd encryption engine.
+                    String encryptEngine = (pool.getType() == StoragePoolType.RBD) ? "librbd" : null;
+                    DiskDef.LibvirtDiskEncryptDetails encryptDetails = new DiskDef.LibvirtDiskEncryptDetails(secretUuid, QemuObject.EncryptFormat.enumValue(volumeObjectTO.getEncryptFormat()), encryptEngine);
                     disk.setLibvirtDiskEncryptDetails(encryptDetails);
                 }
             }
@@ -4410,6 +4414,7 @@ public StartupCommand[] initialize() {
         cmd.setGatewayIpAddress(localGateway);
         cmd.setIqn(getIqn());
         cmd.getHostDetails().put(HOST_VOLUME_ENCRYPTION, String.valueOf(hostSupportsVolumeEncryption()));
+        cmd.getHostDetails().put(HOST_RBD_VOLUME_ENCRYPTION, String.valueOf(hostSupportsRbdVolumeEncryption()));
         cmd.setHostTags(getHostTags());
         boolean instanceConversionSupported = hostSupportsInstanceConversion();
         cmd.getHostDetails().put(HOST_INSTANCE_CONVERSION, String.valueOf(instanceConversionSupported));
@@ -6195,7 +6200,10 @@ public boolean isHostSecured() {
     }
 
     /**
-     * Test host for volume encryption support
+     * Test host for qemu-native LUKS volume encryption (qemu-img LUKS support + cryptsetup),
+     * reported as {@code host.volume.encryption}. RBD/librbd encryption support is a separate
+     * capability, reported as {@code host.volume.encryption.rbd}
+     * (see {@link #hostSupportsRbdVolumeEncryption()}).
      * @return boolean
      */
     public boolean hostSupportsVolumeEncryption() {
@@ -6220,6 +6228,13 @@ public boolean hostSupportsVolumeEncryption() {
         return true;
     }
 
+    /**
+     * Test host for librbd native LUKS encryption support (rbd CLI with the encryption subcommand).
+     */
+    public boolean hostSupportsRbdVolumeEncryption() {
+        return new RbdEncryption().isSupported();
+    }
+
     public boolean isSecureMode(String bootMode) {
         if (StringUtils.isNotBlank(bootMode) && "secure".equalsIgnoreCase(bootMode)) {
             return true;