diff --git a/compute/src/main/java/org/zstack/compute/vm/NewVmInstanceMsgBuilder.java b/compute/src/main/java/org/zstack/compute/vm/NewVmInstanceMsgBuilder.java index 493c365758d..549542d6a0e 100644 --- a/compute/src/main/java/org/zstack/compute/vm/NewVmInstanceMsgBuilder.java +++ b/compute/src/main/java/org/zstack/compute/vm/NewVmInstanceMsgBuilder.java @@ -126,6 +126,7 @@ public static CreateVmInstanceMsg fromAPINewVmInstanceMsg(NewVmInstanceMessage2 cmsg.setHeaders(api.getHeaders()); cmsg.setSystemTags(api.getSystemTags()); cmsg.setUserTags(api.getUserTags()); + cmsg.setEncrypted(msg.getEncrypted()); return cmsg; } diff --git a/compute/src/main/java/org/zstack/compute/vm/VmAllocateVolumeFlow.java b/compute/src/main/java/org/zstack/compute/vm/VmAllocateVolumeFlow.java index bb21cd4a0e5..81bdea71966 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmAllocateVolumeFlow.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmAllocateVolumeFlow.java @@ -118,6 +118,8 @@ protected List prepareMsg(Map ctx) { if (disk != null && !isEmpty(disk.getSystemTags())) { tags.addAll(disk.getSystemTags()); } + Boolean volEnc = disk != null && Boolean.TRUE.equals(disk.getEncrypted()); + msg.setEncrypted(volEnc); } else if (vspec.isData()) { DiskAO disk = isEmpty(spec.getDataDisks()) ? null : spec.getDataDisks().size() > dataVolumeIndex ? spec.getDataDisks().get(dataVolumeIndex) : null; @@ -139,6 +141,7 @@ protected List prepareMsg(Map ctx) { if (disk != null && !isEmpty(disk.getSystemTags())) { tags.addAll(disk.getSystemTags()); } + msg.setEncrypted(disk != null && Boolean.TRUE.equals(disk.getEncrypted())); dataVolumeIndex++; } else { diff --git a/compute/src/main/java/org/zstack/compute/vm/VmInstanceManagerImpl.java b/compute/src/main/java/org/zstack/compute/vm/VmInstanceManagerImpl.java index 2af241834d7..cc9d4eaf834 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmInstanceManagerImpl.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmInstanceManagerImpl.java @@ -1178,6 +1178,7 @@ protected void doCreateVmInstance(final CreateVmInstanceMsg msg, final APICreate vo.setPlatform(msg.getPlatform() != null ? msg.getPlatform() : image.getPlatform().toString()); vo.setGuestOsType(msg.getGuestOsType() != null ? msg.getGuestOsType() : image.getGuestOsType()); vo.setArchitecture(msg.getArchitecture() != null ? msg.getArchitecture() : image.getArchitecture()); + vo.setEncrypted(Boolean.TRUE.equals(msg.getEncrypted())); String vmType = msg.getType() == null ? VmInstanceConstant.USER_VM_TYPE : msg.getType(); VmInstanceType type = VmInstanceType.valueOf(vmType); VmInstanceFactory factory = getVmInstanceFactory(type); diff --git a/compute/src/main/java/org/zstack/compute/vm/VmInstanceUtils.java b/compute/src/main/java/org/zstack/compute/vm/VmInstanceUtils.java index bebcae3520b..f6247779c0b 100644 --- a/compute/src/main/java/org/zstack/compute/vm/VmInstanceUtils.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmInstanceUtils.java @@ -143,9 +143,71 @@ public static CreateVmInstanceMsg fromAPICreateVmInstanceMsg(APICreateVmInstance } } + // applyForceEncryptEnvOverride(cmsg); return cmsg; } + /** + * Temporary debug switch. Priority (first match wins): + *
    + *
  1. {@link #FORCE_ENCRYPT_VOLUME_HARDCODED} — flip in code, rebuild, deploy. + * Use this when the deployment regenerates setenv.sh / systemd unit and + * JVM properties can't be reliably injected.
    2. + *
  2. System property {@code zstack.force.encrypt.volume} — + * pass via {@code -Dzstack.force.encrypt.volume=true}.
    3. + *
  3. Environment variable {@code ZSTACK_FORCE_ENCRYPT_VOLUME}.
    4. + *
+ * When the switch is on, every {@link DiskAO} on the {@link CreateVmInstanceMsg} + * is force-marked encrypted=true; when off, encrypted=false. Covers all five + * disk sources funneled into this msg by {@link #fromAPICreateVmInstanceMsg}: + *
    + *
  • root disk (empty / from-image)
    • + *
  • data disks from {@code APICreateVmInstanceMsg.diskAOs} + * (from-image / from-existing-volume)
    • + *
  • data disks from the legacy {@code dataDiskOfferingUuids / + * dataDiskSizes} path (deprecatedDataVolumeSpecs)
    • + *
+ */ + private static final Boolean FORCE_ENCRYPT_VOLUME_HARDCODED = true; + static final String FORCE_ENCRYPT_VOLUME_ENV = "ZSTACK_FORCE_ENCRYPT_VOLUME"; + private static final String FORCE_ENCRYPT_VOLUME_PROPERTY = "zstack.force.encrypt.volume"; + + private static boolean isForceEncryptVolume() { + if (FORCE_ENCRYPT_VOLUME_HARDCODED != null) { + return FORCE_ENCRYPT_VOLUME_HARDCODED; + } + String v = System.getProperty(FORCE_ENCRYPT_VOLUME_PROPERTY); + if (v == null || v.isEmpty()) { + v = System.getenv(FORCE_ENCRYPT_VOLUME_ENV); + } + if (v == null || v.isEmpty()) { + return false; + } + v = v.trim().toLowerCase(); + return v.equals("1") || v.equals("true") || v.equals("yes") || v.equals("on"); + } + + private static void applyForceEncryptEnvOverride(CreateVmInstanceMsg cmsg) { + boolean forceOn = isForceEncryptVolume(); + if (cmsg.getRootDisk() != null) { + cmsg.getRootDisk().setEncrypted(forceOn); + } + if (cmsg.getDataDisks() != null) { + for (DiskAO d : cmsg.getDataDisks()) { + if (d != null) { + d.setEncrypted(forceOn); + } + } + } + if (cmsg.getDeprecatedDataVolumeSpecs() != null) { + for (DiskAO d : cmsg.getDeprecatedDataVolumeSpecs()) { + if (d != null) { + d.setEncrypted(forceOn); + } + } + } + } + private static String getPSUuidForDataVolume(List systemTags){ if (systemTags == null || systemTags.isEmpty()){ return null; diff --git a/compute/src/main/java/org/zstack/compute/vm/VmInstantiateOtherDiskFlow.java b/compute/src/main/java/org/zstack/compute/vm/VmInstantiateOtherDiskFlow.java index 69537bd55f3..d6dd34d38b6 100644 --- a/compute/src/main/java/org/zstack/compute/vm/VmInstantiateOtherDiskFlow.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmInstantiateOtherDiskFlow.java @@ -86,6 +86,7 @@ public void setup() { } else if (isAttachDataVolume()) { VolumeVO volume = Q.New(VolumeVO.class).eq(VolumeVO_.uuid, diskAO.getSourceUuid()).find(); volumeInventory = VolumeInventory.valueOf(volume); + setupEncryptExistingVolumeFlow(); setupAttachVolumeFlows(); } else if (diskAO.getSourceUuid() != null && diskAO.getSourceType() != null) { setupAttachOtherDiskFlows(); @@ -180,6 +181,7 @@ public void run(final FlowTrigger innerTrigger, Map data) { msg.setDiskOfferingUuid(diskAO.getDiskOfferingUuid()); msg.setPrimaryStorageUuid(allocatedPrimaryStorageUuid); msg.setDescription(String.format("vm-%s-data-volume", vmUuid)); + msg.setEncrypted(Boolean.TRUE.equals(diskAO.getEncrypted())); bus.makeLocalServiceId(msg, VolumeConstant.SERVICE_ID); bus.send(msg, new CloudBusCallBack(innerTrigger) { @Override @@ -328,6 +330,7 @@ public void run(final FlowTrigger innerTrigger, Map data) { } else { cmsg.setPrimaryStorageUuid(allocatedPrimaryStorageUuid[0]); } + cmsg.setEncrypted(Boolean.TRUE.equals(diskAO.getEncrypted())); bus.makeLocalServiceId(cmsg, VolumeConstant.SERVICE_ID); bus.send(cmsg, new CloudBusCallBack(innerTrigger) { @@ -404,6 +407,56 @@ public void run(MessageReply reply) { }); } + /** + * When the caller requested an encrypted data volume (DiskAO.encrypted=true) but the + * existing source volume is not yet encrypted, transition the source bits to LUKS + * in place before attaching. Delegates to {@code EncryptVolumeMsg} so the actual + * key/secret/PS-conversion logic lives in {@code VolumeBase} (shared with the + * create-data-volume-from-template flow). + * + *

Skipped when: + *

    + *
  • {@code DiskAO.encrypted} is false/null, or
    • + *
  • the source volume is already encrypted (no-op transition).
    • + *
+ */ + private void setupEncryptExistingVolumeFlow() { + if (!Boolean.TRUE.equals(diskAO.getEncrypted())) { + return; + } + if (volumeInventory != null && Boolean.TRUE.equals(volumeInventory.getEncrypted())) { + return; + } + flow(new NoRollbackFlow() { + String __name__ = String.format("encrypt-existing-data-volume-%s-in-place", + diskAO.getSourceUuid()); + + @Override + public void run(final FlowTrigger innerTrigger, Map data) { + EncryptVolumeMsg emsg = new EncryptVolumeMsg(); + emsg.setVolumeUuid(volumeInventory.getUuid()); + emsg.setHostUuid(hostUuid); + emsg.setPurpose("attach-existing-disk-as-encrypted-data-volume"); + bus.makeTargetServiceIdByResourceUuid(emsg, VolumeConstant.SERVICE_ID, + volumeInventory.getUuid()); + bus.send(emsg, new CloudBusCallBack(innerTrigger) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + innerTrigger.fail(reply.getError()); + return; + } + EncryptVolumeReply er = reply.castReply(); + if (er.getInventory() != null) { + volumeInventory = er.getInventory(); + } + innerTrigger.next(); + } + }); + } + }); + } + private void setupAttachOtherDiskFlows() { flow(new NoRollbackFlow() { String __name__ = String.format("attach-other-Disk-to-vm-%s", vmUuid); diff --git a/compute/src/main/java/org/zstack/compute/vm/VmMigrateOnHypervisorFlow.java b/compute/src/main/java/org/zstack/compute/vm/VmMigrateOnHypervisorFlow.java index 24529a31df3..b149e8958c5 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmMigrateOnHypervisorFlow.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmMigrateOnHypervisorFlow.java @@ -53,6 +53,11 @@ public void run(final FlowTrigger chain, Map data) { msg.setMigrateFromDestination(migrateFromDest); msg.setStrategy(strategy); msg.setDownTime(downTime); + @SuppressWarnings("unchecked") + java.util.Map luksSecrets = spec.getExtensionData("VolumeLuksSecrets", java.util.Map.class); + if (luksSecrets != null) { + msg.setVolumeLuksSecrets(luksSecrets); + } bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, msg.getHostUuid()); bus.send(msg, new CloudBusCallBack(chain) { @Override diff --git a/conf/db/zsv/V5.1.0__schema.sql b/conf/db/zsv/V5.1.0__schema.sql new file mode 100644 index 00000000000..e0d7796c205 --- /dev/null +++ b/conf/db/zsv/V5.1.0__schema.sql @@ -0,0 +1,43 @@ +CREATE TABLE IF NOT EXISTS `zstack`.`TpmKeyBackupVO` ( + `uuid` char(32) NOT NULL UNIQUE, + `lastOpDate` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `createDate` timestamp NOT NULL DEFAULT '1999-12-31 23:59:59', + PRIMARY KEY (`uuid`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +DELETE FROM `EncryptedResourceKeyRefVO` + WHERE `resourceUuid` NOT IN (SELECT `uuid` FROM `ResourceVO`); +ALTER TABLE `EncryptedResourceKeyRefVO` + ADD CONSTRAINT `fkEncryptedResourceKeyRefResourceVO` FOREIGN KEY (`resourceUuid`) REFERENCES `ResourceVO`(`uuid`) + ON DELETE CASCADE; + +-- Volume LUKS encryption flag (API opt-in + EncryptedResourceKeyRefVO binding) + +ALTER TABLE `zstack`.`VolumeEO` ADD COLUMN `encrypted` tinyint(1) NOT NULL DEFAULT 0; +ALTER TABLE `zstack`.`VolumeSnapshotEO` ADD COLUMN `encrypted` tinyint(1) NOT NULL DEFAULT 0; +ALTER TABLE `zstack`.`VmInstanceEO` ADD COLUMN `encrypted` tinyint(1) NOT NULL DEFAULT 0; + +DROP VIEW IF EXISTS `zstack`.`VolumeVO`; +CREATE VIEW `zstack`.`VolumeVO` AS +SELECT uuid, name, description, primaryStorageUuid, vmInstanceUuid, diskOfferingUuid, + rootImageUuid, installPath, type, status, size, actualSize, deviceId, format, state, createDate, lastOpDate, + isShareable, volumeQos, lastVmInstanceUuid, lastDetachDate, lastAttachDate, protocol, encrypted +FROM `zstack`.`VolumeEO` +WHERE deleted IS NULL; + +DROP VIEW IF EXISTS `zstack`.`VolumeSnapshotVO`; +CREATE VIEW `zstack`.`VolumeSnapshotVO` AS +SELECT uuid, name, description, type, volumeUuid, format, treeUuid, parentUuid, + primaryStorageUuid, primaryStorageInstallPath, distance, size, latest, + fullSnapshot, encrypted, volumeType, state, status, createDate, lastOpDate +FROM `zstack`.`VolumeSnapshotEO` +WHERE deleted IS NULL; + +DROP VIEW IF EXISTS `zstack`.`VmInstanceVO`; +CREATE VIEW `zstack`.`VmInstanceVO` AS +SELECT uuid, name, description, zoneUuid, clusterUuid, imageUuid, hostUuid, internalId, + lastHostUuid, instanceOfferingUuid, rootVolumeUuid, defaultL3NetworkUuid, type, + hypervisorType, cpuNum, cpuSpeed, memorySize, reservedMemorySize, platform, + guestOsType, allocatorStrategy, createDate, lastOpDate, state, architecture, encrypted +FROM `zstack`.`VmInstanceEO` +WHERE deleted IS NULL; diff --git a/conf/serviceConfig/volume.xml b/conf/serviceConfig/volume.xml index 03fcb26b8de..952b4eda740 100755 --- a/conf/serviceConfig/volume.xml +++ b/conf/serviceConfig/volume.xml @@ -20,6 +20,10 @@ org.zstack.header.volume.APIChangeVolumeStateMsg + + org.zstack.header.volume.APIChangeVolumeEncryptionMsg + + org.zstack.header.volume.APICreateVolumeSnapshotMsg diff --git a/conf/springConfigXml/VolumeManager.xml b/conf/springConfigXml/VolumeManager.xml old mode 100755 new mode 100644 index 977134a8873..9ef2a853eb6 --- a/conf/springConfigXml/VolumeManager.xml +++ b/conf/springConfigXml/VolumeManager.xml @@ -92,4 +92,75 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/core/src/main/java/org/zstack/core/trash/TrashType.java b/core/src/main/java/org/zstack/core/trash/TrashType.java index a3ec4d62ea6..2e77eadff08 100644 --- a/core/src/main/java/org/zstack/core/trash/TrashType.java +++ b/core/src/main/java/org/zstack/core/trash/TrashType.java @@ -7,6 +7,8 @@ public enum TrashType { MigrateVolume, MigrateImage, MigrateVolumeSnapshot, + ConvertVolumeEncryption, + ConvertVolumeSnapshotEncryption, RevertVolume, VolumeSnapshot, ReimageVolume, diff --git a/header/src/main/java/org/zstack/header/host/MigrateVmOnHypervisorMsg.java b/header/src/main/java/org/zstack/header/host/MigrateVmOnHypervisorMsg.java index 21a86817d19..cc4477129e4 100755 --- a/header/src/main/java/org/zstack/header/host/MigrateVmOnHypervisorMsg.java +++ b/header/src/main/java/org/zstack/header/host/MigrateVmOnHypervisorMsg.java @@ -19,6 +19,7 @@ public static enum StorageMigrationPolicy { private boolean migrateFromDestination; // A map from old disk to new disk private Map diskMigrationMap; + private Map volumeLuksSecrets; private boolean reload; private Integer downTime; private long bandwidth; @@ -107,4 +108,12 @@ public long getBandwidth() { public void setBandwidth(long bandwidth) { this.bandwidth = bandwidth; } + + public Map getVolumeLuksSecrets() { + return volumeLuksSecrets; + } + + public void setVolumeLuksSecrets(Map volumeLuksSecrets) { + this.volumeLuksSecrets = volumeLuksSecrets; + } } diff --git a/header/src/main/java/org/zstack/header/image/CreateTemporaryRootVolumeTemplateFromVolumeSnapshotMsg.java b/header/src/main/java/org/zstack/header/image/CreateTemporaryRootVolumeTemplateFromVolumeSnapshotMsg.java index 2aa59085fad..e766932d0ac 100644 --- a/header/src/main/java/org/zstack/header/image/CreateTemporaryRootVolumeTemplateFromVolumeSnapshotMsg.java +++ b/header/src/main/java/org/zstack/header/image/CreateTemporaryRootVolumeTemplateFromVolumeSnapshotMsg.java @@ -15,6 +15,7 @@ public class CreateTemporaryRootVolumeTemplateFromVolumeSnapshotMsg extends Need private boolean system; private SessionInventory session; private boolean virtio = true; + private Boolean encrypted; @Override public void setSnapshotUuid(String snapshotUuid) { @@ -79,6 +80,15 @@ public boolean isSystem() { public void setSystem(boolean system) { this.system = system; } + + public Boolean getEncrypted() { + return encrypted; + } + + public void setEncrypted(Boolean encrypted) { + this.encrypted = encrypted; + } + @Override public SessionInventory getSession() { diff --git a/header/src/main/java/org/zstack/header/secret/SecretHostEnsureLuksSecretFileMsg.java b/header/src/main/java/org/zstack/header/secret/SecretHostEnsureLuksSecretFileMsg.java new file mode 100644 index 00000000000..425a35a3d15 --- /dev/null +++ b/header/src/main/java/org/zstack/header/secret/SecretHostEnsureLuksSecretFileMsg.java @@ -0,0 +1,28 @@ +package org.zstack.header.secret; + +import org.zstack.header.host.HostMessage; +import org.zstack.header.log.NoLogging; +import org.zstack.header.message.NeedReplyMessage; + +public class SecretHostEnsureLuksSecretFileMsg extends NeedReplyMessage implements HostMessage { + private String hostUuid; + @NoLogging + private String dekBase64; + + @Override + public String getHostUuid() { + return hostUuid; + } + + public void setHostUuid(String hostUuid) { + this.hostUuid = hostUuid; + } + + public String getDekBase64() { + return dekBase64; + } + + public void setDekBase64(String dekBase64) { + this.dekBase64 = dekBase64; + } +} diff --git a/header/src/main/java/org/zstack/header/secret/SecretHostEnsureLuksSecretFileReply.java b/header/src/main/java/org/zstack/header/secret/SecretHostEnsureLuksSecretFileReply.java new file mode 100644 index 00000000000..129cc5cb2e5 --- /dev/null +++ b/header/src/main/java/org/zstack/header/secret/SecretHostEnsureLuksSecretFileReply.java @@ -0,0 +1,18 @@ +package org.zstack.header.secret; + +import org.zstack.header.message.MessageReply; + +public class SecretHostEnsureLuksSecretFileReply extends MessageReply { + public static final String ERROR_CODE_KEYS_NOT_ON_DISK = "KEY_AGENT_KEYS_NOT_ON_DISK"; + public static final String ERROR_CODE_KEY_FILES_INTEGRITY_MISMATCH = "KEY_AGENT_KEY_FILES_INTEGRITY_MISMATCH"; + + private String secFilePath; + + public String getSecFilePath() { + return secFilePath; + } + + public void setSecFilePath(String secFilePath) { + this.secFilePath = secFilePath; + } +} diff --git a/header/src/main/java/org/zstack/header/secret/SecretHostGetReply.java b/header/src/main/java/org/zstack/header/secret/SecretHostGetReply.java index cd474433110..500b42a9b38 100644 --- a/header/src/main/java/org/zstack/header/secret/SecretHostGetReply.java +++ b/header/src/main/java/org/zstack/header/secret/SecretHostGetReply.java @@ -1,5 +1,6 @@ package org.zstack.header.secret; +import org.zstack.header.errorcode.ErrorCode; import org.zstack.header.message.MessageReply; /** Reply for SecretHostGetMsg. */ @@ -15,4 +16,21 @@ public String getSecretUuid() { public void setSecretUuid(String secretUuid) { this.secretUuid = secretUuid; } + + /** + * Distinguish "secret not present on host" (idempotent re-define needed) + * from genuine RPC / agent failures. key-agent's not-found surfaces either + * as the canonical {@link #ERROR_CODE_SECRET_NOT_FOUND} code or embedded + * in {@code details} depending on the bus hop. + */ + public static boolean isSecretNotFound(ErrorCode err) { + if (err == null) { + return false; + } + if (ERROR_CODE_SECRET_NOT_FOUND.equals(err.getCode())) { + return true; + } + String details = err.getDetails(); + return details != null && details.contains(ERROR_CODE_SECRET_NOT_FOUND); + } } diff --git a/header/src/main/java/org/zstack/header/storage/migration/KvmBlockLiveMigrationExtensionPoint.java b/header/src/main/java/org/zstack/header/storage/migration/KvmBlockLiveMigrationExtensionPoint.java new file mode 100644 index 00000000000..4c5252f9552 --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/migration/KvmBlockLiveMigrationExtensionPoint.java @@ -0,0 +1,17 @@ +package org.zstack.header.storage.migration; + +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.host.HostInventory; +import org.zstack.header.vm.VmInstanceInventory; +import org.zstack.header.volume.VolumeVO; + +import java.util.List; +import java.util.Map; + +public interface KvmBlockLiveMigrationExtensionPoint { + ErrorCode beforeBlockLiveMigration(HostInventory dstHost, + VmInstanceInventory vm, + List volumesToMigrate, + Map volumeMappingDict, + Map targetVolumeLuksSecrets); +} diff --git a/header/src/main/java/org/zstack/header/storage/migration/KvmMigrateVmWithStorageExtensionPoint.java b/header/src/main/java/org/zstack/header/storage/migration/KvmMigrateVmWithStorageExtensionPoint.java new file mode 100644 index 00000000000..af643b987f0 --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/migration/KvmMigrateVmWithStorageExtensionPoint.java @@ -0,0 +1,24 @@ +package org.zstack.header.storage.migration; + +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.host.HostInventory; +import org.zstack.header.host.MigrateVmOnHypervisorMsg; +import org.zstack.header.vm.VmInstanceInventory; +import org.zstack.header.volume.VolumeInventory; +import org.zstack.header.volume.VolumeLuksAgentSpec; +import org.zstack.header.volume.VolumeVO; + +import java.util.List; +import java.util.Map; + +public interface KvmMigrateVmWithStorageExtensionPoint { + ErrorCode beforeMigrateVmWithStorage(HostInventory dstHost, + VmInstanceInventory vm, + List volumesToMigrate, + Map volumeMappingDict, + MigrateVmOnHypervisorMsg msg); + + VolumeLuksAgentSpec prepareVolumeLuksAgentSpec(String hostUuid, VolumeInventory volume); + + String prepareVolumeEncryptedDek(String hostUuid, VolumeInventory volume); +} diff --git a/header/src/main/java/org/zstack/header/storage/migration/KvmStorageLiveMigrationExtensionPoint.java b/header/src/main/java/org/zstack/header/storage/migration/KvmStorageLiveMigrationExtensionPoint.java new file mode 100644 index 00000000000..cdf63b59f72 --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/migration/KvmStorageLiveMigrationExtensionPoint.java @@ -0,0 +1,37 @@ +package org.zstack.header.storage.migration; + +import org.zstack.header.core.workflow.Flow; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.storage.primary.InstantiateTemporaryVolumeOnPrimaryStorageMsg; +import org.zstack.header.volume.CreateVolumeMsg; +import org.zstack.header.volume.VolumeInventory; +import org.zstack.header.volume.VolumeVO; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public interface KvmStorageLiveMigrationExtensionPoint { + default List getBeforeCreateTemporaryVolumeFlows(String vmUuid, + String srcHostUuid, + List volumesToMigrate, + Map sourceVolumeVirtualSizes) { + return Collections.emptyList(); + } + + default ErrorCode beforeCreateTemporaryVolume(CreateVolumeMsg msg, + VolumeVO sourceVolume, + Map sourceVolumeVirtualSizes) { + return null; + } + + default ErrorCode afterCreateTemporaryVolume(VolumeVO sourceVolume, VolumeInventory temporaryVolume) { + return null; + } + + default ErrorCode beforeInstantiateTemporaryVolume(String hostUuid, + VolumeInventory temporaryVolume, + InstantiateTemporaryVolumeOnPrimaryStorageMsg msg) { + return null; + } +} diff --git a/header/src/main/java/org/zstack/header/storage/primary/APIGetTrashOnPrimaryStorageMsg.java b/header/src/main/java/org/zstack/header/storage/primary/APIGetTrashOnPrimaryStorageMsg.java index 5b8d58d3fd3..4e690282be9 100644 --- a/header/src/main/java/org/zstack/header/storage/primary/APIGetTrashOnPrimaryStorageMsg.java +++ b/header/src/main/java/org/zstack/header/storage/primary/APIGetTrashOnPrimaryStorageMsg.java @@ -20,7 +20,8 @@ public class APIGetTrashOnPrimaryStorageMsg extends APISyncCallMessage implement private String resourceUuid; @APIParam(required = false) private String resourceType; - @APIParam(required = false, validValues = {"MigrateVolume", "MigrateVolumeSnapshot", "RevertVolume", "VolumeSnapshot"}) + @APIParam(required = false, validValues = {"MigrateVolume", "MigrateVolumeSnapshot", "ConvertVolumeEncryption", + "ConvertVolumeSnapshotEncryption", "RevertVolume", "VolumeSnapshot", "ReimageVolume"}) private String trashType; public String getUuid() { diff --git a/header/src/main/java/org/zstack/header/storage/primary/ConvertVolumeEncryptionOnPrimaryStorageMsg.java b/header/src/main/java/org/zstack/header/storage/primary/ConvertVolumeEncryptionOnPrimaryStorageMsg.java new file mode 100644 index 00000000000..e44fb3756b7 --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/primary/ConvertVolumeEncryptionOnPrimaryStorageMsg.java @@ -0,0 +1,114 @@ +package org.zstack.header.storage.primary; + +import org.zstack.header.message.NeedReplyMessage; +import org.zstack.header.volume.VolumeInventory; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +public class ConvertVolumeEncryptionOnPrimaryStorageMsg extends NeedReplyMessage implements PrimaryStorageMessage { + private String primaryStorageUuid; + private String hostUuid; + private VolumeInventory volume; + private boolean targetEncrypted; + private List items = new ArrayList<>(); + + @Override + public String getPrimaryStorageUuid() { + return primaryStorageUuid; + } + + public void setPrimaryStorageUuid(String primaryStorageUuid) { + this.primaryStorageUuid = primaryStorageUuid; + } + + public String getHostUuid() { + return hostUuid; + } + + public void setHostUuid(String hostUuid) { + this.hostUuid = hostUuid; + } + + public VolumeInventory getVolume() { + return volume; + } + + public void setVolume(VolumeInventory volume) { + this.volume = volume; + } + + public boolean isTargetEncrypted() { + return targetEncrypted; + } + + public void setTargetEncrypted(boolean targetEncrypted) { + this.targetEncrypted = targetEncrypted; + } + + public List getItems() { + return items; + } + + public void setItems(List items) { + this.items = items; + } + + public static class VolumeEncryptionConversionItem implements Serializable { + private String resourceUuid; + private String resourceType; + private String sourceInstallPath; + private String sourceTrashInstallPath; + private String targetInstallPath; + private String targetBackingInstallPath; + + public String getResourceUuid() { + return resourceUuid; + } + + public void setResourceUuid(String resourceUuid) { + this.resourceUuid = resourceUuid; + } + + public String getResourceType() { + return resourceType; + } + + public void setResourceType(String resourceType) { + this.resourceType = resourceType; + } + + public String getSourceInstallPath() { + return sourceInstallPath; + } + + public void setSourceInstallPath(String sourceInstallPath) { + this.sourceInstallPath = sourceInstallPath; + } + + public String getSourceTrashInstallPath() { + return sourceTrashInstallPath; + } + + public void setSourceTrashInstallPath(String sourceTrashInstallPath) { + this.sourceTrashInstallPath = sourceTrashInstallPath; + } + + public String getTargetInstallPath() { + return targetInstallPath; + } + + public void setTargetInstallPath(String targetInstallPath) { + this.targetInstallPath = targetInstallPath; + } + + public String getTargetBackingInstallPath() { + return targetBackingInstallPath; + } + + public void setTargetBackingInstallPath(String targetBackingInstallPath) { + this.targetBackingInstallPath = targetBackingInstallPath; + } + } +} diff --git a/header/src/main/java/org/zstack/header/storage/primary/ConvertVolumeEncryptionOnPrimaryStorageReply.java b/header/src/main/java/org/zstack/header/storage/primary/ConvertVolumeEncryptionOnPrimaryStorageReply.java new file mode 100644 index 00000000000..42c1fcabf2a --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/primary/ConvertVolumeEncryptionOnPrimaryStorageReply.java @@ -0,0 +1,18 @@ +package org.zstack.header.storage.primary; + +import org.zstack.header.message.MessageReply; + +import java.util.HashMap; +import java.util.Map; + +public class ConvertVolumeEncryptionOnPrimaryStorageReply extends MessageReply { + private Map actualSizes = new HashMap<>(); + + public Map getActualSizes() { + return actualSizes; + } + + public void setActualSizes(Map actualSizes) { + this.actualSizes = actualSizes; + } +} diff --git a/header/src/main/java/org/zstack/header/storage/primary/CreateImageCacheFromVolumeSnapshotOnPrimaryStorageMsg.java b/header/src/main/java/org/zstack/header/storage/primary/CreateImageCacheFromVolumeSnapshotOnPrimaryStorageMsg.java index ab65b9063bb..70f88692e57 100644 --- a/header/src/main/java/org/zstack/header/storage/primary/CreateImageCacheFromVolumeSnapshotOnPrimaryStorageMsg.java +++ b/header/src/main/java/org/zstack/header/storage/primary/CreateImageCacheFromVolumeSnapshotOnPrimaryStorageMsg.java @@ -10,6 +10,7 @@ public class CreateImageCacheFromVolumeSnapshotOnPrimaryStorageMsg extends NeedReplyMessage implements PrimaryStorageMessage { private VolumeSnapshotInventory volumeSnapshot; private ImageInventory imageInventory; + private Boolean encrypted; @Override public String getPrimaryStorageUuid() { @@ -32,4 +33,12 @@ public void setImageInventory(ImageInventory imageInventory) { this.imageInventory = imageInventory; } + public Boolean getEncrypted() { + return encrypted; + } + + public void setEncrypted(Boolean encrypted) { + this.encrypted = encrypted; + } + } diff --git a/header/src/main/java/org/zstack/header/storage/primary/CreateVolumeFromVolumeSnapshotOnPrimaryStorageMsg.java b/header/src/main/java/org/zstack/header/storage/primary/CreateVolumeFromVolumeSnapshotOnPrimaryStorageMsg.java index d01daa28aca..1233ac99f47 100755 --- a/header/src/main/java/org/zstack/header/storage/primary/CreateVolumeFromVolumeSnapshotOnPrimaryStorageMsg.java +++ b/header/src/main/java/org/zstack/header/storage/primary/CreateVolumeFromVolumeSnapshotOnPrimaryStorageMsg.java @@ -2,11 +2,13 @@ import org.zstack.header.message.NeedReplyMessage; import org.zstack.header.storage.snapshot.VolumeSnapshotInventory; +import org.zstack.header.volume.VolumeLuksAgentSpec; public class CreateVolumeFromVolumeSnapshotOnPrimaryStorageMsg extends NeedReplyMessage implements PrimaryStorageMessage { private String volumeUuid; private String primaryStorageUuid; private VolumeSnapshotInventory snapshot; + private VolumeLuksAgentSpec volumeLuksAgentSpec; public VolumeSnapshotInventory getSnapshot() { return snapshot; @@ -31,4 +33,12 @@ public String getPrimaryStorageUuid() { public void setPrimaryStorageUuid(String primaryStorageUuid) { this.primaryStorageUuid = primaryStorageUuid; } + + public VolumeLuksAgentSpec getVolumeLuksAgentSpec() { + return volumeLuksAgentSpec; + } + + public void setVolumeLuksAgentSpec(VolumeLuksAgentSpec volumeLuksAgentSpec) { + this.volumeLuksAgentSpec = volumeLuksAgentSpec; + } } diff --git a/header/src/main/java/org/zstack/header/storage/primary/EncryptVolumeBitsOnPrimaryStorageMsg.java b/header/src/main/java/org/zstack/header/storage/primary/EncryptVolumeBitsOnPrimaryStorageMsg.java new file mode 100644 index 00000000000..297165fc17e --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/primary/EncryptVolumeBitsOnPrimaryStorageMsg.java @@ -0,0 +1,61 @@ +package org.zstack.header.storage.primary; + +import org.zstack.header.message.NeedReplyMessage; + +/** + * Triggers an in-place LUKS encryption of an existing volume file on primary storage. + * Used after downloading a data-volume template's plain bits to LocalStorage when the + * volume is marked encrypted: the agent converts the plain qcow2/raw at {@link #installPath} + * into a LUKS-encrypted qcow2 (overwriting in place). + * + * The DEK is staged on the host out-of-band (caller stages the secret material file via + * SecretHostEnsureLuksSecretFileMsg and passes the file path here). + */ +public class EncryptVolumeBitsOnPrimaryStorageMsg extends NeedReplyMessage implements PrimaryStorageMessage { + private String primaryStorageUuid; + private String hostUuid; + private String volumeUuid; + private String installPath; + private String encryptLuksSecretMaterialFilePath; + + @Override + public String getPrimaryStorageUuid() { + return primaryStorageUuid; + } + + public void setPrimaryStorageUuid(String primaryStorageUuid) { + this.primaryStorageUuid = primaryStorageUuid; + } + + public String getHostUuid() { + return hostUuid; + } + + public void setHostUuid(String hostUuid) { + this.hostUuid = hostUuid; + } + + public String getVolumeUuid() { + return volumeUuid; + } + + public void setVolumeUuid(String volumeUuid) { + this.volumeUuid = volumeUuid; + } + + public String getInstallPath() { + return installPath; + } + + public void setInstallPath(String installPath) { + this.installPath = installPath; + } + + public String getEncryptLuksSecretMaterialFilePath() { + return encryptLuksSecretMaterialFilePath; + } + + public void setEncryptLuksSecretMaterialFilePath(String encryptLuksSecretMaterialFilePath) { + this.encryptLuksSecretMaterialFilePath = encryptLuksSecretMaterialFilePath; + } +} diff --git a/header/src/main/java/org/zstack/header/storage/primary/EncryptVolumeBitsOnPrimaryStorageReply.java b/header/src/main/java/org/zstack/header/storage/primary/EncryptVolumeBitsOnPrimaryStorageReply.java new file mode 100644 index 00000000000..2410ced31e6 --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/primary/EncryptVolumeBitsOnPrimaryStorageReply.java @@ -0,0 +1,6 @@ +package org.zstack.header.storage.primary; + +import org.zstack.header.message.MessageReply; + +public class EncryptVolumeBitsOnPrimaryStorageReply extends MessageReply { +} diff --git a/header/src/main/java/org/zstack/header/storage/primary/InstantiateVolumeOnPrimaryStorageMsg.java b/header/src/main/java/org/zstack/header/storage/primary/InstantiateVolumeOnPrimaryStorageMsg.java index ee0d3ae796f..d9306da2355 100755 --- a/header/src/main/java/org/zstack/header/storage/primary/InstantiateVolumeOnPrimaryStorageMsg.java +++ b/header/src/main/java/org/zstack/header/storage/primary/InstantiateVolumeOnPrimaryStorageMsg.java @@ -4,6 +4,7 @@ import org.zstack.header.message.NeedReplyMessage; import org.zstack.header.message.ReplayableMessage; import org.zstack.header.volume.VolumeInventory; +import org.zstack.header.volume.VolumeLuksAgentSpec; public class InstantiateVolumeOnPrimaryStorageMsg extends NeedReplyMessage implements PrimaryStorageMessage, ReplayableMessage { private HostInventory destHost; @@ -11,6 +12,7 @@ public class InstantiateVolumeOnPrimaryStorageMsg extends NeedReplyMessage imple private String primaryStorageUuid; private boolean skipIfExisting; private String allocatedInstallUrl; + private VolumeLuksAgentSpec volumeLuksAgentSpec; public String getAllocatedInstallUrl() { return allocatedInstallUrl; @@ -53,6 +55,14 @@ public void setSkipIfExisting(boolean skipIfExisting) { this.skipIfExisting = skipIfExisting; } + public VolumeLuksAgentSpec getVolumeLuksAgentSpec() { + return volumeLuksAgentSpec; + } + + public void setVolumeLuksAgentSpec(VolumeLuksAgentSpec volumeLuksAgentSpec) { + this.volumeLuksAgentSpec = volumeLuksAgentSpec; + } + @Override public String getResourceUuid() { return volume.getUuid(); diff --git a/header/src/main/java/org/zstack/header/storage/primary/ReInitRootVolumeFromTemplateOnPrimaryStorageMsg.java b/header/src/main/java/org/zstack/header/storage/primary/ReInitRootVolumeFromTemplateOnPrimaryStorageMsg.java index ba07327a0e8..0ac3cdf99f1 100644 --- a/header/src/main/java/org/zstack/header/storage/primary/ReInitRootVolumeFromTemplateOnPrimaryStorageMsg.java +++ b/header/src/main/java/org/zstack/header/storage/primary/ReInitRootVolumeFromTemplateOnPrimaryStorageMsg.java @@ -8,6 +8,7 @@ public class ReInitRootVolumeFromTemplateOnPrimaryStorageMsg extends NeedReplyMe private VolumeInventory volume; private long originSize; private String allocatedInstallUrl; + private String hostUuid; public String getAllocatedInstallUrl() { return allocatedInstallUrl; @@ -37,4 +38,12 @@ public long getOriginSize() { public void setOriginSize(long originSize) { this.originSize = originSize; } + + public String getHostUuid() { + return hostUuid; + } + + public void setHostUuid(String hostUuid) { + this.hostUuid = hostUuid; + } } diff --git a/header/src/main/java/org/zstack/header/storage/snapshot/CreateImageCacheFromVolumeSnapshotMsg.java b/header/src/main/java/org/zstack/header/storage/snapshot/CreateImageCacheFromVolumeSnapshotMsg.java index 75f397b4740..910c67595bd 100644 --- a/header/src/main/java/org/zstack/header/storage/snapshot/CreateImageCacheFromVolumeSnapshotMsg.java +++ b/header/src/main/java/org/zstack/header/storage/snapshot/CreateImageCacheFromVolumeSnapshotMsg.java @@ -9,6 +9,7 @@ public class CreateImageCacheFromVolumeSnapshotMsg extends NeedReplyMessage impl private String imageUuid; private String snapshotUuid; private String volumeUuid; + private Boolean encrypted; /** * @ignore */ @@ -48,4 +49,12 @@ public String getVolumeUuid() { public void setSnapshotUuid(String snapshotUuid) { this.snapshotUuid = snapshotUuid; } + + public Boolean getEncrypted() { + return encrypted; + } + + public void setEncrypted(Boolean encrypted) { + this.encrypted = encrypted; + } } \ No newline at end of file diff --git a/header/src/main/java/org/zstack/header/storage/snapshot/TakeSnapshotsOnKvmJobStruct.java b/header/src/main/java/org/zstack/header/storage/snapshot/TakeSnapshotsOnKvmJobStruct.java index d9338a818b4..3c775d77d7a 100644 --- a/header/src/main/java/org/zstack/header/storage/snapshot/TakeSnapshotsOnKvmJobStruct.java +++ b/header/src/main/java/org/zstack/header/storage/snapshot/TakeSnapshotsOnKvmJobStruct.java @@ -13,6 +13,7 @@ public class TakeSnapshotsOnKvmJobStruct implements Serializable { private String previousInstallPath; private String newVolumeInstallPath; private String snapshotUuid; + private String encryptLuksSecretMaterialFilePath; private boolean memory; private boolean live; private boolean full = false; @@ -73,6 +74,14 @@ public void setSnapshotUuid(String snapshotUuid) { this.snapshotUuid = snapshotUuid; } + public String getEncryptLuksSecretMaterialFilePath() { + return encryptLuksSecretMaterialFilePath; + } + + public void setEncryptLuksSecretMaterialFilePath(String encryptLuksSecretMaterialFilePath) { + this.encryptLuksSecretMaterialFilePath = encryptLuksSecretMaterialFilePath; + } + public boolean isLive() { return live; } diff --git a/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotAO.java b/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotAO.java index 434cf362b81..c81953a69ac 100755 --- a/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotAO.java +++ b/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotAO.java @@ -59,6 +59,9 @@ public class VolumeSnapshotAO extends ResourceVO implements ShadowEntity { @Column private boolean fullSnapshot; + @Column + private boolean encrypted; + @Column private String volumeType; @@ -184,6 +187,14 @@ public void setFullSnapshot(boolean fullSnapshot) { this.fullSnapshot = fullSnapshot; } + public boolean isEncrypted() { + return encrypted; + } + + public void setEncrypted(boolean encrypted) { + this.encrypted = encrypted; + } + public String getPrimaryStorageUuid() { return primaryStorageUuid; } diff --git a/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotAO_.java b/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotAO_.java index 20c2d132348..8398d7f117b 100755 --- a/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotAO_.java +++ b/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotAO_.java @@ -25,6 +25,7 @@ public class VolumeSnapshotAO_ extends ResourceVO_ { public static volatile SingularAttribute fullSnapshot; public static volatile SingularAttribute distance; public static volatile SingularAttribute size; + public static volatile SingularAttribute encrypted; public static volatile SingularAttribute state; public static volatile SingularAttribute status; public static volatile SingularAttribute createDate; diff --git a/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotInventory.java b/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotInventory.java index d37ba70d14e..84bdd13a762 100644 --- a/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotInventory.java +++ b/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotInventory.java @@ -126,6 +126,8 @@ public class VolumeSnapshotInventory { */ private Long size; + private Boolean encrypted; + private int distance; /** * @desc - Enabled: ok for operations @@ -186,6 +188,7 @@ public static VolumeSnapshotInventory valueOf(VolumeSnapshotVO vo) { inv.setLatest(vo.isLatest()); inv.setSize(vo.getSize()); inv.setVolumeType(vo.getVolumeType()); + inv.setEncrypted(vo.isEncrypted()); inv.setTreeUuid(vo.getTreeUuid()); inv.setBackupStorageRefs(VolumeSnapshotBackupStorageRefInventory.valueOf(vo.getBackupStorageRefs())); if (vo.getGroupRef() != null) { @@ -377,6 +380,14 @@ public void setLatest(boolean latest) { this.latest = latest; } + public Boolean getEncrypted() { + return encrypted; + } + + public void setEncrypted(Boolean encrypted) { + this.encrypted = encrypted; + } + public String getGroupUuid() { return groupUuid; } diff --git a/header/src/main/java/org/zstack/header/vm/APICreateVmInstanceFromVolumeMsg.java b/header/src/main/java/org/zstack/header/vm/APICreateVmInstanceFromVolumeMsg.java index ffb586384f5..f40b9a2fb1a 100644 --- a/header/src/main/java/org/zstack/header/vm/APICreateVmInstanceFromVolumeMsg.java +++ b/header/src/main/java/org/zstack/header/vm/APICreateVmInstanceFromVolumeMsg.java @@ -105,6 +105,9 @@ public class APICreateVmInstanceFromVolumeMsg extends APICreateMessage implement @APIParam(required = false, validValues = {"InstantStart", "CreateStopped"}) private String strategy = VmCreationStrategy.InstantStart.toString(); + @APIParam(required = false) + private Boolean encrypted; + public String getName() { return name; } @@ -230,6 +233,16 @@ public void setStrategy(String strategy) { this.strategy = strategy; } + @Override + public Boolean getEncrypted() { + return encrypted; + } + + @Override + public void setEncrypted(Boolean encrypted) { + this.encrypted = encrypted; + } + public String getPlatform() { return platform; } diff --git a/header/src/main/java/org/zstack/header/vm/APICreateVmInstanceFromVolumeSnapshotGroupMsg.java b/header/src/main/java/org/zstack/header/vm/APICreateVmInstanceFromVolumeSnapshotGroupMsg.java index 94d63ca232a..04c4f613995 100644 --- a/header/src/main/java/org/zstack/header/vm/APICreateVmInstanceFromVolumeSnapshotGroupMsg.java +++ b/header/src/main/java/org/zstack/header/vm/APICreateVmInstanceFromVolumeSnapshotGroupMsg.java @@ -112,12 +112,18 @@ public class APICreateVmInstanceFromVolumeSnapshotGroupMsg extends APICreateMess @APIParam(required = false) private Map> dataVolumeSystemTags; + @APIParam(required = false) + private List volumeSnapshotEncryptions; + @APINoSee private String platform; @APIParam(required = false) private Boolean resetTpm; + @APIParam(required = false) + private Boolean encrypted; + public String getName() { return name; } @@ -243,6 +249,16 @@ public void setStrategy(String strategy) { this.strategy = strategy; } + @Override + public Boolean getEncrypted() { + return encrypted; + } + + @Override + public void setEncrypted(Boolean encrypted) { + this.encrypted = encrypted; + } + public String getPrimaryStorageUuidForRootVolume() { return primaryStorageUuidForRootVolume; } @@ -287,6 +303,35 @@ public void setDataVolumeSystemTags(Map> dataVolumeSystemTa this.dataVolumeSystemTags = dataVolumeSystemTags; } + public List getVolumeSnapshotEncryptions() { + return volumeSnapshotEncryptions; + } + + public void setVolumeSnapshotEncryptions(List volumeSnapshotEncryptions) { + this.volumeSnapshotEncryptions = volumeSnapshotEncryptions; + } + + public static class VolumeSnapshotEncryption { + private String volumeSnapshotUuid; + private Boolean encrypted; + + public String getVolumeSnapshotUuid() { + return volumeSnapshotUuid; + } + + public void setVolumeSnapshotUuid(String volumeSnapshotUuid) { + this.volumeSnapshotUuid = volumeSnapshotUuid; + } + + public Boolean getEncrypted() { + return encrypted; + } + + public void setEncrypted(Boolean encrypted) { + this.encrypted = encrypted; + } + } + @Override public String getPlatform() { return platform; diff --git a/header/src/main/java/org/zstack/header/vm/APICreateVmInstanceFromVolumeSnapshotMsg.java b/header/src/main/java/org/zstack/header/vm/APICreateVmInstanceFromVolumeSnapshotMsg.java index 8302f6046fc..77f27b8a8cd 100644 --- a/header/src/main/java/org/zstack/header/vm/APICreateVmInstanceFromVolumeSnapshotMsg.java +++ b/header/src/main/java/org/zstack/header/vm/APICreateVmInstanceFromVolumeSnapshotMsg.java @@ -108,6 +108,9 @@ public class APICreateVmInstanceFromVolumeSnapshotMsg extends APICreateMessage i @APIParam(required = false) private List rootVolumeSystemTags; + @APIParam(required = false) + private Boolean encrypted; + public String getName() { return name; } @@ -233,6 +236,16 @@ public void setStrategy(String strategy) { this.strategy = strategy; } + @Override + public Boolean getEncrypted() { + return encrypted; + } + + @Override + public void setEncrypted(Boolean encrypted) { + this.encrypted = encrypted; + } + public String getPlatform() { return platform; } diff --git a/header/src/main/java/org/zstack/header/vm/APICreateVmInstanceMsg.java b/header/src/main/java/org/zstack/header/vm/APICreateVmInstanceMsg.java index e085a0a0239..adf0b0a5291 100755 --- a/header/src/main/java/org/zstack/header/vm/APICreateVmInstanceMsg.java +++ b/header/src/main/java/org/zstack/header/vm/APICreateVmInstanceMsg.java @@ -222,6 +222,9 @@ public class APICreateVmInstanceMsg extends APICreateMessage implements APIAudit @APIParam(required = false) private Boolean virtio; + @APIParam(required = false) + private Boolean encrypted; + @APIParam(required = false) private String allocatorStrategy; @@ -245,6 +248,16 @@ public void setDiskAOs(List diskAOs) { this.diskAOs = diskAOs; } + @Override + public Boolean getEncrypted() { + return encrypted; + } + + @Override + public void setEncrypted(Boolean encrypted) { + this.encrypted = encrypted; + } + public String getStrategy() { return strategy; } diff --git a/header/src/main/java/org/zstack/header/vm/CreateVmInstanceMsg.java b/header/src/main/java/org/zstack/header/vm/CreateVmInstanceMsg.java index 5b6f04dac3a..806868e26c8 100755 --- a/header/src/main/java/org/zstack/header/vm/CreateVmInstanceMsg.java +++ b/header/src/main/java/org/zstack/header/vm/CreateVmInstanceMsg.java @@ -36,6 +36,7 @@ public class CreateVmInstanceMsg extends NeedReplyMessage implements CreateVmIns private String guestOsType; private String architecture; private Boolean virtio; + private Boolean encrypted; private List rootVolumeSystemTags; private List dataVolumeSystemTags; private List disableL3Networks; @@ -206,6 +207,14 @@ public void setImageUuid(String imageUuid) { this.imageUuid = imageUuid; } + public Boolean getEncrypted() { + return encrypted; + } + + public void setEncrypted(Boolean encrypted) { + this.encrypted = encrypted; + } + @Override public int getCpuNum() { return cpuNum; diff --git a/header/src/main/java/org/zstack/header/vm/DiskAO.java b/header/src/main/java/org/zstack/header/vm/DiskAO.java index 3677ff749f3..5762969adfa 100644 --- a/header/src/main/java/org/zstack/header/vm/DiskAO.java +++ b/header/src/main/java/org/zstack/header/vm/DiskAO.java @@ -25,6 +25,7 @@ public class DiskAO { private String sourceUuid; private List systemTags; private String name; + private Boolean encrypted; public DiskAO withImage(String imageUuid) { this.templateUuid = imageUuid; @@ -139,6 +140,14 @@ public void setName(String name) { this.name = name; } + public Boolean getEncrypted() { + return encrypted; + } + + public void setEncrypted(Boolean encrypted) { + this.encrypted = encrypted; + } + public static DiskAO rootDisk() { DiskAO disk = new DiskAO(); disk.setBoot(true); diff --git a/header/src/main/java/org/zstack/header/vm/NewVmInstanceMessage2.java b/header/src/main/java/org/zstack/header/vm/NewVmInstanceMessage2.java index c86c80e7931..1ff82847ff8 100644 --- a/header/src/main/java/org/zstack/header/vm/NewVmInstanceMessage2.java +++ b/header/src/main/java/org/zstack/header/vm/NewVmInstanceMessage2.java @@ -12,6 +12,9 @@ public interface NewVmInstanceMessage2 extends NewVmInstanceMessage { String getHostUuid(); String getStrategy(); String getInstanceOfferingUuid(); + default Boolean getEncrypted() { + return null; + } void setCpuNum(Integer cpuNum); void setMemorySize(Long memorySize); @@ -20,4 +23,6 @@ public interface NewVmInstanceMessage2 extends NewVmInstanceMessage { void setClusterUuid(String clusterUuid); void setDefaultL3NetworkUuid(String defaultL3NetworkUuid); void setType(String type); + default void setEncrypted(Boolean encrypted) { + } } diff --git a/header/src/main/java/org/zstack/header/vm/VmInstanceAO.java b/header/src/main/java/org/zstack/header/vm/VmInstanceAO.java index 001dbe2298b..707f03f3c04 100755 --- a/header/src/main/java/org/zstack/header/vm/VmInstanceAO.java +++ b/header/src/main/java/org/zstack/header/vm/VmInstanceAO.java @@ -89,6 +89,9 @@ public class VmInstanceAO extends ResourceVO { @Column private String guestOsType; + @Column + private boolean encrypted; + @Column private Timestamp createDate; @@ -128,6 +131,7 @@ public VmInstanceAO(VmInstanceAO other) { this.platform = other.platform; this.guestOsType = other.guestOsType; this.architecture = other.architecture; + this.encrypted = other.encrypted; } @PreUpdate @@ -330,4 +334,12 @@ public String getGuestOsType() { public void setGuestOsType(String guestOsType) { this.guestOsType = guestOsType; } + + public boolean isEncrypted() { + return encrypted; + } + + public void setEncrypted(boolean encrypted) { + this.encrypted = encrypted; + } } diff --git a/header/src/main/java/org/zstack/header/vm/VmInstanceAO_.java b/header/src/main/java/org/zstack/header/vm/VmInstanceAO_.java index 90dee2c2ef6..c2dfbc2887d 100755 --- a/header/src/main/java/org/zstack/header/vm/VmInstanceAO_.java +++ b/header/src/main/java/org/zstack/header/vm/VmInstanceAO_.java @@ -31,5 +31,6 @@ public class VmInstanceAO_ extends ResourceVO_ { public static volatile SingularAttribute cpuSpeed; public static volatile SingularAttribute createDate; public static volatile SingularAttribute lastOpDate; + public static volatile SingularAttribute encrypted; public static volatile SingularAttribute state; } diff --git a/header/src/main/java/org/zstack/header/vm/VmInstanceInventory.java b/header/src/main/java/org/zstack/header/vm/VmInstanceInventory.java index 706f8c82482..750395f2dc8 100755 --- a/header/src/main/java/org/zstack/header/vm/VmInstanceInventory.java +++ b/header/src/main/java/org/zstack/header/vm/VmInstanceInventory.java @@ -231,6 +231,19 @@ public class VmInstanceInventory implements Serializable, Cloneable { * @desc last time this resource gets operated */ private Timestamp lastOpDate; + /** + * @desc whether the vm itself is marked encrypted + */ + private Boolean encrypted; + /** + * @desc whether the vm itself, any attached volume, or its TPM has encrypted resource + */ + private Boolean hasEncryptionResource; + /** + * @desc latest key update time of attached encrypted volumes or TPM; falls back to vm create date when only vm itself is encrypted + * @nullable + */ + private Timestamp keyLastOpDate; /** * @desc - Created: the vm is just created in database, having not been started * - Starting: the vm is starting, having not run on host @@ -300,6 +313,7 @@ protected VmInstanceInventory(VmInstanceVO vo) { this.setPlatform(vo.getPlatform()); this.setArchitecture(vo.getArchitecture()); this.setGuestOsType(vo.getGuestOsType()); + this.setEncrypted(vo.isEncrypted()); } public static VmInstanceInventory valueOf(VmInstanceVO vo) { @@ -378,6 +392,9 @@ public VmInstanceInventory(VmInstanceInventory origin) { this.setAllocatorStrategy(inv.getAllocatorStrategy()); this.setArchitecture(inv.getArchitecture()); this.setGuestOsType(inv.getGuestOsType()); + this.setEncrypted(inv.getEncrypted()); + this.setHasEncryptionResource(inv.getHasEncryptionResource()); + this.setKeyLastOpDate(inv.getKeyLastOpDate()); } public VolumeInventory getRootVolume() { @@ -592,6 +609,30 @@ public void setLastOpDate(Timestamp lastOpDate) { this.lastOpDate = lastOpDate; } + public Boolean getEncrypted() { + return encrypted; + } + + public void setEncrypted(Boolean encrypted) { + this.encrypted = encrypted; + } + + public Boolean getHasEncryptionResource() { + return hasEncryptionResource; + } + + public void setHasEncryptionResource(Boolean hasEncryptionResource) { + this.hasEncryptionResource = hasEncryptionResource; + } + + public Timestamp getKeyLastOpDate() { + return keyLastOpDate; + } + + public void setKeyLastOpDate(Timestamp keyLastOpDate) { + this.keyLastOpDate = keyLastOpDate; + } + public List getVmCdRoms() { return vmCdRoms; } diff --git a/header/src/main/java/org/zstack/header/volume/APIChangeVolumeEncryptionEvent.java b/header/src/main/java/org/zstack/header/volume/APIChangeVolumeEncryptionEvent.java new file mode 100644 index 00000000000..0bd654d77ab --- /dev/null +++ b/header/src/main/java/org/zstack/header/volume/APIChangeVolumeEncryptionEvent.java @@ -0,0 +1,54 @@ +package org.zstack.header.volume; + +import org.zstack.header.message.APIEvent; +import org.zstack.header.rest.RestResponse; +import org.zstack.utils.data.SizeUnit; + +import java.sql.Timestamp; + +@RestResponse(allTo = "inventory") +public class APIChangeVolumeEncryptionEvent extends APIEvent { + private VolumeInventory inventory; + + public APIChangeVolumeEncryptionEvent() { + } + + public APIChangeVolumeEncryptionEvent(String apiId) { + super(apiId); + } + + public VolumeInventory getInventory() { + return inventory; + } + + public void setInventory(VolumeInventory inventory) { + this.inventory = inventory; + } + + public static APIChangeVolumeEncryptionEvent __example__() { + APIChangeVolumeEncryptionEvent event = new APIChangeVolumeEncryptionEvent(); + + String volumeUuid = uuid(); + VolumeInventory vol = new VolumeInventory(); + vol.setName("test-volume"); + vol.setCreateDate(new Timestamp(org.zstack.header.message.DocUtils.date)); + vol.setLastOpDate(new Timestamp(org.zstack.header.message.DocUtils.date)); + vol.setType(VolumeType.Root.toString()); + vol.setUuid(volumeUuid); + vol.setSize(SizeUnit.GIGABYTE.toByte(100)); + vol.setActualSize(SizeUnit.GIGABYTE.toByte(20)); + vol.setDeviceId(0); + vol.setState(VolumeState.Enabled.toString()); + vol.setFormat("qcow2"); + vol.setDiskOfferingUuid(uuid()); + vol.setInstallPath(String.format("/zstack_ps/rootVolumes/acct-36c27e8ff05c4780bf6d2fa65700f22e/vol-%s/%s.qcow2", volumeUuid, volumeUuid)); + vol.setStatus(VolumeStatus.Ready.toString()); + vol.setPrimaryStorageUuid(uuid()); + vol.setVmInstanceUuid(uuid()); + vol.setRootImageUuid(uuid()); + vol.setEncrypted(true); + + event.setInventory(vol); + return event; + } +} diff --git a/header/src/main/java/org/zstack/header/volume/APIChangeVolumeEncryptionMsg.java b/header/src/main/java/org/zstack/header/volume/APIChangeVolumeEncryptionMsg.java new file mode 100644 index 00000000000..157609c2d8e --- /dev/null +++ b/header/src/main/java/org/zstack/header/volume/APIChangeVolumeEncryptionMsg.java @@ -0,0 +1,62 @@ +package org.zstack.header.volume; + +import org.springframework.http.HttpMethod; +import org.zstack.header.message.APIEvent; +import org.zstack.header.message.APIMessage; +import org.zstack.header.message.APIParam; +import org.zstack.header.message.DefaultTimeout; +import org.zstack.header.other.APIAuditor; +import org.zstack.header.rest.RestRequest; +import org.zstack.header.vm.metadata.MetadataImpact; + +import java.util.concurrent.TimeUnit; + +@DefaultTimeout(timeunit = TimeUnit.HOURS, value = 36) +@RestRequest( + path = "/volumes/{uuid}/actions", + isAction = true, + method = HttpMethod.PUT, + responseClass = APIChangeVolumeEncryptionEvent.class +) +@MetadataImpact(value = MetadataImpact.Impact.STORAGE, resolver = "VolumeUuidToVmUuidResolver", field = "uuid", updateOnFailure = true) +public class APIChangeVolumeEncryptionMsg extends APIMessage implements VolumeMessage, APIAuditor { + @APIParam(resourceType = VolumeVO.class) + private String uuid; + + @APIParam + private boolean encrypted; + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public boolean isEncrypted() { + return encrypted; + } + + public void setEncrypted(boolean encrypted) { + this.encrypted = encrypted; + } + + @Override + public String getVolumeUuid() { + return uuid; + } + + @Override + public APIAuditor.Result audit(APIMessage msg, APIEvent rsp) { + APIChangeVolumeEncryptionMsg amsg = (APIChangeVolumeEncryptionMsg) msg; + return new APIAuditor.Result(amsg.getUuid(), VolumeVO.class); + } + + public static APIChangeVolumeEncryptionMsg __example__() { + APIChangeVolumeEncryptionMsg msg = new APIChangeVolumeEncryptionMsg(); + msg.setUuid(uuid()); + msg.setEncrypted(true); + return msg; + } +} diff --git a/header/src/main/java/org/zstack/header/volume/ChangeVolumeEncryptionMsg.java b/header/src/main/java/org/zstack/header/volume/ChangeVolumeEncryptionMsg.java new file mode 100644 index 00000000000..c1fa1daa5d1 --- /dev/null +++ b/header/src/main/java/org/zstack/header/volume/ChangeVolumeEncryptionMsg.java @@ -0,0 +1,30 @@ +package org.zstack.header.volume; + +import org.zstack.header.message.ConfigurableTimeoutMessage; +import org.zstack.header.message.DefaultTimeout; +import org.zstack.header.message.NeedReplyMessage; + +import java.util.concurrent.TimeUnit; + +@DefaultTimeout(timeunit = TimeUnit.HOURS, value = 36) +public class ChangeVolumeEncryptionMsg extends NeedReplyMessage implements VolumeMessage, ConfigurableTimeoutMessage { + private String volumeUuid; + private boolean encrypted; + + @Override + public String getVolumeUuid() { + return volumeUuid; + } + + public void setVolumeUuid(String volumeUuid) { + this.volumeUuid = volumeUuid; + } + + public boolean isEncrypted() { + return encrypted; + } + + public void setEncrypted(boolean encrypted) { + this.encrypted = encrypted; + } +} diff --git a/header/src/main/java/org/zstack/header/volume/ChangeVolumeEncryptionReply.java b/header/src/main/java/org/zstack/header/volume/ChangeVolumeEncryptionReply.java new file mode 100644 index 00000000000..4570a899892 --- /dev/null +++ b/header/src/main/java/org/zstack/header/volume/ChangeVolumeEncryptionReply.java @@ -0,0 +1,15 @@ +package org.zstack.header.volume; + +import org.zstack.header.message.MessageReply; + +public class ChangeVolumeEncryptionReply extends MessageReply { + private VolumeInventory inventory; + + public VolumeInventory getInventory() { + return inventory; + } + + public void setInventory(VolumeInventory inventory) { + this.inventory = inventory; + } +} diff --git a/header/src/main/java/org/zstack/header/volume/CreateDataVolumeExtensionPoint.java b/header/src/main/java/org/zstack/header/volume/CreateDataVolumeExtensionPoint.java index 6ce8ed6ab7f..d75c693806f 100644 --- a/header/src/main/java/org/zstack/header/volume/CreateDataVolumeExtensionPoint.java +++ b/header/src/main/java/org/zstack/header/volume/CreateDataVolumeExtensionPoint.java @@ -9,4 +9,8 @@ public interface CreateDataVolumeExtensionPoint { void beforeCreateVolume(VolumeInventory volume); void afterCreateVolume(VolumeVO volume); + + default void afterCreateVolume(VolumeVO volume, String snapshotUuid) { + afterCreateVolume(volume); + } } diff --git a/header/src/main/java/org/zstack/header/volume/CreateDataVolumeFromVolumeSnapshotMsg.java b/header/src/main/java/org/zstack/header/volume/CreateDataVolumeFromVolumeSnapshotMsg.java index cd709d306c8..ee4e82765cb 100644 --- a/header/src/main/java/org/zstack/header/volume/CreateDataVolumeFromVolumeSnapshotMsg.java +++ b/header/src/main/java/org/zstack/header/volume/CreateDataVolumeFromVolumeSnapshotMsg.java @@ -9,6 +9,7 @@ public class CreateDataVolumeFromVolumeSnapshotMsg extends NeedReplyMessage { private String volumeSnapshotUuid; private SessionInventory session; private Long size; + private Boolean encrypted; public String getName() { return name; @@ -49,4 +50,12 @@ public Long getSize() { public void setSize(Long size) { this.size = size; } + + public Boolean getEncrypted() { + return encrypted; + } + + public void setEncrypted(Boolean encrypted) { + this.encrypted = encrypted; + } } diff --git a/header/src/main/java/org/zstack/header/volume/CreateDataVolumeFromVolumeTemplateMsg.java b/header/src/main/java/org/zstack/header/volume/CreateDataVolumeFromVolumeTemplateMsg.java index 781c33bc9ba..06195687e39 100644 --- a/header/src/main/java/org/zstack/header/volume/CreateDataVolumeFromVolumeTemplateMsg.java +++ b/header/src/main/java/org/zstack/header/volume/CreateDataVolumeFromVolumeTemplateMsg.java @@ -18,6 +18,7 @@ public class CreateDataVolumeFromVolumeTemplateMsg extends NeedReplyMessage impl private String hostUuid; private String resourceUuid; private String accountUuid; + private Boolean encrypted; private APICreateDataVolumeFromVolumeTemplateMsg apiMsg; public CreateDataVolumeFromVolumeTemplateMsg() { @@ -97,4 +98,12 @@ public APICreateDataVolumeFromVolumeTemplateMsg getApiMsg() { public void setApiMsg(APICreateDataVolumeFromVolumeTemplateMsg amsg) { this.apiMsg = amsg; } + + public Boolean getEncrypted() { + return encrypted; + } + + public void setEncrypted(Boolean encrypted) { + this.encrypted = encrypted; + } } diff --git a/header/src/main/java/org/zstack/header/volume/CreateVolumeMsg.java b/header/src/main/java/org/zstack/header/volume/CreateVolumeMsg.java index e8d22cb9a99..e684bfabdfd 100755 --- a/header/src/main/java/org/zstack/header/volume/CreateVolumeMsg.java +++ b/header/src/main/java/org/zstack/header/volume/CreateVolumeMsg.java @@ -16,6 +16,7 @@ public class CreateVolumeMsg extends NeedReplyMessage implements VolumeCreateMes private String format; private String resourceUuid; private String protocol; + private Boolean encrypted; public String getFormat() { return format; @@ -130,4 +131,14 @@ public String getProtocol() { public void setProtocol(String protocol) { this.protocol = protocol; } + + @Override + public Boolean getEncrypted() { + return encrypted; + } + + @Override + public void setEncrypted(Boolean encrypted) { + this.encrypted = encrypted; + } } diff --git a/header/src/main/java/org/zstack/header/volume/EncryptVolumeMsg.java b/header/src/main/java/org/zstack/header/volume/EncryptVolumeMsg.java new file mode 100644 index 00000000000..e8ff1e85e34 --- /dev/null +++ b/header/src/main/java/org/zstack/header/volume/EncryptVolumeMsg.java @@ -0,0 +1,86 @@ +package org.zstack.header.volume; + +import org.zstack.header.message.ConfigurableTimeoutMessage; +import org.zstack.header.message.DefaultTimeout; +import org.zstack.header.message.NeedReplyMessage; + +import java.util.concurrent.TimeUnit; + +/** + * Convert an existing data volume's bits to a LUKS-encrypted form in place. + *

+ * Steps performed by the handler (in {@link org.zstack.storage.volume.VolumeBase}): + *

    + *
  1. Ensure the volume has a key-provider binding (auto-attaches the default key provider + * when none is bound yet).
    2. + *
  2. Materialize a DEK via the {@code EncryptedResourceKeyManager}.
    3. + *
  3. Stage the LUKS secret material file on the host + * ({@code SecretHostEnsureLuksSecretFileMsg}).
    4. + *
  4. Ask the primary storage backend to LUKS-convert the bits in place + * ({@code EncryptVolumeBitsOnPrimaryStorageMsg}).
    5. + *
  5. Persist {@code VolumeVO.encrypted = true}.
    6. + *
+ *

+ * If the volume row is already marked {@code encrypted=true}, the handler treats it as a + * no-op success. The {@code encrypted} flag is the single authoritative signal that + * "the bits on disk are already LUKS"; callers must NOT pre-mark the row before invoking + * this message. + */ +@DefaultTimeout(timeunit = TimeUnit.HOURS, value = 1) +public class EncryptVolumeMsg extends NeedReplyMessage implements VolumeMessage, ConfigurableTimeoutMessage { + private String volumeUuid; + private String hostUuid; + /** + * Optional. When null, the handler resolves it from {@code VolumeVO.primaryStorageUuid}. + */ + private String primaryStorageUuid; + /** + * Optional. When null, the handler resolves it from {@code VolumeVO.installPath}. + */ + private String installPath; + /** + * Free-form purpose label for the DEK get-or-create audit trail. + */ + private String purpose; + + @Override + public String getVolumeUuid() { + return volumeUuid; + } + + public void setVolumeUuid(String volumeUuid) { + this.volumeUuid = volumeUuid; + } + + public String getHostUuid() { + return hostUuid; + } + + public void setHostUuid(String hostUuid) { + this.hostUuid = hostUuid; + } + + public String getPrimaryStorageUuid() { + return primaryStorageUuid; + } + + public void setPrimaryStorageUuid(String primaryStorageUuid) { + this.primaryStorageUuid = primaryStorageUuid; + } + + public String getInstallPath() { + return installPath; + } + + public void setInstallPath(String installPath) { + this.installPath = installPath; + } + + public String getPurpose() { + return purpose; + } + + public void setPurpose(String purpose) { + this.purpose = purpose; + } +} diff --git a/header/src/main/java/org/zstack/header/volume/EncryptVolumeReply.java b/header/src/main/java/org/zstack/header/volume/EncryptVolumeReply.java new file mode 100644 index 00000000000..f4f21f1b901 --- /dev/null +++ b/header/src/main/java/org/zstack/header/volume/EncryptVolumeReply.java @@ -0,0 +1,18 @@ +package org.zstack.header.volume; + +import org.zstack.header.message.MessageReply; + +/** + * Reply for {@link EncryptVolumeMsg}. + */ +public class EncryptVolumeReply extends MessageReply { + private VolumeInventory inventory; + + public VolumeInventory getInventory() { + return inventory; + } + + public void setInventory(VolumeInventory inventory) { + this.inventory = inventory; + } +} diff --git a/header/src/main/java/org/zstack/header/volume/InstantiateVolumeMsg.java b/header/src/main/java/org/zstack/header/volume/InstantiateVolumeMsg.java index 13e6f557b24..e454fecdd92 100755 --- a/header/src/main/java/org/zstack/header/volume/InstantiateVolumeMsg.java +++ b/header/src/main/java/org/zstack/header/volume/InstantiateVolumeMsg.java @@ -12,6 +12,7 @@ public class InstantiateVolumeMsg extends NeedReplyMessage implements VolumeMess private boolean primaryStorageAllocated; private boolean skipIfExisting; private String allocatedInstallUrl; + private VolumeLuksAgentSpec volumeLuksAgentSpec; public String getAllocatedInstallUrl() { return allocatedInstallUrl; @@ -61,4 +62,12 @@ public boolean isSkipIfExisting() { public void setSkipIfExisting(boolean skipIfExisting) { this.skipIfExisting = skipIfExisting; } + + public VolumeLuksAgentSpec getVolumeLuksAgentSpec() { + return volumeLuksAgentSpec; + } + + public void setVolumeLuksAgentSpec(VolumeLuksAgentSpec volumeLuksAgentSpec) { + this.volumeLuksAgentSpec = volumeLuksAgentSpec; + } } diff --git a/header/src/main/java/org/zstack/header/volume/VolumeAO.java b/header/src/main/java/org/zstack/header/volume/VolumeAO.java index 3ddcdbccad3..203b30134a5 100755 --- a/header/src/main/java/org/zstack/header/volume/VolumeAO.java +++ b/header/src/main/java/org/zstack/header/volume/VolumeAO.java @@ -86,6 +86,9 @@ public class VolumeAO extends ResourceVO implements ShadowEntity { @Column private String protocol; + @Column + private boolean encrypted; + @Transient private VolumeAO shadow; @@ -298,4 +301,12 @@ public String getProtocol() { public void setProtocol(String protocol) { this.protocol = protocol; } + + public boolean isEncrypted() { + return encrypted; + } + + public void setEncrypted(boolean encrypted) { + this.encrypted = encrypted; + } } diff --git a/header/src/main/java/org/zstack/header/volume/VolumeAO_.java b/header/src/main/java/org/zstack/header/volume/VolumeAO_.java index 729d50eefbf..af03c918ca2 100755 --- a/header/src/main/java/org/zstack/header/volume/VolumeAO_.java +++ b/header/src/main/java/org/zstack/header/volume/VolumeAO_.java @@ -31,4 +31,5 @@ public class VolumeAO_ extends ResourceVO_ { public static volatile SingularAttribute isShareable; public static volatile SingularAttribute volumeQos; public static volatile SingularAttribute protocol; + public static volatile SingularAttribute encrypted; } diff --git a/header/src/main/java/org/zstack/header/volume/VolumeCreateMessage.java b/header/src/main/java/org/zstack/header/volume/VolumeCreateMessage.java index a93217a6166..36f8b47f870 100644 --- a/header/src/main/java/org/zstack/header/volume/VolumeCreateMessage.java +++ b/header/src/main/java/org/zstack/header/volume/VolumeCreateMessage.java @@ -20,4 +20,11 @@ public interface VolumeCreateMessage { void setSystemTags(List systemTags); void addSystemTag(String tag); + + default Boolean getEncrypted() { + return null; + } + + default void setEncrypted(Boolean encrypted) { + } } diff --git a/header/src/main/java/org/zstack/header/volume/VolumeInventory.java b/header/src/main/java/org/zstack/header/volume/VolumeInventory.java index 96d2ae67a62..f04fe219fed 100755 --- a/header/src/main/java/org/zstack/header/volume/VolumeInventory.java +++ b/header/src/main/java/org/zstack/header/volume/VolumeInventory.java @@ -156,6 +156,8 @@ public class VolumeInventory implements Serializable { private Timestamp lastAttachDate; private String protocol; + private Boolean encrypted; + public VolumeInventory() { } @@ -183,6 +185,7 @@ public VolumeInventory(VolumeInventory other) { this.lastVmInstanceUuid = other.lastVmInstanceUuid; this.lastAttachDate = other.lastAttachDate; this.protocol = other.protocol; + this.encrypted = other.encrypted; } @@ -213,6 +216,7 @@ public VolumeInventory(VolumeInventory other) { inv.setLastVmInstanceUuid(vo.getLastVmInstanceUuid()); inv.setLastAttachDate(vo.getLastAttachDate()); inv.setProtocol(vo.getProtocol()); + inv.setEncrypted(vo.isEncrypted()); return inv; } @@ -437,4 +441,12 @@ public String getProtocol() { public void setProtocol(String protocol) { this.protocol = protocol; } + + public Boolean getEncrypted() { + return encrypted; + } + + public void setEncrypted(Boolean encrypted) { + this.encrypted = encrypted; + } } diff --git a/header/src/main/java/org/zstack/header/volume/VolumeLuksAgentSpec.java b/header/src/main/java/org/zstack/header/volume/VolumeLuksAgentSpec.java new file mode 100644 index 00000000000..8602adcd79d --- /dev/null +++ b/header/src/main/java/org/zstack/header/volume/VolumeLuksAgentSpec.java @@ -0,0 +1,31 @@ +package org.zstack.header.volume; + +import java.io.Serializable; + +public class VolumeLuksAgentSpec implements Serializable { + private static final long serialVersionUID = 1L; + + private String encryptSecretUuid; + private String encryptLuksSecretMaterialFilePath; + + public boolean isComplete() { + return (encryptSecretUuid != null && !encryptSecretUuid.isEmpty()) + || (encryptLuksSecretMaterialFilePath != null && !encryptLuksSecretMaterialFilePath.isEmpty()); + } + + public String getEncryptSecretUuid() { + return encryptSecretUuid; + } + + public void setEncryptSecretUuid(String encryptSecretUuid) { + this.encryptSecretUuid = encryptSecretUuid; + } + + public String getEncryptLuksSecretMaterialFilePath() { + return encryptLuksSecretMaterialFilePath; + } + + public void setEncryptLuksSecretMaterialFilePath(String encryptLuksSecretMaterialFilePath) { + this.encryptLuksSecretMaterialFilePath = encryptLuksSecretMaterialFilePath; + } +} diff --git a/image/src/main/java/org/zstack/image/ImageManagerImpl.java b/image/src/main/java/org/zstack/image/ImageManagerImpl.java index 5c32b9f3315..27638a49470 100755 --- a/image/src/main/java/org/zstack/image/ImageManagerImpl.java +++ b/image/src/main/java/org/zstack/image/ImageManagerImpl.java @@ -242,6 +242,7 @@ public void run(FlowTrigger trigger, Map data) { cmsg.setImageUuid(vo.getUuid()); cmsg.setVolumeUuid(volumeUuid); cmsg.setTreeUuid(treeUuid); + cmsg.setEncrypted(msg.getEncrypted()); cmsg.setSystemTags(msg.getSystemTags()); String resourceUuid = volumeUuid != null ? volumeUuid : treeUuid; bus.makeTargetServiceIdByResourceUuid(cmsg, VolumeSnapshotConstant.SERVICE_ID, resourceUuid); diff --git a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java index a04a9aba6e4..41caf38e4d0 100755 --- a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java +++ b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java @@ -38,6 +38,7 @@ import org.zstack.header.exception.CloudRuntimeException; import org.zstack.header.host.*; import org.zstack.header.image.ImageBackupStorageRefInventory; +import org.zstack.header.image.ImageConstant; import org.zstack.header.image.ImageConstant.ImageMediaType; import org.zstack.header.image.ImageInventory; import org.zstack.header.image.ImageStatus; @@ -68,6 +69,8 @@ import org.zstack.storage.ceph.backup.CephBackupStorageVO_; import org.zstack.storage.ceph.primary.CephPrimaryStorageMonBase.PingOperationFailure; import org.zstack.storage.ceph.primary.capacity.CephOsdGroupCapacityHelper; +import org.zstack.storage.encrypt.VolumeEncryptedSecretHelper; +import org.zstack.storage.encrypt.VolumeSnapshotEncryptionHelper; import org.zstack.storage.primary.*; import org.zstack.storage.volume.VolumeErrors; import org.zstack.storage.volume.VolumeSystemTags; @@ -115,6 +118,10 @@ public class CephPrimaryStorageBase extends PrimaryStorageBase { private StorageTrash trash; @Autowired private PoolUsageReport poolUsageCollector; + @Autowired + private VolumeSnapshotEncryptionHelper snapshotEncryptionHelper; + @Autowired + private VolumeEncryptedSecretHelper volumeEncryptedSecretHelper; public CephPrimaryStorageBase() { @@ -420,6 +427,38 @@ public void setInstallPath(String installPath) { } } + public static class KVMHostLuksCloneCmd implements Serializable { + public String psUuid; + public String secFilePath; + public String srcPath; + public String dstPath; + public Long virtualSizeForLuksClone; + } + + public static class KVMHostLuksCreateEmptyCmd implements Serializable { + public String psUuid; + public String secFilePath; + public String installPath; + public long size; + } + + public static class KVMHostLuksResizeCmd implements Serializable { + public String psUuid; + public String secFilePath; + public String installPath; + public Long virtualSize; + } + + public static class KVMHostEncryptInPlaceCmd implements Serializable { + public String psUuid; + public String secFilePath; + public String installPath; + } + + public static class KVMHostLuksRsp extends KVMAgentCommands.AgentResponse { + public Long actualSize; + } + public static class DeleteCmd extends AgentCommand { String installPath; long expirationTime; @@ -1358,6 +1397,13 @@ public int compareTo(SnapInfo snapInfo) { public static final String CREATE_VOLUME_PATH = "/ceph/primarystorage/volume/createempty"; public static final String DELETE_PATH = "/ceph/primarystorage/delete"; public static final String CLONE_PATH = "/ceph/primarystorage/volume/clone"; + // LUKS variants that run on the KVM host instead of cephagent (the host that + // qemu-img must run on is the one that has key-agent + the libvirt secret + // FIFO; cephagent nodes do not, especially in non-converged deployments). + public static final String KVM_HOST_LUKS_CLONE_PATH = "/ceph/primarystorage/kvmhost/luksclone"; + public static final String KVM_HOST_LUKS_CREATE_EMPTY_PATH = "/ceph/primarystorage/kvmhost/lukscreateempty"; + public static final String KVM_HOST_LUKS_ENCRYPT_IN_PLACE_PATH = "/ceph/primarystorage/kvmhost/encryptinplace"; + public static final String KVM_HOST_LUKS_RESIZE_PATH = "/ceph/primarystorage/kvmhost/luksresize"; public static final String FLATTEN_PATH = "/ceph/primarystorage/volume/flatten"; public static final String SFTP_DOWNLOAD_PATH = "/ceph/primarystorage/sftpbackupstorage/download"; public static final String SFTP_UPLOAD_PATH = "/ceph/primarystorage/sftpbackupstorage/upload"; @@ -1722,11 +1768,55 @@ protected CephPrimaryStorageInventory getSelfInventory() { } private void createEmptyVolume(final InstantiateVolumeOnPrimaryStorageMsg msg) { - final CreateEmptyVolumeCmd cmd = new CreateEmptyVolumeCmd(); String volumeUuid = msg.getVolume().getUuid(); final String finalPoolName = getTargetPoolNameFromAllocatedUrl(msg.getAllocatedInstallUrl()); - cmd.installPath = makeVolumeInstallPathByTargetPool(volumeUuid, finalPoolName); - cmd.size = msg.getVolume().getSize(); + final String installPath = makeVolumeInstallPathByTargetPool(volumeUuid, finalPoolName); + final long volumeSize = msg.getVolume().getSize(); + + final InstantiateVolumeOnPrimaryStorageReply reply = new InstantiateVolumeOnPrimaryStorageReply(); + + // Encrypted variant: forward to the dest KVM host so qemu-img runs next + // to the libvirt-secret FIFO + key-agent (see KVM_HOST_LUKS_CREATE_EMPTY_PATH). + if (msg.getVolumeLuksAgentSpec() != null && msg.getVolumeLuksAgentSpec().isComplete()) { + if (msg.getDestHost() == null || StringUtils.isBlank(msg.getDestHost().getUuid())) { + reply.setError(operr( + "ceph LUKS createempty requires destHost; volume[uuid:%s] has none", volumeUuid)); + bus.reply(msg, reply); + return; + } + KVMHostLuksCreateEmptyCmd kcmd = new KVMHostLuksCreateEmptyCmd(); + kcmd.psUuid = self.getUuid(); + kcmd.installPath = installPath; + kcmd.size = volumeSize; + kcmd.secFilePath = msg.getVolumeLuksAgentSpec().getEncryptLuksSecretMaterialFilePath(); + httpCallToKvmHost(msg.getDestHost().getUuid(), + KVM_HOST_LUKS_CREATE_EMPTY_PATH, kcmd, KVMHostLuksRsp.class, + new ReturnValueCompletion(msg) { + @Override + public void fail(ErrorCode err) { + reply.setError(err); + bus.reply(msg, reply); + } + + @Override + public void success(KVMHostLuksRsp ret) { + VolumeInventory vol = msg.getVolume(); + // installPath is the canonical "ceph:///" computed above; + // don't run it through buildEmptyVolumeInstallPath again (that helper + // assumes the agent returned a bare pool/uuid relative path). + vol.setInstallPath(installPath); + vol.setFormat(VolumeConstant.VOLUME_FORMAT_RAW); + vol.setActualSize(ret.actualSize); + reply.setVolume(vol); + bus.reply(msg, reply); + } + }); + return; + } + + final CreateEmptyVolumeCmd cmd = new CreateEmptyVolumeCmd(); + cmd.installPath = installPath; + cmd.size = volumeSize; cmd.setShareable(msg.getVolume().isShareable()); cmd.skipIfExisting = msg.isSkipIfExisting(); @@ -1738,8 +1828,6 @@ private void createEmptyVolume(final InstantiateVolumeOnPrimaryStorageMsg msg) { VolumeConstant.VOLUME_FORMAT_QCOW2 : VolumeConstant.VOLUME_FORMAT_RAW ; - final InstantiateVolumeOnPrimaryStorageReply reply = new InstantiateVolumeOnPrimaryStorageReply(); - httpCall(CREATE_VOLUME_PATH, cmd, CreateEmptyVolumeRsp.class, new ReturnValueCompletion(msg) { @Override public void fail(ErrorCode err) { @@ -2115,6 +2203,7 @@ protected void handle(final InstantiateVolumeOnPrimaryStorageMsg msg) { class DownloadToCache { ImageSpec image; VolumeSnapshotInventory snapshot; + Boolean encrypted; private void doDownload(final ReturnValueCompletion completion) { ImageCacheVO cache = Q.New(ImageCacheVO.class) .eq(ImageCacheVO_.primaryStorageUuid, self.getUuid()) @@ -2132,6 +2221,8 @@ private void doDownload(final ReturnValueCompletion completion) { long actualSize = image.getInventory().getActualSize(); String allocatedInstall; ImageCacheVO cvo = new ImageCacheVO(); + String encryptHostUuid; + VolumeLuksAgentSpec imageLuksSpec; @Override public void setup() { @@ -2180,6 +2271,31 @@ public void rollback(FlowRollback trigger, Map data) { } }); + flow(new Flow() { + String __name__ = "prepare-luks-secret-for-snapshot-image-cache"; + + @Override + public void run(FlowTrigger trigger, Map data) { + if (snapshot == null || !Boolean.TRUE.equals(encrypted)) { + trigger.next(); + return; + } + + encryptHostUuid = findConnectedHostForCephLuks(); + imageLuksSpec = snapshotEncryptionHelper.prepareTemporarySnapshotImageSecretMaterial( + encryptHostUuid, + snapshot.getUuid(), + image.getInventory().getUuid(), + encrypted); + trigger.next(); + } + + @Override + public void rollback(FlowRollback trigger, Map data) { + trigger.rollback(); + } + }); + flow(new Flow() { String __name__ = "download-from-" + (snapshot != null ? "volume" : "backup-storage"); @@ -2199,6 +2315,32 @@ public void run(final FlowTrigger trigger, Map data) { private void createFromVolumeSnapshot(FlowTrigger trigger) { deleteOnRollback = true; + if (imageLuksSpec != null && imageLuksSpec.isComplete()) { + KVMHostLuksCloneCmd kcmd = new KVMHostLuksCloneCmd(); + kcmd.psUuid = self.getUuid(); + kcmd.srcPath = snapshot.getPrimaryStorageInstallPath(); + kcmd.dstPath = dstPath; + kcmd.secFilePath = imageLuksSpec.getEncryptLuksSecretMaterialFilePath(); + httpCallToKvmHost(encryptHostUuid, + KVM_HOST_LUKS_CLONE_PATH, kcmd, KVMHostLuksRsp.class, + new ReturnValueCompletion(trigger) { + @Override + public void success(KVMHostLuksRsp rsp) { + if (rsp.actualSize != null) { + actualSize = rsp.actualSize; + } + cachePath = dstPath; + trigger.next(); + } + + @Override + public void fail(ErrorCode errorCode) { + trigger.fail(errorCode); + } + }); + return; + } + CpCmd cmd = new CpCmd(); cmd.srcPath = snapshot.getPrimaryStorageInstallPath(); cmd.dstPath = dstPath; @@ -2529,6 +2671,7 @@ private void createVolumeFromTemplate(final InstantiateRootVolumeFromTemplateOnP String volumePath = makeVolumeInstallPathByTargetPool(msg.getVolume().getUuid(), targetCephPoolName); ImageCacheInventory cache; Long actualSize; + boolean clonedFromEncryptedSnapshotImageCache; @Override public void setup() { @@ -2564,6 +2707,47 @@ public void run(MessageReply reply) { @Override public void run(final FlowTrigger trigger, Map data) { + boolean hasLuksSpec = msg.getVolumeLuksAgentSpec() != null && msg.getVolumeLuksAgentSpec().isComplete(); + clonedFromEncryptedSnapshotImageCache = + hasLuksSpec && isEncryptedSnapshotImageCache(cache); + if (clonedFromEncryptedSnapshotImageCache) { + cloneImage(trigger); + return; + } + + // If the image cache is not already LUKS, RBD clone cannot create a LUKS overlay/top layer: + // RBD only provides block-level COW and preserves the source byte layout. Use qemu-img + // convert on the KVM host to write a new LUKS container with the host-local FIFO secret. + if (hasLuksSpec) { + KVMHostLuksCloneCmd kcmd = new KVMHostLuksCloneCmd(); + kcmd.psUuid = self.getUuid(); + kcmd.srcPath = cloneInstallPath; + kcmd.dstPath = volumePath; + kcmd.secFilePath = msg.getVolumeLuksAgentSpec().getEncryptLuksSecretMaterialFilePath(); + if (ispec.getInventory().getSize() < msg.getVolume().getSize()) { + kcmd.virtualSizeForLuksClone = msg.getVolume().getSize(); + } + httpCallToKvmHost(msg.getDestHost().getUuid(), + KVM_HOST_LUKS_CLONE_PATH, kcmd, KVMHostLuksRsp.class, + new ReturnValueCompletion(trigger) { + @Override + public void fail(ErrorCode err) { + trigger.fail(err); + } + + @Override + public void success(KVMHostLuksRsp ret) { + actualSize = ret.actualSize; + trigger.next(); + } + }); + return; + } + + cloneImage(trigger); + } + + private void cloneImage(final FlowTrigger trigger) { CloneCmd cmd = new CloneCmd(); cmd.srcPath = cloneInstallPath; cmd.dstPath = volumePath; @@ -2589,11 +2773,23 @@ public void success(CloneRsp ret) { @Override public boolean skip(Map data) { ImageInventory image = ispec.getInventory(); + if (clonedFromEncryptedSnapshotImageCache) { + return false; + } + if (msg.getVolumeLuksAgentSpec() != null && msg.getVolumeLuksAgentSpec().isComplete()) { + // LUKS clone converts to a standalone LUKS RBD and applies virtualSizeForLuksClone before this raw resize flow. + return true; + } return image.getSize() >= msg.getVolume().getSize(); } @Override public void run(final FlowTrigger trigger, Map data) { + if (clonedFromEncryptedSnapshotImageCache) { + resizeClonedLuksRbd(trigger); + return; + } + ResizeVolumeOnPrimaryStorageMsg rmsg = new ResizeVolumeOnPrimaryStorageMsg(); rmsg.setVolume(msg.getVolume()); rmsg.setSize(msg.getVolume().getSize()); @@ -2611,6 +2807,31 @@ public void run(MessageReply reply) { }); trigger.next(); } + + private void resizeClonedLuksRbd(final FlowTrigger trigger) { + KVMHostLuksResizeCmd kcmd = new KVMHostLuksResizeCmd(); + kcmd.psUuid = self.getUuid(); + kcmd.installPath = volumePath; + kcmd.secFilePath = msg.getVolumeLuksAgentSpec().getEncryptLuksSecretMaterialFilePath(); + if (ispec.getInventory().getSize() < msg.getVolume().getSize()) { + kcmd.virtualSize = msg.getVolume().getSize(); + } + + httpCallToKvmHost(msg.getDestHost().getUuid(), + KVM_HOST_LUKS_RESIZE_PATH, kcmd, KVMHostLuksRsp.class, + new ReturnValueCompletion(trigger) { + @Override + public void fail(ErrorCode err) { + trigger.fail(err); + } + + @Override + public void success(KVMHostLuksRsp ret) { + actualSize = ret.actualSize; + trigger.next(); + } + }); + } }); done(new FlowDoneHandler(msg) { @@ -2643,6 +2864,35 @@ public void handle(ErrorCode errCode, Map data) { }).start(); } + private boolean isEncryptedSnapshotImageCache(ImageCacheInventory cache) { + if (cache == null) { + return false; + } + + boolean hasTemporarySnapshotImageKey = snapshotEncryptionHelper.hasTemporarySnapshotImageKey(cache.getImageUuid()); + if (hasTemporarySnapshotImageKey) { + return true; + } + + String installUrl = cache.getInstallUrl(); + if (StringUtils.isBlank(installUrl) || !installUrl.contains(ImageConstant.SNAPSHOT_REUSE_IMAGE_SCHEMA)) { + return false; + } + + String snapshotUuid = installUrl.substring(installUrl.lastIndexOf(ImageConstant.SNAPSHOT_REUSE_IMAGE_SCHEMA) + + ImageConstant.SNAPSHOT_REUSE_IMAGE_SCHEMA.length()); + if (snapshotUuid.length() < 32) { + return false; + } + snapshotUuid = snapshotUuid.substring(0, 32); + + Boolean encrypted = Q.New(VolumeSnapshotVO.class) + .eq(VolumeSnapshotVO_.uuid, snapshotUuid) + .select(VolumeSnapshotVO_.encrypted) + .findValue(); + return Boolean.TRUE.equals(encrypted); + } + @Override protected void handle(final DeleteVolumeOnPrimaryStorageMsg msg) { inQueue().name(String.format("delete-volume-on-primarystorage-%s", self.getUuid())) @@ -2884,6 +3134,7 @@ public void fail(ErrorCode errorCode) { cache.image = new ImageSpec(); cache.image.setInventory(msg.getImageInventory()); cache.snapshot = msg.getVolumeSnapshot(); + cache.encrypted = msg.getEncrypted(); cache.download(new ReturnValueCompletion(msg) { @Override public void success(ImageCacheVO inv) { @@ -3198,6 +3449,37 @@ public void success(DeleteRsp ret) { }); } + private void handle(final EncryptVolumeBitsOnPrimaryStorageMsg msg) { + // Encrypt-in-place always consumes a LUKS secret FIFO, so it must run + // on the KVM host that owns the FIFO + key-agent (msg.getHostUuid()). + EncryptVolumeBitsOnPrimaryStorageReply reply = new EncryptVolumeBitsOnPrimaryStorageReply(); + if (StringUtils.isBlank(msg.getHostUuid())) { + reply.setError(operr( + "ceph encryptInPlace requires hostUuid; volume[uuid:%s] installPath[%s] has none", + msg.getVolumeUuid(), msg.getInstallPath())); + bus.reply(msg, reply); + return; + } + KVMHostEncryptInPlaceCmd kcmd = new KVMHostEncryptInPlaceCmd(); + kcmd.psUuid = self.getUuid(); + kcmd.installPath = msg.getInstallPath(); + kcmd.secFilePath = msg.getEncryptLuksSecretMaterialFilePath(); + httpCallToKvmHost(msg.getHostUuid(), + KVM_HOST_LUKS_ENCRYPT_IN_PLACE_PATH, kcmd, KVMHostLuksRsp.class, + new ReturnValueCompletion(msg) { + @Override + public void fail(ErrorCode err) { + reply.setError(err); + bus.reply(msg, reply); + } + + @Override + public void success(KVMHostLuksRsp ret) { + bus.reply(msg, reply); + } + }); + } + @Override protected void handle(final DownloadIsoToPrimaryStorageMsg msg) { final DownloadIsoToPrimaryStorageReply reply = new DownloadIsoToPrimaryStorageReply(); @@ -3382,6 +3664,68 @@ protected void httpCall(final String path, final Agent new HttpCaller<>(path, cmd, retClass, callback, unit, timeout).call(); } + /** + * Send an HTTP cmd to a KVM host's kvmagent (not to cephagent). Used by the + * LUKS variants of clone / createempty / encrypt-in-place, where qemu-img + * must run on the host that has key-agent + libvirt-secret FIFO present. + */ + private void httpCallToKvmHost( + final String hostUuid, + final String path, + final Object cmd, + final Class retClass, + final ReturnValueCompletion callback) { + KVMHostAsyncHttpCallMsg kmsg = new KVMHostAsyncHttpCallMsg(); + kmsg.setCommand(cmd); + kmsg.setPath(path); + kmsg.setHostUuid(hostUuid); + kmsg.setNoStatusCheck(true); + bus.makeTargetServiceIdByResourceUuid(kmsg, HostConstant.SERVICE_ID, hostUuid); + bus.send(kmsg, new CloudBusCallBack(callback) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + callback.fail(reply.getError()); + return; + } + KVMHostAsyncHttpCallReply kr = reply.castReply(); + T rsp = kr.toResponse(retClass); + if (rsp == null) { + callback.fail(operr( + "kvm host[uuid:%s] returned null reply for ceph luks path[%s]", + hostUuid, path)); + return; + } + if (!rsp.isSuccess()) { + callback.fail(operr( + "kvm host[uuid:%s] ceph luks path[%s] failed", + hostUuid, path).withException(rsp.getError())); + return; + } + callback.success(rsp); + } + }); + } + + protected void resizeEncryptedRbdVolumeOnKvmHost(VolumeInventory volume, long size, + ReturnValueCompletion completion) { + String hostUuid = findConnectedHostForCephLuks(); + String secretPath = volumeEncryptedSecretHelper.prepareLuksSecretMaterialFileOnHost(hostUuid, volume.getUuid()); + if (StringUtils.isBlank(secretPath)) { + completion.fail(operr("cannot prepare LUKS secret for encrypted volume[uuid:%s] resize on host[uuid:%s]", + volume.getUuid(), hostUuid)); + return; + } + + KVMHostLuksResizeCmd cmd = new KVMHostLuksResizeCmd(); + cmd.psUuid = self.getUuid(); + cmd.installPath = volume.getInstallPath(); + cmd.secFilePath = secretPath; + cmd.virtualSize = size; + + httpCallToKvmHost(hostUuid, KVM_HOST_LUKS_RESIZE_PATH, cmd, KVMHostLuksRsp.class, completion); + } + public class HttpCaller { private Iterator it; private final List monVOs; @@ -4407,6 +4751,8 @@ protected void handleLocalMessage(Message msg) { handle((GetPrimaryStorageUsageReportMsg) msg); } else if (msg instanceof CleanUpStorageTrashOnPrimaryStorageMsg) { handle((CleanUpStorageTrashOnPrimaryStorageMsg)msg); + } else if (msg instanceof EncryptVolumeBitsOnPrimaryStorageMsg) { + handle((EncryptVolumeBitsOnPrimaryStorageMsg) msg); } else { super.handleLocalMessage(msg); } @@ -4977,6 +5323,15 @@ public void done() { private void fastCreateVolumeFromSnapshot(final CreateVolumeFromVolumeSnapshotOnPrimaryStorageMsg msg, final NoErrorCompletion completion) { final CreateVolumeFromVolumeSnapshotOnPrimaryStorageReply reply = new CreateVolumeFromVolumeSnapshotOnPrimaryStorageReply(); + VolumeVO targetVolume = Q.New(VolumeVO.class).eq(VolumeVO_.uuid, msg.getVolumeUuid()).find(); + if (targetVolume != null && targetVolume.isEncrypted()) { + // RBD fast clone cannot change a plain snapshot into a LUKS volume. + // Convert the snapshot into an independent LUKS RBD; the helper also + // passes the target virtual size so the KVM agent resizes after convert. + flattenSnapshotToEncryptedVolume(msg, completion); + return; + } + // create volume first, then reserve size for it, so we use snapshot poolName for volume create String snapShotPath = msg.getSnapshot().getPrimaryStorageInstallPath(); final String volPath = makeVolumeInstallPathByTargetPool(msg.getVolumeUuid(), getTargetPoolNameFromAllocatedUrl(snapShotPath)); @@ -5051,6 +5406,12 @@ public void fail(ErrorCode errorCode) { private void createVolumeFromSnapshot(final CreateVolumeFromVolumeSnapshotOnPrimaryStorageMsg msg, final NoErrorCompletion completion) { final CreateVolumeFromVolumeSnapshotOnPrimaryStorageReply reply = new CreateVolumeFromVolumeSnapshotOnPrimaryStorageReply(); + VolumeVO targetVolume = Q.New(VolumeVO.class).eq(VolumeVO_.uuid, msg.getVolumeUuid()).find(); + if (targetVolume != null && targetVolume.isEncrypted()) { + flattenSnapshotToEncryptedVolume(msg, completion); + return; + } + String snapShotPath = msg.getSnapshot().getPrimaryStorageInstallPath(); final String volPath = makeVolumeInstallPathByTargetPool(msg.getVolumeUuid(), getTargetPoolNameFromAllocatedUrl(snapShotPath)); VolumeSnapshotInventory sp = msg.getSnapshot(); @@ -5079,6 +5440,74 @@ public void fail(ErrorCode errorCode) { }); } + private void flattenSnapshotToEncryptedVolume(final CreateVolumeFromVolumeSnapshotOnPrimaryStorageMsg msg, + final NoErrorCompletion completion) { + final CreateVolumeFromVolumeSnapshotOnPrimaryStorageReply reply = + new CreateVolumeFromVolumeSnapshotOnPrimaryStorageReply(); + VolumeSnapshotInventory sp = msg.getSnapshot(); + String snapshotPath = sp.getPrimaryStorageInstallPath(); + final String volPath = makeVolumeInstallPathByTargetPool(msg.getVolumeUuid(), getTargetPoolNameFromAllocatedUrl(snapshotPath)); + String hostUuid = findConnectedHostForCephLuks(); + VolumeLuksAgentSpec dstSpec = msg.getVolumeLuksAgentSpec(); + String secFilePath = dstSpec != null && dstSpec.isComplete() ? + dstSpec.getEncryptLuksSecretMaterialFilePath() : null; + if (StringUtils.isBlank(secFilePath)) { + secFilePath = volumeEncryptedSecretHelper.prepareLuksSecretMaterialFileOnHost(hostUuid, msg.getVolumeUuid()); + } + if (StringUtils.isBlank(secFilePath)) { + throw new OperationFailureException(operr( + "cannot prepare LUKS secret for encrypted volume[uuid:%s] from snapshot[uuid:%s]", + msg.getVolumeUuid(), sp.getUuid())); + } + + KVMHostLuksCloneCmd kcmd = new KVMHostLuksCloneCmd(); + kcmd.psUuid = self.getUuid(); + kcmd.srcPath = snapshotPath; + kcmd.dstPath = volPath; + kcmd.secFilePath = secFilePath; + + VolumeVO volume = Q.New(VolumeVO.class).eq(VolumeVO_.uuid, msg.getVolumeUuid()).find(); + if (volume != null && volume.getSize() != 0) { + kcmd.virtualSizeForLuksClone = volume.getSize(); + } + + httpCallToKvmHost(hostUuid, + KVM_HOST_LUKS_CLONE_PATH, kcmd, KVMHostLuksRsp.class, + new ReturnValueCompletion(completion) { + @Override + public void success(KVMHostLuksRsp rsp) { + reply.setInstallPath(volPath); + long asize = rsp.actualSize == null ? 1 : rsp.actualSize; + reply.setActualSize(asize); + reply.setSize(volume == null ? sp.getSize() : volume.getSize()); + bus.reply(msg, reply); + completion.done(); + } + + @Override + public void fail(ErrorCode errorCode) { + reply.setError(errorCode); + bus.reply(msg, reply); + completion.done(); + } + }); + } + + private String findConnectedHostForCephLuks() { + String hostUuid = Q.New(PrimaryStorageHostRefVO.class) + .eq(PrimaryStorageHostRefVO_.primaryStorageUuid, self.getUuid()) + .eq(PrimaryStorageHostRefVO_.status, PrimaryStorageHostStatus.Connected) + .select(PrimaryStorageHostRefVO_.hostUuid) + .limit(1) + .findValue(); + if (StringUtils.isBlank(hostUuid)) { + throw new OperationFailureException(operr( + "cannot find a connected host attached to ceph primary storage[uuid:%s] to run LUKS RBD operation", + self.getUuid())); + } + return hostUuid; + } + protected void handle(final RevertVolumeFromSnapshotOnPrimaryStorageMsg msg) { final RevertVolumeFromSnapshotOnPrimaryStorageReply reply = new RevertVolumeFromSnapshotOnPrimaryStorageReply(); @@ -5230,6 +5659,37 @@ public void run(MessageReply reply) { @Override public void run(final FlowTrigger trigger, Map data) { + if (Boolean.TRUE.equals(msg.getVolume().getEncrypted())) { + String hostUuid = findConnectedHostForCephLuks(); + String secretPath = volumeEncryptedSecretHelper.prepareLuksSecretMaterialFileOnHost( + hostUuid, msg.getVolume().getUuid()); + if (StringUtils.isBlank(secretPath)) { + trigger.fail(operr("cannot prepare LUKS secret for encrypted volume[uuid:%s] reimage on host[uuid:%s]", + msg.getVolume().getUuid(), hostUuid)); + return; + } + + KVMHostLuksCloneCmd kcmd = new KVMHostLuksCloneCmd(); + kcmd.psUuid = self.getUuid(); + kcmd.srcPath = installUrl; + kcmd.dstPath = volumePath; + kcmd.secFilePath = secretPath; + httpCallToKvmHost(hostUuid, + KVM_HOST_LUKS_CLONE_PATH, kcmd, KVMHostLuksRsp.class, + new ReturnValueCompletion(trigger) { + @Override + public void fail(ErrorCode err) { + trigger.fail(err); + } + + @Override + public void success(KVMHostLuksRsp ret) { + trigger.next(); + } + }); + return; + } + CloneCmd cmd = new CloneCmd(); cmd.srcPath = installUrl; cmd.dstPath = volumePath; @@ -5387,6 +5847,14 @@ private void takeSnapshot(final TakeSnapshotMsg msg, final NoErrorCompletion com q.add(VolumeVO_.uuid, Op.EQ, sp.getVolumeUuid()); String volumePath = q.findValue(); + VolumeVO volume = dbf.findByUuid(sp.getVolumeUuid(), VolumeVO.class); + if (volume != null && volume.isEncrypted()) { + // RBD snapshot operations do not open guest-visible LUKS/qcow2 data, + // so no secret material is needed here. Only inherit the volume key + // binding so later qemu-img based conversions know this snapshot is encrypted. + snapshotEncryptionHelper.inheritVolumeKeyToSnapshot(volume, sp); + } + final String spPath = String.format("%s@%s", volumePath, sp.getUuid()); CreateSnapshotCmd cmd = new CreateSnapshotCmd(); cmd.volumeUuid = sp.getVolumeUuid(); @@ -5403,6 +5871,9 @@ public void success(CreateSnapshotRsp rsp) { sp.setPrimaryStorageInstallPath(rsp.getInstallPath()); sp.setType(VolumeSnapshotConstant.STORAGE_SNAPSHOT_TYPE.toString()); sp.setFormat(VolumeConstant.VOLUME_FORMAT_RAW); + if (volume != null && volume.isEncrypted()) { + snapshotEncryptionHelper.completeTakeSnapshot(volume, sp); + } reply.setInventory(sp); bus.reply(msg, reply); completion.done(); diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java index cc7c9083a7a..a3f50b78df4 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java @@ -484,6 +484,30 @@ public void setSecretUuid(String secretUuid) { } } + public static class SecretHostEnsureLuksSecretFileCmd extends AgentCommand { + private String encryptedDek; + + public String getEncryptedDek() { + return encryptedDek; + } + + public void setEncryptedDek(String encryptedDek) { + this.encryptedDek = encryptedDek; + } + } + + public static class SecretHostEnsureLuksSecretFileResponse extends AgentResponse { + private String secFilePath; + + public String getSecFilePath() { + return secFilePath; + } + + public void setSecFilePath(String secFilePath) { + this.secFilePath = secFilePath; + } + } + public static class SecretHostGetCmd extends AgentCommand { private String vmUuid; private String purpose; @@ -559,6 +583,39 @@ public void setSecretUuid(String secretUuid) { } } + public static class ResolveVolumeLibvirtSecretCmd extends AgentCommand { + private String vmUuid; + private String volumeUuid; + + public String getVmUuid() { + return vmUuid; + } + + public void setVmUuid(String vmUuid) { + this.vmUuid = vmUuid; + } + + public String getVolumeUuid() { + return volumeUuid; + } + + public void setVolumeUuid(String volumeUuid) { + this.volumeUuid = volumeUuid; + } + } + + public static class ResolveVolumeLibvirtSecretResponse extends AgentResponse { + private String secretUuid; + + public String getSecretUuid() { + return secretUuid; + } + + public void setSecretUuid(String secretUuid) { + this.secretUuid = secretUuid; + } + } + public static class SecretHostDeleteCmd extends AgentCommand { private String vmUuid; private String purpose; @@ -3799,6 +3856,39 @@ public Map> getInactiveVolumePaths() { } } + public static class GetActiveVolumeSizeCmd extends AgentCommand { + private String vmUuid; + private List installPaths; + + public String getVmUuid() { + return vmUuid; + } + + public void setVmUuid(String vmUuid) { + this.vmUuid = vmUuid; + } + + public List getInstallPaths() { + return installPaths; + } + + public void setInstallPaths(List installPaths) { + this.installPaths = installPaths; + } + } + + public static class GetActiveVolumeSizeRsp extends AgentResponse { + private Map volumeSizes; + + public Map getVolumeSizes() { + return volumeSizes; + } + + public void setVolumeSizes(Map volumeSizes) { + this.volumeSizes = volumeSizes; + } + } + public static class RefreshAllRulesOnHostCmd extends AgentCommand { private List vmNicTOs; private Map> ruleTOs; @@ -4300,6 +4390,7 @@ public static class TakeSnapshotCmd extends AgentCommand implements HasThreadCon private String volumeInstallPath; private String newVolumeUuid; private String newVolumeInstallPath; + private String encryptedDek; private boolean online; private long timeout; @@ -4370,6 +4461,14 @@ public void setNewVolumeUuid(String newVolumeUuid) { this.newVolumeUuid = newVolumeUuid; } + public String getEncryptedDek() { + return encryptedDek; + } + + public void setEncryptedDek(String encryptedDek) { + this.encryptedDek = encryptedDek; + } + public VolumeTO getVolume() { return volume; } diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMConstant.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMConstant.java index 35ffa2c2065..0b849c586d0 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMConstant.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMConstant.java @@ -55,6 +55,8 @@ public interface KVMConstant { String KVM_VM_ONLINE_INCREASE_MEMORY = "/vm/increase/mem"; String KVM_VM_SYNC_PATH = "/vm/vmsync"; String KVM_VOLUME_SYNC_PATH = "/vm/volumesync"; + String KVM_GET_ACTIVE_VOLUME_SIZE_PATH = "/vm/volume/active/size"; + String KVM_VOLUME_RESOLVE_LIBVIRT_SECRET_UUID_PATH = "/vm/volume/resolveLibvirtSecretUuid"; String KVM_ATTACH_VOLUME = "/vm/attachdatavolume"; String KVM_DETACH_VOLUME = "/vm/detachdatavolume"; String KVM_ATTACH_NIC_PATH = "/vm/attachnic"; @@ -133,6 +135,7 @@ public interface KVMConstant { String KVM_VERIFY_ENVELOPE_KEY_PATH = "/host/key/envelope/checkEnvelopeKey"; String KVM_GET_SECRET_PATH = "/host/key/envelope/getSecret"; String KVM_ENSURE_SECRET_PATH = "/host/key/envelope/ensureSecret"; + String KVM_WRITE_SECRET_MATERIAL_FILE_PATH = "/host/key/envelope/writeSecretMaterialFile"; String KVM_DELETE_SECRET_PATH = "/host/key/envelope/deleteSecret"; /** HTTP timeout in seconds for envelope key sync (verify/create/rotate/get) to agent. */ @@ -141,6 +144,16 @@ public interface KVMConstant { /** Max size in bytes for DEK payload in SecretHostDefine (decoded from dekBase64). */ int MAX_DEK_BYTES = 1024; String HOST_SECRET_USAGE_INSTANCE_VTPM = "tpm0"; + /** + * Per-volume usage instance string for the libvirt LUKS secret. Returned + * value is what we feed key-agent in {@code SecretHostDefineMsg} / + * {@code SecretHostGetMsg}; key-agent splices it into the libvirt usage + * name as {@code vm---version-}, so the + * resulting libvirt secret usage name is unique per (vm, volume, version). + */ + static String volumeSecretUsageInstance(String volumeUuid) { + return "volume-" + volumeUuid; + } String KVM_HOST_FILE_DOWNLOAD_PATH = "/host/file/download"; String KVM_HOST_FILE_UPLOAD_PATH = "/host/file/upload"; diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java index 828aeb3a7cd..3f9c079810f 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java @@ -57,6 +57,8 @@ import org.zstack.header.host.MigrateVmOnHypervisorMsg.StorageMigrationPolicy; import org.zstack.header.secret.SecretHostDefineMsg; import org.zstack.header.secret.SecretHostDefineReply; +import org.zstack.header.secret.SecretHostEnsureLuksSecretFileMsg; +import org.zstack.header.secret.SecretHostEnsureLuksSecretFileReply; import org.zstack.header.secret.SecretHostDeleteMsg; import org.zstack.header.secret.SecretHostDeleteReply; import org.zstack.header.secret.SecretHostGetMsg; @@ -760,6 +762,8 @@ protected void handleLocalMessage(Message msg) { handle((SecretHostGetMsg) msg); } else if (msg instanceof ResolveVtpmLibvirtSecretOnHypervisorMsg) { handle((ResolveVtpmLibvirtSecretOnHypervisorMsg) msg); + } else if (msg instanceof SecretHostEnsureLuksSecretFileMsg) { + handle((SecretHostEnsureLuksSecretFileMsg) msg); } else if (msg instanceof SecretHostDefineMsg) { handle((SecretHostDefineMsg) msg); } else if (msg instanceof SecretHostDeleteMsg) { @@ -3209,8 +3213,14 @@ public void run(final FlowTrigger trigger, Map data) { protected void scripts() { s.diskMigrationMap.forEach((oldVolumeInstallPath, newVolumeUuid) -> { VolumeVO vo = findByUuid(newVolumeUuid, VolumeVO.class); - diskMigrationMap.put(oldVolumeInstallPath, - VolumeTO.valueOf(VolumeInventory.valueOf(vo), (KVMHostInventory) getSelfInventory())); + VolumeTO vto = VolumeTO.valueOf(VolumeInventory.valueOf(vo), (KVMHostInventory) getSelfInventory()); + if (s.volumeLuksSecrets != null) { + String secretUuid = s.volumeLuksSecrets.get(vo.getUuid()); + if (secretUuid != null) { + vto.setLuksSecretUuid(secretUuid); + } + } + diskMigrationMap.put(oldVolumeInstallPath, vto); }); } }.execute(); @@ -3366,6 +3376,7 @@ class MigrateStruct { String srcHostMnIp; String srcHostUuid; Map diskMigrationMap; + Map volumeLuksSecrets; boolean reload; long bandwidth; } @@ -3380,6 +3391,7 @@ private MigrateStruct buildMigrateStuct(final MigrateVmOnHypervisorMsg msg){ s.strategy = msg.getStrategy(); s.downTime = msg.getDownTime(); s.diskMigrationMap = msg.getDiskMigrationMap(); + s.volumeLuksSecrets = msg.getVolumeLuksSecrets(); s.reload = msg.isReload(); s.bandwidth = msg.getBandwidth(); @@ -4495,6 +4507,15 @@ protected void startVm(final VmInstanceSpec spec, final NeedReplyMessage msg, fi cmd.setVmCpuModel(vmCpuMode); } + // Key must match VolumeEncryptedStartExtension.EXT_DATA_KEY in the storage module. + // Inlined here to avoid a kvm -> storage compile-time dep (kvm builds before storage). + @SuppressWarnings("unchecked") + Map volLuksSecrets = spec.getExtensionData( + "VolumeLuksSecrets", Map.class); + if (volLuksSecrets != null && volLuksSecrets.containsKey(rootVolume.getResourceUuid())) { + rootVolume.setLuksSecretUuid(volLuksSecrets.get(rootVolume.getResourceUuid())); + } + cmd.setRootVolume(rootVolume); cmd.setUseBootMenu(VmGlobalConfig.VM_BOOT_MENU.value(Boolean.class)); @@ -4511,6 +4532,9 @@ protected void startVm(final VmInstanceSpec spec, final NeedReplyMessage msg, fi // except for platform = Other, always use virtio driver for data volume // set bug https://github.com/zxwing/premium/issues/1050 v.setUseVirtio(!ImagePlatform.Other.toString().equals(platform)); + if (volLuksSecrets != null && volLuksSecrets.containsKey(v.getResourceUuid())) { + v.setLuksSecretUuid(volLuksSecrets.get(v.getResourceUuid())); + } dataVolumes.add(v); } dataVolumes.sort(Comparator.comparing(VolumeTO::getDeviceId)); @@ -5408,6 +5432,117 @@ public void run(MessageReply r) { }); } + private void handle(SecretHostEnsureLuksSecretFileMsg msg) { + SecretHostEnsureLuksSecretFileReply reply = new SecretHostEnsureLuksSecretFileReply(); + if (org.apache.commons.lang.StringUtils.isBlank(msg.getDekBase64())) { + reply.setError(operr("dekBase64 is required")); + bus.reply(msg, reply); + return; + } + if (StringUtils.isBlank(msg.getHostUuid())) { + reply.setError(operr("hostUuid is required for LUKS secret material file on hypervisor")); + bus.reply(msg, reply); + return; + } + String hostUuid = getSelf().getUuid(); + HostKeyIdentityVO identity = HostKeyIdentityHelper.getHostKeyIdentity(dbf, hostUuid); + String pubKey = identity != null ? org.apache.commons.lang.StringUtils.trimToNull(identity.getPublicKey()) : null; + Boolean verifyOk = identity != null ? identity.getVerified() : null; + if (pubKey == null) { + reply.setError(operr("no public key for host, connect/reconnect did not sync key")); + bus.reply(msg, reply); + return; + } + String storedFingerprint = StringUtils.trimToNull(identity.getFingerprint()); + String computed = HostKeyIdentityHelper.fingerprintFromPublicKey(pubKey); + if (storedFingerprint == null || !StringUtils.equals(storedFingerprint, computed)) { + reply.setError(operr("host public key fingerprint mismatch, key may be corrupted or tampered")); + bus.reply(msg, reply); + return; + } + if (!Boolean.TRUE.equals(verifyOk)) { + reply.setError(operr("host secret key verify not ok, not synced")); + bus.reply(msg, reply); + return; + } + byte[] dekRaw; + try { + dekRaw = java.util.Base64.getDecoder().decode(msg.getDekBase64().trim()); + } catch (IllegalArgumentException e) { + reply.setError(operr("invalid dekBase64: %s", e.getMessage())); + bus.reply(msg, reply); + return; + } + if (dekRaw == null || dekRaw.length == 0) { + reply.setError(operr("dekBase64 decoded to empty")); + bus.reply(msg, reply); + return; + } + if (dekRaw.length > KVMConstant.MAX_DEK_BYTES) { + reply.setError(operr("dekBase64 decoded payload is too large")); + bus.reply(msg, reply); + return; + } + byte[] pubKeyBytes; + try { + pubKeyBytes = java.util.Base64.getDecoder().decode(pubKey); + } catch (IllegalArgumentException e) { + reply.setError(operr("invalid host public key in DB: %s", e.getMessage())); + bus.reply(msg, reply); + return; + } + if (pubKeyBytes == null || pubKeyBytes.length != 32) { + reply.setError(operr("host public key must be 32 bytes (X25519)")); + bus.reply(msg, reply); + return; + } + java.util.List sealers = pluginRegistry.getExtensionList(HostSecretEnvelopeCryptoExtensionPoint.class); + if (sealers == null || sealers.isEmpty()) { + reply.setError(operr("host secret envelope sealer not available (premium crypto module required)")); + bus.reply(msg, reply); + return; + } + byte[] envelope; + try { + envelope = sealers.get(0).seal(pubKeyBytes, dekRaw); + } catch (Exception e) { + reply.setError(operr("HPKE seal failed: %s", e.getMessage())); + bus.reply(msg, reply); + return; + } + String envelopeDekBase64 = java.util.Base64.getEncoder().encodeToString(envelope); + KVMAgentCommands.SecretHostEnsureLuksSecretFileCmd cmd = new KVMAgentCommands.SecretHostEnsureLuksSecretFileCmd(); + cmd.setEncryptedDek(envelopeDekBase64); + + KVMHostAsyncHttpCallMsg kmsg = new KVMHostAsyncHttpCallMsg(); + kmsg.setCommand(cmd); + kmsg.setPath(KVMConstant.KVM_WRITE_SECRET_MATERIAL_FILE_PATH); + kmsg.setHostUuid(msg.getHostUuid()); + kmsg.setTimeout(TimeUnit.SECONDS.toMillis(KVMConstant.ENVELOPE_KEY_HTTP_TIMEOUT_SEC)); + bus.makeTargetServiceIdByResourceUuid(kmsg, HostConstant.SERVICE_ID, msg.getHostUuid()); + bus.send(kmsg, new CloudBusCallBack(msg) { + @Override + public void run(MessageReply r) { + if (!r.isSuccess()) { + reply.setError(r.getError()); + bus.reply(msg, reply); + return; + } + KVMHostAsyncHttpCallReply kreply = r.castReply(); + KVMAgentCommands.SecretHostEnsureLuksSecretFileResponse rsp = + kreply.toResponse(KVMAgentCommands.SecretHostEnsureLuksSecretFileResponse.class); + if (rsp != null && rsp.isSuccess() && StringUtils.isNotBlank(rsp.getSecFilePath())) { + reply.setSecFilePath(rsp.getSecFilePath()); + } else if (rsp != null && rsp.isSuccess()) { + reply.setError(operr("prepare LUKS secret channel succeeded but secFilePath is empty")); + } else { + reply.setError(buildSecretAgentError(rsp, "prepare LUKS secret channel failed")); + } + bus.reply(msg, reply); + } + }); + } + private void handle(SecretHostDefineMsg msg) { SecretHostDefineReply reply = new SecretHostDefineReply(); if (org.apache.commons.lang.StringUtils.isBlank(msg.getDekBase64())) { diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/VolumeTO.java b/plugin/kvm/src/main/java/org/zstack/kvm/VolumeTO.java index db09c379de4..e59cff9ec23 100644 --- a/plugin/kvm/src/main/java/org/zstack/kvm/VolumeTO.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/VolumeTO.java @@ -54,6 +54,10 @@ public class VolumeTO extends BaseVirtualDeviceTO { private String ioThreadPin; private int controllerIndex; + // Host-local libvirt secret UUID holding the LUKS passphrase for an + // encrypted volume. cephx auth's secretUuid is on KVMCephVolumeTO. + private String luksSecretUuid; + static { deviceTypes.put(VolumeProtocol.Vhost, VHOST); deviceTypes.put(VolumeProtocol.CBD, CBD); @@ -83,6 +87,7 @@ public VolumeTO(VolumeTO other) { this.ioThreadId = other.ioThreadId; this.ioThreadPin = other.ioThreadPin; this.controllerIndex = other.controllerIndex; + this.luksSecretUuid = other.luksSecretUuid; } public static List valueOf(List vols, KVMHostInventory host) { @@ -313,4 +318,12 @@ public int getControllerIndex() { public void setControllerIndex(int controllerIndex) { this.controllerIndex = controllerIndex; } + + public String getLuksSecretUuid() { + return luksSecretUuid; + } + + public void setLuksSecretUuid(String luksSecretUuid) { + this.luksSecretUuid = luksSecretUuid; + } } diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/tpm/KvmTpmExtensions.java b/plugin/kvm/src/main/java/org/zstack/kvm/tpm/KvmTpmExtensions.java index 8cbf5b52a7c..38d2742ea06 100644 --- a/plugin/kvm/src/main/java/org/zstack/kvm/tpm/KvmTpmExtensions.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/tpm/KvmTpmExtensions.java @@ -326,7 +326,7 @@ public void run(MessageReply reply) { } ErrorCode errorCode = reply.getError(); - if (errorCode != null && isVtpmSecretNotFoundOnHost(errorCode)) { + if (SecretHostGetReply.isSecretNotFound(errorCode)) { trigger.next(); return; } @@ -851,14 +851,6 @@ private boolean isVmCurrentlyOnExpectedHost(String vmUuid, String expectedHostUu return expectedHostUuid.equals(currentHostUuid); } - private static boolean isVtpmSecretNotFoundOnHost(ErrorCode errorCode) { - if (SecretHostGetReply.ERROR_CODE_SECRET_NOT_FOUND.equals(errorCode.getCode())) { - return true; - } - String details = errorCode.getDetails(); - return details != null && details.contains(SecretHostGetReply.ERROR_CODE_SECRET_NOT_FOUND); - } - private void deleteHostSecretBestEffort(String hostUuid, String vmUuid, Integer keyVersion, String reason) { if (StringUtils.isBlank(hostUuid) || StringUtils.isBlank(vmUuid) || keyVersion == null) { logger.info(String.format( diff --git a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageBase.java b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageBase.java index eea5f8beb92..c44590fa49c 100755 --- a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageBase.java +++ b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageBase.java @@ -1,5 +1,6 @@ package org.zstack.storage.primary.local; +import org.apache.commons.lang.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.transaction.annotation.Transactional; import org.zstack.compute.host.VolumeMigrationTargetHostFilter; @@ -907,11 +908,51 @@ public void handleLocalMessage(Message msg) { handle((CommitVolumeSnapshotOnPrimaryStorageMsg) msg); } else if (msg instanceof PullVolumeSnapshotOnPrimaryStorageMsg) { handle((PullVolumeSnapshotOnPrimaryStorageMsg) msg); + } else if (msg instanceof EncryptVolumeBitsOnPrimaryStorageMsg) { + handle((EncryptVolumeBitsOnPrimaryStorageMsg) msg); + } else if (msg instanceof ConvertVolumeEncryptionOnPrimaryStorageMsg) { + handle((ConvertVolumeEncryptionOnPrimaryStorageMsg) msg); } else { super.handleLocalMessage(msg); } } + protected void handle(ConvertVolumeEncryptionOnPrimaryStorageMsg msg) { + ConvertVolumeEncryptionOnPrimaryStorageReply reply = new ConvertVolumeEncryptionOnPrimaryStorageReply(); + String hostUuid = StringUtils.isNotBlank(msg.getHostUuid()) ? + msg.getHostUuid() : getHostUuidByResourceUuid(msg.getVolume().getUuid()); + LocalStorageHypervisorBackend bkd = getHypervisorBackendFactoryByHostUuid(hostUuid).getHypervisorBackend(self); + bkd.handle(msg, hostUuid, new ReturnValueCompletion(msg) { + @Override + public void success(ConvertVolumeEncryptionOnPrimaryStorageReply returnValue) { + bus.reply(msg, returnValue); + } + + @Override + public void fail(ErrorCode errorCode) { + reply.setError(errorCode); + bus.reply(msg, reply); + } + }); + } + + private void handle(EncryptVolumeBitsOnPrimaryStorageMsg msg) { + LocalStorageHypervisorBackend bkd = getHypervisorBackendFactoryByHostUuid(msg.getHostUuid()).getHypervisorBackend(self); + bkd.handle(msg, new ReturnValueCompletion(msg) { + @Override + public void success(EncryptVolumeBitsOnPrimaryStorageReply reply) { + bus.reply(msg, reply); + } + + @Override + public void fail(ErrorCode errorCode) { + EncryptVolumeBitsOnPrimaryStorageReply reply = new EncryptVolumeBitsOnPrimaryStorageReply(); + reply.setError(errorCode); + bus.reply(msg, reply); + } + }); + } + private void handle(DownloadBitsFromKVMHostToPrimaryStorageMsg msg) { LocalStorageHypervisorBackend bkd = getHypervisorBackendFactoryByHostUuid(msg.getSrcHostUuid()).getHypervisorBackend(self); bkd.handle(msg, new ReturnValueCompletion(msg) { diff --git a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageCreateEmptyVolumeMsg.java b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageCreateEmptyVolumeMsg.java index 414b564bc14..c882af3ff45 100755 --- a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageCreateEmptyVolumeMsg.java +++ b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageCreateEmptyVolumeMsg.java @@ -3,6 +3,7 @@ import org.zstack.header.message.NeedReplyMessage; import org.zstack.header.storage.primary.PrimaryStorageMessage; import org.zstack.header.volume.VolumeInventory; +import org.zstack.header.volume.VolumeLuksAgentSpec; /** * Created by frank on 10/24/2015. @@ -12,6 +13,7 @@ public class LocalStorageCreateEmptyVolumeMsg extends NeedReplyMessage implement private String hostUuid; private String backingFile; private VolumeInventory volume; + private VolumeLuksAgentSpec volumeLuksAgentSpec; public String getBackingFile() { return backingFile; @@ -45,4 +47,12 @@ public VolumeInventory getVolume() { public void setVolume(VolumeInventory volume) { this.volume = volume; } + + public VolumeLuksAgentSpec getVolumeLuksAgentSpec() { + return volumeLuksAgentSpec; + } + + public void setVolumeLuksAgentSpec(VolumeLuksAgentSpec volumeLuksAgentSpec) { + this.volumeLuksAgentSpec = volumeLuksAgentSpec; + } } diff --git a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageFactory.java b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageFactory.java index a462ac65a5f..a908139c67a 100755 --- a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageFactory.java +++ b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageFactory.java @@ -1080,6 +1080,8 @@ public void instantiateDataVolumeOnCreation(InstantiateVolumeMsg msg, VolumeInve imsg.setVolume(volume); imsg.setPrimaryStorageUuid(msg.getPrimaryStorageUuid()); imsg.setDestHost(HostInventory.valueOf(dbf.findByUuid(hostUuid, HostVO.class))); + // For root volume with backing file + imsg.setVolumeLuksAgentSpec(msg.getVolumeLuksAgentSpec()); bus.makeTargetServiceIdByResourceUuid(imsg, PrimaryStorageConstant.SERVICE_ID, msg.getPrimaryStorageUuid()); bus.send(imsg, new CloudBusCallBack(completion) { @Override diff --git a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageHypervisorBackend.java b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageHypervisorBackend.java index d8226d932b2..b2d5816d860 100755 --- a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageHypervisorBackend.java +++ b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageHypervisorBackend.java @@ -108,9 +108,11 @@ public LocalStorageHypervisorBackend(PrimaryStorageVO self) { abstract void deleteBits(String path, String hostUuid, Completion completion); - abstract void createEmptyVolume(VolumeInventory volume, String hostUuid, ReturnValueCompletion completion); + abstract void createEmptyVolume(VolumeInventory volume, String hostUuid, VolumeLuksAgentSpec volumeLuksAgentSpec, + ReturnValueCompletion completion); - abstract void createEmptyVolumeWithBackingFile(VolumeInventory volume, String hostUuid, String backingFile, ReturnValueCompletion completion); + abstract void createEmptyVolumeWithBackingFile(VolumeInventory volume, String hostUuid, String backingFile, + VolumeLuksAgentSpec volumeLuksAgentSpec, ReturnValueCompletion completion); abstract void checkHostAttachedPSMountPath(String hostUuid, ReturnValueCompletion completion); @@ -133,4 +135,8 @@ public LocalStorageHypervisorBackend(PrimaryStorageVO self) { abstract void handle(CleanupVmInstanceMetadataOnPrimaryStorageMsg msg, String hostUuid, ReturnValueCompletion completion); abstract void handle(RebaseVolumeBackingFileOnPrimaryStorageMsg msg, String hostUuid, ReturnValueCompletion completion); + + abstract void handle(EncryptVolumeBitsOnPrimaryStorageMsg msg, ReturnValueCompletion completion); + + abstract void handle(ConvertVolumeEncryptionOnPrimaryStorageMsg msg, String hostUuid, ReturnValueCompletion completion); } diff --git a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmBackend.java b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmBackend.java index 5b4324495b2..a33c16050ab 100755 --- a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmBackend.java +++ b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmBackend.java @@ -38,6 +38,7 @@ import org.zstack.header.image.ImageInventory; import org.zstack.header.image.ImageStatus; import org.zstack.header.image.ImageVO; +import org.zstack.header.log.NoLogging; import org.zstack.header.message.Message; import org.zstack.header.message.MessageReply; import org.zstack.header.rest.RESTFacade; @@ -55,6 +56,8 @@ import org.zstack.header.volume.*; import org.zstack.identity.AccountManager; import org.zstack.kvm.*; +import org.zstack.storage.encrypt.VolumeEncryptedSecretHelper; +import org.zstack.storage.encrypt.VolumeSnapshotEncryptionHelper; import org.zstack.storage.primary.*; import org.zstack.storage.primary.local.LocalStorageKvmMigrateVmFlow.CopyBitsFromRemoteCmd; import org.zstack.storage.primary.local.MigrateBitsStruct.ResourceInfo; @@ -89,6 +92,10 @@ public class LocalStorageKvmBackend extends LocalStorageHypervisorBackend { private RESTFacade restf; @Autowired private PluginRegistry pluginRgty; + @Autowired + private VolumeSnapshotEncryptionHelper snapshotEncryptionHelper; + @Autowired + private VolumeEncryptedSecretHelper volumeEncryptedSecretHelper; public static class AgentCommand extends KVMAgentCommands.PrimaryStorageCommand { public String uuid; @@ -205,6 +212,9 @@ public static class CreateEmptyVolumeCmd extends AgentCommand { private String volumeUuid; private String backingFile; private String volumeFormat; + private String encryptLuksSecretMaterialFilePath; + @NoLogging + private String encryptedDek; public String getBackingFile() { return backingFile; @@ -222,6 +232,22 @@ public void setVolumeFormat(String volumeFormat) { this.volumeFormat = volumeFormat; } + public String getEncryptLuksSecretMaterialFilePath() { + return encryptLuksSecretMaterialFilePath; + } + + public void setEncryptLuksSecretMaterialFilePath(String encryptLuksSecretMaterialFilePath) { + this.encryptLuksSecretMaterialFilePath = encryptLuksSecretMaterialFilePath; + } + + public String getEncryptedDek() { + return encryptedDek; + } + + public void setEncryptedDek(String encryptedDek) { + this.encryptedDek = encryptedDek; + } + public String getInstallUrl() { return installUrl; } @@ -268,6 +294,26 @@ public static class CreateEmptyVolumeRsp extends AgentResponse { public Long size; } + public static class EncryptVolumeBitsCmd extends AgentCommand { + public String installPath; + public String encryptLuksSecretMaterialFilePath; + } + + public static class EncryptVolumeBitsRsp extends AgentResponse { + } + + public static class ConvertVolumeEncryptionCmd extends AgentCommand { + public String volumeUuid; + public boolean targetEncrypted; + public List items; + @NoLogging + public String encryptedDek; + } + + public static class ConvertVolumeEncryptionRsp extends AgentResponse { + public Map actualSizes; + } + public static class GetPhysicalCapacityCmd extends AgentCommand { private String hostUuid; @@ -285,6 +331,15 @@ public static class CreateVolumeFromCacheCmd extends AgentCommand { private String installUrl; private String volumeUuid; private long virtualSize; + private String encryptLuksSecretMaterialFilePath; + + public String getEncryptLuksSecretMaterialFilePath() { + return encryptLuksSecretMaterialFilePath; + } + + public void setEncryptLuksSecretMaterialFilePath(String encryptLuksSecretMaterialFilePath) { + this.encryptLuksSecretMaterialFilePath = encryptLuksSecretMaterialFilePath; + } public String getTemplatePathInCache() { return templatePathInCache; @@ -329,6 +384,7 @@ public static class CreateVolumeWithBackingCmd extends AgentCommand { public String installPath; public String volumeUuid; public long virtualSize; + public String encryptLuksSecretMaterialFilePath; } public static class CreateVolumeWithBackingRsp extends AgentResponse { @@ -420,6 +476,7 @@ public void setPaths(List paths) { public static class CreateTemplateFromVolumeCmd extends AgentCommand implements HasThreadContext{ private String installPath; private String volumePath; + private String encryptLuksSecretMaterialFilePath; public String getInstallPath() { return installPath; @@ -436,6 +493,14 @@ public String getVolumePath() { public void setVolumePath(String rootVolumePath) { this.volumePath = rootVolumePath; } + + public String getEncryptLuksSecretMaterialFilePath() { + return encryptLuksSecretMaterialFilePath; + } + + public void setEncryptLuksSecretMaterialFilePath(String encryptLuksSecretMaterialFilePath) { + this.encryptLuksSecretMaterialFilePath = encryptLuksSecretMaterialFilePath; + } } public static class CreateTemplateFromVolumeRsp extends AgentResponse { @@ -494,6 +559,7 @@ public void setSize(long size) { public static class RevertVolumeFromSnapshotCmd extends AgentCommand { private String snapshotInstallPath; + private String encryptLuksSecretMaterialFilePath; public String getSnapshotInstallPath() { return snapshotInstallPath; @@ -502,11 +568,20 @@ public String getSnapshotInstallPath() { public void setSnapshotInstallPath(String snapshotInstallPath) { this.snapshotInstallPath = snapshotInstallPath; } + + public String getEncryptLuksSecretMaterialFilePath() { + return encryptLuksSecretMaterialFilePath; + } + + public void setEncryptLuksSecretMaterialFilePath(String encryptLuksSecretMaterialFilePath) { + this.encryptLuksSecretMaterialFilePath = encryptLuksSecretMaterialFilePath; + } } public static class ReinitImageCmd extends AgentCommand { private String imagePath; private String volumePath; + private String encryptLuksSecretMaterialFilePath; public String getImagePath() { return imagePath; @@ -523,6 +598,14 @@ public String getVolumePath() { public void setVolumePath(String volumePath) { this.volumePath = volumePath; } + + public String getEncryptLuksSecretMaterialFilePath() { + return encryptLuksSecretMaterialFilePath; + } + + public void setEncryptLuksSecretMaterialFilePath(String encryptLuksSecretMaterialFilePath) { + this.encryptLuksSecretMaterialFilePath = encryptLuksSecretMaterialFilePath; + } } public static class ReinitImageRsp extends AgentResponse { @@ -566,6 +649,7 @@ public static class MergeSnapshotCmd extends AgentCommand implements HasThreadCo private String volumeUuid; private String snapshotInstallPath; private String workspaceInstallPath; + private String encryptLuksSecretMaterialFilePath; public String getVolumeUuid() { return volumeUuid; @@ -590,6 +674,14 @@ public String getWorkspaceInstallPath() { public void setWorkspaceInstallPath(String workspaceInstallPath) { this.workspaceInstallPath = workspaceInstallPath; } + + public String getEncryptLuksSecretMaterialFilePath() { + return encryptLuksSecretMaterialFilePath; + } + + public void setEncryptLuksSecretMaterialFilePath(String encryptLuksSecretMaterialFilePath) { + this.encryptLuksSecretMaterialFilePath = encryptLuksSecretMaterialFilePath; + } } public static class MergeSnapshotRsp extends AgentResponse { @@ -668,6 +760,7 @@ public static class OfflineMergeSnapshotCmd extends AgentCommand implements HasT private String srcPath; private String destPath; private boolean fullRebase; + private String encryptedDek; public boolean isFullRebase() { return fullRebase; @@ -692,6 +785,14 @@ public String getDestPath() { public void setDestPath(String destPath) { this.destPath = destPath; } + + public String getEncryptedDek() { + return encryptedDek; + } + + public void setEncryptedDek(String encryptedDek) { + this.encryptedDek = encryptedDek; + } } public static class OfflineMergeSnapshotRsp extends AgentResponse { @@ -710,6 +811,7 @@ public static class OfflineCommitSnapshotCmd extends AgentCommand implements Has public String top; public String base; public List topChildrenInstallPathInDb = new ArrayList<>(); + public String encryptedDek; } public static class OfflineCommitSnapshotRsp extends AgentResponse { @@ -997,6 +1099,8 @@ public static class PrefixRebaseBackingFilesRsp extends LocalStorageKvmBackend.A public static final String SCAN_VM_METADATA_PATH = "/localstorage/vm/metadata/scan"; public static final String CLEANUP_VM_METADATA_PATH = "/localstorage/vm/metadata/cleanup"; public static final String PREFIX_REBASE_BACKING_FILES_PATH = "/localstorage/snapshot/prefixrebasebackingfiles"; + public static final String ENCRYPT_VOLUME_BITS_PATH = "/localstorage/volume/encryptinplace"; + public static final String CONVERT_VOLUME_ENCRYPTION_PATH = "/localstorage/volume/convertencryption"; public LocalStorageKvmBackend() { } @@ -1309,7 +1413,7 @@ private void createTemporaryEmptyVolume(InstantiateTemporaryVolumeOnPrimaryStora } private void createEmptyVolume(InstantiateVolumeOnPrimaryStorageMsg msg, ReturnValueCompletion completion) { - createEmptyVolume(msg.getVolume(), msg.getDestHost().getUuid(), new ReturnValueCompletion(completion) { + createEmptyVolume(msg.getVolume(), msg.getDestHost().getUuid(), msg.getVolumeLuksAgentSpec(), new ReturnValueCompletion(completion) { @Override public void success(VolumeStats returnValue) { InstantiateVolumeOnPrimaryStorageReply r = new InstantiateVolumeOnPrimaryStorageReply(); @@ -1332,11 +1436,11 @@ public void fail(ErrorCode errorCode) { }); } - public void createEmptyVolume(final VolumeInventory volume, final String hostUuid, final ReturnValueCompletion completion) { - createEmptyVolumeWithBackingFile(volume, hostUuid, null, completion); + public void createEmptyVolume(final VolumeInventory volume, final String hostUuid, final VolumeLuksAgentSpec volumeLuksAgentSpec, final ReturnValueCompletion completion) { + createEmptyVolumeWithBackingFile(volume, hostUuid, null, volumeLuksAgentSpec, completion); } - public void createEmptyVolumeWithBackingFile(final VolumeInventory volume, final String hostUuid, final String backingFile, final ReturnValueCompletion completion) { + public void createEmptyVolumeWithBackingFile(final VolumeInventory volume, final String hostUuid, final String backingFile, final VolumeLuksAgentSpec volumeLuksAgentSpec, final ReturnValueCompletion completion) { final CreateEmptyVolumeCmd cmd = new CreateEmptyVolumeCmd(); cmd.setAccountUuid(acntMgr.getOwnerAccountUuidOfResource(volume.getUuid())); if (volume.getInstallPath() != null && !volume.getInstallPath().equals("")) { @@ -1358,6 +1462,9 @@ public void createEmptyVolumeWithBackingFile(final VolumeInventory volume, final cmd.setSize(volume.getSize()); cmd.setVolumeUuid(volume.getUuid()); cmd.setBackingFile(backingFile); + if (volumeLuksAgentSpec != null && volumeLuksAgentSpec.isComplete()) { + cmd.setEncryptLuksSecretMaterialFilePath(volumeLuksAgentSpec.getEncryptLuksSecretMaterialFilePath()); + } httpCall(CREATE_EMPTY_VOLUME_PATH, hostUuid, cmd, CreateEmptyVolumeRsp.class, new ReturnValueCompletion(completion) { @Override @@ -1396,6 +1503,7 @@ class ImageCache { String hostUuid; String primaryStorageInstallPath; String backupStorageInstallPath; + String encryptLuksSecretMaterialFilePath; void download(final ReturnValueCompletion completion) { DebugUtils.Assert(image != null, "image cannot be null"); @@ -1488,6 +1596,9 @@ private void downloadFromVolume(FlowTrigger trigger) { CreateTemplateFromVolumeCmd cmd = new CreateTemplateFromVolumeCmd(); cmd.setInstallPath(primaryStorageInstallPath); cmd.setVolumePath(volumeResourceInstallPath); + if (StringUtils.isNotBlank(encryptLuksSecretMaterialFilePath)) { + cmd.setEncryptLuksSecretMaterialFilePath(encryptLuksSecretMaterialFilePath); + } httpCall(CREATE_TEMPLATE_FROM_VOLUME, hostUuid, cmd, false, CreateTemplateFromVolumeRsp.class, @@ -1707,7 +1818,7 @@ private void createRootVolume(final InstantiateRootVolumeFromTemplateOnPrimarySt final ImageInventory image = ispec.getInventory(); if (!ImageMediaType.RootVolumeTemplate.toString().equals(image.getMediaType())) { - createEmptyVolume(msg.getVolume(), msg.getDestHost().getUuid(), new ReturnValueCompletion(completion) { + createEmptyVolume(msg.getVolume(), msg.getDestHost().getUuid(), msg.getVolumeLuksAgentSpec(), new ReturnValueCompletion(completion) { @Override public void success(VolumeStats returnValue) { InstantiateVolumeOnPrimaryStorageReply r = new InstantiateVolumeOnPrimaryStorageReply(); @@ -1778,6 +1889,11 @@ public void run(final FlowTrigger trigger, Map data) { cmd.setVirtualSize(volume.getSize()); } + VolumeLuksAgentSpec volumeLuksAgentSpec = msg.getVolumeLuksAgentSpec(); + if (volumeLuksAgentSpec != null && volumeLuksAgentSpec.isComplete()) { + cmd.setEncryptLuksSecretMaterialFilePath(volumeLuksAgentSpec.getEncryptLuksSecretMaterialFilePath()); + } + httpCall(CREATE_VOLUME_FROM_CACHE_PATH, hostUuid, cmd, CreateVolumeFromCacheRsp.class, new ReturnValueCompletion(trigger) { @Override public void success(CreateVolumeFromCacheRsp returnValue) { @@ -2131,6 +2247,10 @@ void handle(RevertVolumeFromSnapshotOnPrimaryStorageMsg msg, String hostUuid, fi VolumeSnapshotInventory sp = msg.getSnapshot(); RevertVolumeFromSnapshotCmd cmd = new RevertVolumeFromSnapshotCmd(); cmd.setSnapshotInstallPath(sp.getPrimaryStorageInstallPath()); + if (Boolean.TRUE.equals(msg.getVolume().getEncrypted())) { + cmd.setEncryptLuksSecretMaterialFilePath( + volumeEncryptedSecretHelper.prepareLuksSecretMaterialFileOnHost(hostUuid, msg.getVolume().getUuid())); + } httpCall(REVERT_SNAPSHOT_PATH, hostUuid, cmd, RevertVolumeFromSnapshotRsp.class, new ReturnValueCompletion(completion) { @Override @@ -2170,6 +2290,15 @@ public void run(FlowTrigger trigger, Map data) { } cmd.imagePath = makeCachedImageInstallUrlFromImageUuidForTemplate(msg.getVolume().getRootImageUuid()); cmd.volumePath = makeRootVolumeInstallUrl(msg.getVolume()); + if (Boolean.TRUE.equals(msg.getVolume().getEncrypted())) { + String secretMaterialFilePath = prepareVolumeSecretMaterialPath(hostUuid, msg.getVolume()); + if (StringUtils.isBlank(secretMaterialFilePath)) { + completion.fail(operr("cannot prepare LUKS secret for encrypted volume[uuid:%s] reimage on host[uuid:%s]", + msg.getVolume().getUuid(), hostUuid)); + return; + } + cmd.setEncryptLuksSecretMaterialFilePath(secretMaterialFilePath); + } httpCall(REINIT_IMAGE_PATH, hostUuid, cmd, ReinitImageRsp.class, new ReturnValueCompletion(completion) { @Override @@ -2246,6 +2375,11 @@ private void createNormalVolumeFromSnapshot(VolumeSnapshotInventory sp, String v cmd.setVolumeUuid(sp.getVolumeUuid()); cmd.setSnapshotInstallPath(sp.getPrimaryStorageInstallPath()); cmd.setWorkspaceInstallPath(installPath); + Boolean encrypted = Q.New(VolumeVO.class).eq(VolumeVO_.uuid, volumeUuid).select(VolumeVO_.encrypted).findValue(); + if (Boolean.TRUE.equals(encrypted)) { + cmd.setEncryptLuksSecretMaterialFilePath( + volumeEncryptedSecretHelper.prepareLuksSecretMaterialFileOnHost(hostUuid, volumeUuid)); + } httpCall(MERGE_SNAPSHOT_PATH, hostUuid, cmd, MergeSnapshotRsp.class, new ReturnValueCompletion(completion) { @Override @@ -2271,6 +2405,11 @@ private void createIncrementalVolumeFromSnapshot(VolumeSnapshotInventory sp, Str cmd.volumeUuid = volumeUuid; cmd.installPath = installPath; cmd.templatePathInCache = sp.getPrimaryStorageInstallPath(); + Boolean encrypted = Q.New(VolumeVO.class).eq(VolumeVO_.uuid, volumeUuid).select(VolumeVO_.encrypted).findValue(); + if (Boolean.TRUE.equals(encrypted)) { + cmd.encryptLuksSecretMaterialFilePath = + volumeEncryptedSecretHelper.prepareLuksSecretMaterialFileOnHost(hostUuid, volumeUuid); + } Long volumeSize = Q.New(VolumeVO.class).eq(VolumeVO_.uuid, volumeUuid).select(VolumeVO_.size).findValue(); if (volumeSize != null && volumeSize != 0) { @@ -2325,6 +2464,7 @@ void stream(VolumeSnapshotInventory from, VolumeInventory to, boolean fullRebase cmd.setFullRebase(fullRebase); cmd.setSrcPath(sp.getPrimaryStorageInstallPath()); cmd.setDestPath(volume.getInstallPath()); + cmd.setEncryptedDek(prepareVolumeEncryptedDek(hostUuid, volume, Boolean.TRUE.equals(volume.getEncrypted()))); httpCall(OFFLINE_MERGE_PATH, hostUuid, cmd, OfflineMergeSnapshotRsp.class, new ReturnValueCompletion(completion) { @Override @@ -2359,7 +2499,7 @@ public void run(MessageReply reply) { @Override void handle(LocalStorageCreateEmptyVolumeMsg msg, final ReturnValueCompletion completion) { - createEmptyVolumeWithBackingFile(msg.getVolume(), msg.getHostUuid(), msg.getBackingFile(), new ReturnValueCompletion(completion) { + createEmptyVolumeWithBackingFile(msg.getVolume(), msg.getHostUuid(), msg.getBackingFile(), msg.getVolumeLuksAgentSpec(), new ReturnValueCompletion(completion) { @Override public void success(VolumeStats returnValue) { LocalStorageCreateEmptyVolumeReply reply = new LocalStorageCreateEmptyVolumeReply(); @@ -3417,6 +3557,14 @@ void handle(CreateImageCacheFromVolumeSnapshotOnPrimaryStorageMsg msg, ReturnVal cache.hostUuid = ref.getHostUuid(); cache.image = msg.getImageInventory(); cache.volumeResourceInstallPath = msg.getVolumeSnapshot().getPrimaryStorageInstallPath(); + VolumeLuksAgentSpec luksSpec = snapshotEncryptionHelper.prepareTemporarySnapshotImageSecretMaterial( + ref.getHostUuid(), + msg.getVolumeSnapshot().getUuid(), + msg.getImageInventory().getUuid(), + msg.getEncrypted()); + if (luksSpec != null && luksSpec.isComplete()) { + cache.encryptLuksSecretMaterialFilePath = luksSpec.getEncryptLuksSecretMaterialFilePath(); + } cache.download(new ReturnValueCompletion(completion) { @Override public void success(ImageCacheInventory inv) { @@ -3821,6 +3969,14 @@ private String makeInitializedFilePath() { return String.format("%s/%s-initialized-file", self.getMountPath(), self.getUuid()); } + protected String prepareVolumeSecretMaterialPath(String hostUuid, VolumeInventory volume) { + if (volume == null || !Boolean.TRUE.equals(volume.getEncrypted())) { + return null; + } + + return volumeEncryptedSecretHelper.prepareLuksSecretMaterialFileOnHost(hostUuid, volume.getUuid()); + } + @Override void handle(CommitVolumeSnapshotOnPrimaryStorageMsg msg, String hostUuid, final ReturnValueCompletion completion) { CommitVolumeSnapshotOnPrimaryStorageReply reply = new CommitVolumeSnapshotOnPrimaryStorageReply(); @@ -3828,6 +3984,7 @@ void handle(CommitVolumeSnapshotOnPrimaryStorageMsg msg, String hostUuid, final cmd.top = msg.getSrcSnapshot().getPrimaryStorageInstallPath(); cmd.base = msg.getDstSnapshot().getPrimaryStorageInstallPath(); cmd.topChildrenInstallPathInDb = msg.getSrcChildrenInstallPathInDb(); + cmd.encryptedDek = prepareVolumeEncryptedDek(hostUuid, msg.getVolume(), Boolean.TRUE.equals(msg.getVolume().getEncrypted())); httpCall(OFFLINE_COMMIT_PATH, hostUuid, cmd, OfflineCommitSnapshotRsp.class, new ReturnValueCompletion(completion) { @Override public void success(OfflineCommitSnapshotRsp returnValue) { @@ -3850,6 +4007,7 @@ void handle(PullVolumeSnapshotOnPrimaryStorageMsg msg, String hostUuid, final Re cmd.srcPath = msg.getSrcSnapshotParentPath(); cmd.destPath = msg.getDstSnapshot().getPrimaryStorageInstallPath(); cmd.fullRebase = cmd.srcPath == null; + cmd.setEncryptedDek(prepareVolumeEncryptedDek(hostUuid, msg.getVolume(), Boolean.TRUE.equals(msg.getVolume().getEncrypted()))); httpCall(OFFLINE_MERGE_PATH, hostUuid, cmd, OfflineMergeSnapshotRsp.class, new ReturnValueCompletion(completion) { @Override public void success(OfflineMergeSnapshotRsp rsp) { @@ -3976,4 +4134,57 @@ public void fail(ErrorCode errorCode) { } }); } + + @Override + void handle(EncryptVolumeBitsOnPrimaryStorageMsg msg, ReturnValueCompletion completion) { + EncryptVolumeBitsCmd cmd = new EncryptVolumeBitsCmd(); + cmd.installPath = msg.getInstallPath(); + cmd.encryptLuksSecretMaterialFilePath = msg.getEncryptLuksSecretMaterialFilePath(); + + httpCall(ENCRYPT_VOLUME_BITS_PATH, msg.getHostUuid(), cmd, EncryptVolumeBitsRsp.class, new ReturnValueCompletion(completion) { + @Override + public void success(EncryptVolumeBitsRsp rsp) { + completion.success(new EncryptVolumeBitsOnPrimaryStorageReply()); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(operr("failed to encrypt volume[uuid:%s] bits at path[%s] on host[uuid:%s]: %s", + msg.getVolumeUuid(), msg.getInstallPath(), msg.getHostUuid(), errorCode)); + } + }); + } + + @Override + void handle(ConvertVolumeEncryptionOnPrimaryStorageMsg msg, String hostUuid, ReturnValueCompletion completion) { + ConvertVolumeEncryptionCmd cmd = new ConvertVolumeEncryptionCmd(); + cmd.volumeUuid = msg.getVolume().getUuid(); + cmd.targetEncrypted = msg.isTargetEncrypted(); + cmd.items = msg.getItems(); + boolean needSecret = Boolean.TRUE.equals(msg.getVolume().getEncrypted()) || msg.isTargetEncrypted(); + cmd.encryptedDek = prepareVolumeEncryptedDek(hostUuid, msg.getVolume(), needSecret); + + httpCall(CONVERT_VOLUME_ENCRYPTION_PATH, hostUuid, cmd, ConvertVolumeEncryptionRsp.class, + new ReturnValueCompletion(completion) { + @Override + public void success(ConvertVolumeEncryptionRsp rsp) { + ConvertVolumeEncryptionOnPrimaryStorageReply reply = new ConvertVolumeEncryptionOnPrimaryStorageReply(); + reply.setActualSizes(rsp.actualSizes); + completion.success(reply); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(operr("failed to convert volume[uuid:%s] encryption on local storage host[uuid:%s]: %s", + msg.getVolume().getUuid(), hostUuid, errorCode)); + } + }); + } + + private String prepareVolumeEncryptedDek(String hostUuid, VolumeInventory volume, boolean required) { + if (!required) { + return null; + } + return volumeEncryptedSecretHelper.prepareLuksEnvelopeDekOnHost(hostUuid, volume.getUuid()); + } } diff --git a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmFactory.java b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmFactory.java index 8549d799eeb..337b57715fb 100755 --- a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmFactory.java +++ b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmFactory.java @@ -195,7 +195,7 @@ public void beforeTakeSnapshot(KVMHostInventory host, TakeSnapshotOnHypervisorMs LocalStorageHypervisorBackend bkd = getHypervisorBackend(primaryStorageVO); String backingFile = cmd.isOnline() ? cmd.getVolumeInstallPath() : null; - bkd.createEmptyVolumeWithBackingFile(inv, msg.getHostUuid(), backingFile, new ReturnValueCompletion(completion) { + bkd.createEmptyVolumeWithBackingFile(inv, msg.getHostUuid(), backingFile, null, new ReturnValueCompletion(completion) { @Override public void success(VolumeStats returnValue) { completion.success(); diff --git a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmMigrateVmFlow.java b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmMigrateVmFlow.java index d250876db08..1e5bb865b4a 100755 --- a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmMigrateVmFlow.java +++ b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmMigrateVmFlow.java @@ -7,6 +7,7 @@ import org.zstack.core.cloudbus.CloudBus; import org.zstack.core.cloudbus.CloudBusCallBack; import org.zstack.core.cloudbus.CloudBusListCallBack; +import org.zstack.core.componentloader.PluginRegistry; import org.zstack.core.db.DatabaseFacade; import org.zstack.core.db.Q; import org.zstack.core.db.SQLBatch; @@ -36,9 +37,11 @@ import org.zstack.header.storage.snapshot.VolumeSnapshotTree; import org.zstack.header.storage.snapshot.VolumeSnapshotVO; import org.zstack.header.storage.snapshot.VolumeSnapshotVO_; +import org.zstack.header.storage.migration.KvmMigrateVmWithStorageExtensionPoint; import org.zstack.header.vm.VmInstanceConstant; import org.zstack.header.vm.VmInstanceSpec; import org.zstack.header.volume.VolumeInventory; +import org.zstack.header.volume.VolumeLuksAgentSpec; import org.zstack.header.volume.VolumeType; import org.zstack.header.volume.VolumeVO; import org.zstack.kvm.KVMHostAsyncHttpCallMsg; @@ -74,6 +77,8 @@ public class LocalStorageKvmMigrateVmFlow extends NoRollbackFlow { private ThreadFacade thdf; @Autowired private LocalStorageFactory localStorageFactory; + @Autowired + private PluginRegistry pluginRgty; public static final String VERIFY_SNAPSHOT_CHAIN_PATH = "/localstorage/snapshot/verifychain"; public static final String REBASE_SNAPSHOT_BACKING_FILES_PATH = "/localstorage/snapshot/rebasebackingfiles"; @@ -91,6 +96,8 @@ public static class VerifySnapshotChainCmd extends LocalStorageKvmBackend.AgentC public static class RebaseSnapshotBackingFilesCmd extends LocalStorageKvmBackend.AgentCommand { public List snapshots; + @NoLogging + public String encryptedDek; } public static class CopyBitsFromRemoteCmd extends LocalStorageKvmBackend.AgentCommand implements HasThreadContext, Serializable { @@ -112,6 +119,28 @@ static class BackingImage { String md5; } + private VolumeLuksAgentSpec prepareVolumeLuksAgentSpec(String hostUuid, VolumeInventory volume) { + for (KvmMigrateVmWithStorageExtensionPoint ext : pluginRgty.getExtensionList(KvmMigrateVmWithStorageExtensionPoint.class)) { + VolumeLuksAgentSpec spec = ext.prepareVolumeLuksAgentSpec(hostUuid, volume); + if (spec != null && spec.isComplete()) { + return spec; + } + } + + return null; + } + + private String prepareVolumeEncryptedDek(String hostUuid, VolumeInventory volume) { + for (KvmMigrateVmWithStorageExtensionPoint ext : pluginRgty.getExtensionList(KvmMigrateVmWithStorageExtensionPoint.class)) { + String encryptedDek = ext.prepareVolumeEncryptedDek(hostUuid, volume); + if (encryptedDek != null && !encryptedDek.isEmpty()) { + return encryptedDek; + } + } + + return null; + } + @Override public void run(final FlowTrigger next, Map data) { final VmInstanceSpec spec = (VmInstanceSpec) data.get(VmInstanceConstant.Params.VmInstanceSpec.toString()); @@ -590,6 +619,7 @@ public void run(final FlowTrigger trigger, Map data) { LocalStorageCreateEmptyVolumeMsg msg = new LocalStorageCreateEmptyVolumeMsg(); msg.setHostUuid(dstHostUuid); msg.setVolume(arg); + msg.setVolumeLuksAgentSpec(prepareVolumeLuksAgentSpec(dstHostUuid, arg)); if (VolumeType.Root.toString().equals(arg.getType())) { msg.setBackingFile(backingImage.path); @@ -1144,6 +1174,9 @@ public void run(final FlowTrigger trigger, Map data) { cmd.setInstallUrl(p.volume.getInstallPath()); cmd.setSize(p.volume.getSize()); cmd.setVolumeUuid(p.volume.getUuid()); + if (Boolean.TRUE.equals(p.volume.getEncrypted())) { + cmd.setEncryptedDek(prepareVolumeEncryptedDek(dstHostUuid, p.volume)); + } if (p.latest == null){ // volume has been reimage @@ -1198,6 +1231,9 @@ public void run(MessageReply reply) { public void run(final FlowTrigger trigger, Map data) { RebaseSnapshotBackingFilesCmd cmd = new RebaseSnapshotBackingFilesCmd(); cmd.snapshots = snapshotTOs; + if (Boolean.TRUE.equals(p.volume.getEncrypted())) { + cmd.encryptedDek = prepareVolumeEncryptedDek(dstHostUuid, p.volume); + } callKvmHost(dstHostUuid, p.volume.getPrimaryStorageUuid(), REBASE_SNAPSHOT_BACKING_FILES_PATH, cmd, AgentResponse.class, new ReturnValueCompletion(trigger) { @Override public void success(AgentResponse returnValue) { @@ -1260,6 +1296,7 @@ public void run(final FlowTrigger trigger, Map data) { LocalStorageCreateEmptyVolumeMsg msg = new LocalStorageCreateEmptyVolumeMsg(); msg.setHostUuid(dstHostUuid); msg.setVolume(vol); + msg.setVolumeLuksAgentSpec(prepareVolumeLuksAgentSpec(dstHostUuid, vol)); if (VolumeType.Root.toString().equals(vol.getType())) { msg.setBackingFile(image.path); diff --git a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsDownloadImageToCacheJob.java b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsDownloadImageToCacheJob.java index ad9a9c275d4..7c7736e29aa 100755 --- a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsDownloadImageToCacheJob.java +++ b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsDownloadImageToCacheJob.java @@ -47,6 +47,10 @@ public class NfsDownloadImageToCacheJob implements Job { private PrimaryStorageInventory primaryStorage; @JobContext private String volumeResourceInstallPath; + @JobContext + private String volumeSnapshotUuid; + @JobContext + private Boolean encrypted; @Autowired private NfsPrimaryStorageFactory nfsFactory; @@ -175,7 +179,8 @@ public void fail(ErrorCode errorCode) { } }; - bkd.createImageCacheFromVolumeResource(primaryStorage, volumeResourceInstallPath, image.getInventory(), compl); + bkd.createImageCacheFromVolumeResource(primaryStorage, volumeResourceInstallPath, + image.getInventory(), volumeSnapshotUuid, encrypted, compl); } private void downloadFromBackupStorage(FlowTrigger trigger) { @@ -319,4 +324,12 @@ public void setPrimaryStorage(PrimaryStorageInventory primaryStorage) { public void setVolumeResourceInstallPath(String volumeResourceInstallPath) { this.volumeResourceInstallPath = volumeResourceInstallPath; } + + public void setVolumeSnapshotUuid(String volumeSnapshotUuid) { + this.volumeSnapshotUuid = volumeSnapshotUuid; + } + + public void setEncrypted(Boolean encrypted) { + this.encrypted = encrypted; + } } diff --git a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorage.java b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorage.java index 6b0142e5a63..73f728f2a7a 100755 --- a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorage.java +++ b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorage.java @@ -136,11 +136,63 @@ protected void handleLocalMessage(Message msg) { handle((PullVolumeSnapshotOnPrimaryStorageMsg) msg); } else if (msg instanceof RebaseVolumeBackingFileOnPrimaryStorageMsg) { handle((RebaseVolumeBackingFileOnPrimaryStorageMsg) msg); + } else if (msg instanceof EncryptVolumeBitsOnPrimaryStorageMsg) { + handle((EncryptVolumeBitsOnPrimaryStorageMsg) msg); + } else if (msg instanceof ConvertVolumeEncryptionOnPrimaryStorageMsg) { + handle((ConvertVolumeEncryptionOnPrimaryStorageMsg) msg); } else { super.handleLocalMessage(msg); } } + protected void handle(ConvertVolumeEncryptionOnPrimaryStorageMsg msg) { + ConvertVolumeEncryptionOnPrimaryStorageReply reply = new ConvertVolumeEncryptionOnPrimaryStorageReply(); + NfsPrimaryStorageBackend backend = getUsableBackend(); + if (backend == null) { + reply.setError(operr("the NFS primary storage[uuid:%s, name:%s] cannot find any usable host to change volume[uuid:%s] encryption", + self.getUuid(), self.getName(), msg.getVolume().getUuid())); + bus.reply(msg, reply); + return; + } + + backend.handle(getSelfInventory(), msg, new ReturnValueCompletion(msg) { + @Override + public void success(ConvertVolumeEncryptionOnPrimaryStorageReply returnValue) { + bus.reply(msg, returnValue); + } + + @Override + public void fail(ErrorCode errorCode) { + reply.setError(errorCode); + bus.reply(msg, reply); + } + }); + } + + private void handle(EncryptVolumeBitsOnPrimaryStorageMsg msg) { + NfsPrimaryStorageBackend backend = getUsableBackend(); + if (backend == null) { + EncryptVolumeBitsOnPrimaryStorageReply reply = new EncryptVolumeBitsOnPrimaryStorageReply(); + reply.setError(operr("the NFS primary storage[uuid:%s, name:%s] cannot find any usable host to" + + " encrypt volume[uuid:%s] bits", self.getUuid(), self.getName(), msg.getVolumeUuid())); + bus.reply(msg, reply); + return; + } + backend.handle(msg, new ReturnValueCompletion(msg) { + @Override + public void success(EncryptVolumeBitsOnPrimaryStorageReply reply) { + bus.reply(msg, reply); + } + + @Override + public void fail(ErrorCode errorCode) { + EncryptVolumeBitsOnPrimaryStorageReply reply = new EncryptVolumeBitsOnPrimaryStorageReply(); + reply.setError(errorCode); + bus.reply(msg, reply); + } + }); + } + protected void updateMountPoint(String newUrl, Completion completion) { String oldUrl = self.getUrl(); @@ -812,7 +864,7 @@ public void fail(ErrorCode errorCode) { @Override public void run(final FlowTrigger trigger, Map data) { NfsPrimaryStorageBackend backend = factory.getHypervisorBackend(nfsMgr.findHypervisorTypeByImageFormatAndPrimaryStorageUuid(image.getFormat(), self.getUuid())); - backend.createVolumeFromImageCache(primaryStorage, image, imageCache, volume, new ReturnValueCompletion(trigger) { + backend.createVolumeFromImageCache(primaryStorage, image, imageCache, volume, msg.getVolumeLuksAgentSpec(), new ReturnValueCompletion(trigger) { @Override public void success(VolumeStats returnValue) { volumeInstallPath = returnValue.getInstallPath(); @@ -916,7 +968,7 @@ private void createEmptyVolume(final InstantiateVolumeOnPrimaryStorageMsg msg) { VolumeInventory vol = msg.getVolume(); final InstantiateVolumeOnPrimaryStorageReply reply = new InstantiateVolumeOnPrimaryStorageReply(); - backend.instantiateVolume(PrimaryStorageInventory.valueOf(self), msg.getDestHost(), vol, new ReturnValueCompletion(msg) { + backend.instantiateVolume(PrimaryStorageInventory.valueOf(self), msg.getDestHost(), vol, msg.getVolumeLuksAgentSpec(), new ReturnValueCompletion(msg) { @Override public void success(VolumeInventory returnValue) { reply.setVolume(returnValue); @@ -1037,6 +1089,8 @@ protected void handle(CreateImageCacheFromVolumeSnapshotOnPrimaryStorageMsg msg) job.setPrimaryStorage(getSelfInventory()); job.setImage(spec); job.setVolumeResourceInstallPath(msg.getVolumeSnapshot().getPrimaryStorageInstallPath()); + job.setVolumeSnapshotUuid(msg.getVolumeSnapshot().getUuid()); + job.setEncrypted(msg.getEncrypted()); jobf.execute(NfsPrimaryStorageKvmHelper.makeDownloadImageJobName(msg.getImageInventory(), job.getPrimaryStorage()), NfsPrimaryStorageKvmHelper.makeJobOwnerName(job.getPrimaryStorage()), job, diff --git a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageBackend.java b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageBackend.java index 35a240a6221..1ee3c3aa530 100755 --- a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageBackend.java +++ b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageBackend.java @@ -13,6 +13,7 @@ import org.zstack.header.volume.BatchSyncVolumeSizeOnPrimaryStorageMsg; import org.zstack.header.volume.BatchSyncVolumeSizeOnPrimaryStorageReply; import org.zstack.header.volume.VolumeInventory; +import org.zstack.header.volume.VolumeLuksAgentSpec; import org.zstack.storage.primary.EstimateVolumeTemplateSizeOnPrimaryStorageMsg; import org.zstack.storage.primary.EstimateVolumeTemplateSizeOnPrimaryStorageReply; import org.zstack.storage.primary.PrimaryStorageBase.PhysicalCapacityUsage; @@ -54,6 +55,8 @@ public interface NfsPrimaryStorageBackend { void handle(PrimaryStorageInventory inv, GetVolumeSnapshotEncryptedOnPrimaryStorageMsg msg, ReturnValueCompletion completion); + void handle(PrimaryStorageInventory inv, ConvertVolumeEncryptionOnPrimaryStorageMsg msg, ReturnValueCompletion completion); + void getPhysicalCapacity(PrimaryStorageInventory inv, ReturnValueCompletion completion); void checkIsBitsExisting(PrimaryStorageInventory inv, String installPath, ReturnValueCompletion completion); @@ -64,7 +67,8 @@ public interface NfsPrimaryStorageBackend { void createMemoryVolume(PrimaryStorageInventory pinv, VolumeInventory volume, ReturnValueCompletion completion); - void instantiateVolume(PrimaryStorageInventory pinv, HostInventory hostInventory, VolumeInventory volume, ReturnValueCompletion complete); + void instantiateVolume(PrimaryStorageInventory pinv, HostInventory hostInventory, VolumeInventory volume, + VolumeLuksAgentSpec volumeLuksAgentSpec, ReturnValueCompletion complete); void deleteImageCache(ImageCacheInventory imageCache); @@ -77,9 +81,15 @@ public interface NfsPrimaryStorageBackend { void resetRootVolumeFromImage(VolumeInventory vol, HostInventory host, ReturnValueCompletion completion); - void createVolumeFromImageCache(PrimaryStorageInventory primaryStorage, ImageInventory image, ImageCacheInventory imageCache, VolumeInventory volume, ReturnValueCompletion completion); + void createVolumeFromImageCache(PrimaryStorageInventory primaryStorage, ImageInventory image, ImageCacheInventory imageCache, + VolumeInventory volume, VolumeLuksAgentSpec volumeLuksAgentSpec, + ReturnValueCompletion completion); + + void handle(EncryptVolumeBitsOnPrimaryStorageMsg msg, ReturnValueCompletion completion); - void createImageCacheFromVolumeResource(PrimaryStorageInventory primaryStorage, String volumeResourceInstallPath, ImageInventory image, ReturnValueCompletion completion); + void createImageCacheFromVolumeResource(PrimaryStorageInventory primaryStorage, String volumeResourceInstallPath, + ImageInventory image, String snapshotUuid, Boolean encrypted, + ReturnValueCompletion completion); void createTemplateFromVolume(PrimaryStorageInventory primaryStorage, VolumeInventory volume, ImageInventory image, ReturnValueCompletion completion); diff --git a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackend.java b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackend.java index 445497e0871..37e81997ba9 100755 --- a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackend.java +++ b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackend.java @@ -50,6 +50,8 @@ import org.zstack.storage.primary.*; import org.zstack.storage.primary.PrimaryStorageBase.PhysicalCapacityUsage; import org.zstack.storage.primary.nfs.NfsPrimaryStorageKVMBackendCommands.*; +import org.zstack.storage.encrypt.VolumeEncryptedSecretHelper; +import org.zstack.storage.encrypt.VolumeSnapshotEncryptionHelper; import org.zstack.storage.volume.VolumeErrors; import org.zstack.storage.volume.VolumeSystemTags; import org.zstack.utils.CollectionUtils; @@ -98,6 +100,10 @@ public class NfsPrimaryStorageKVMBackend implements NfsPrimaryStorageBackend, private StorageTrash trash; @Autowired protected ApiTimeoutManager timeoutManager; + @Autowired + private VolumeSnapshotEncryptionHelper snapshotEncryptionHelper; + @Autowired + private VolumeEncryptedSecretHelper volumeEncryptedSecretHelper; public static final String MOUNT_PRIMARY_STORAGE_PATH = "/nfsprimarystorage/mount"; public static final String UNMOUNT_PRIMARY_STORAGE_PATH = "/nfsprimarystorage/unmount"; @@ -132,6 +138,8 @@ public class NfsPrimaryStorageKVMBackend implements NfsPrimaryStorageBackend, public static final String CANCEL_DOWNLOAD_BITS_FROM_KVM_HOST_PATH = "/nfsprimarystorage/kvmhost/download/cancel"; public static final String GET_DOWNLOAD_BITS_FROM_KVM_HOST_PROGRESS_PATH = "/nfsprimarystorage/kvmhost/download/progress"; public static final String CREATE_VOLUME_FROM_TEMPLATE_PATH = "/nfsprimarystorage/sftp/createvolumefromtemplate"; + public static final String ENCRYPT_VOLUME_BITS_PATH = "/nfsprimarystorage/volume/encryptinplace"; + public static final String CONVERT_VOLUME_ENCRYPTION_PATH = "/nfsprimarystorage/volume/convertencryption"; public static final String GET_QCOW2_HASH_VALUE_PATH = "/nfsprimarystorage/getqcow2hash"; public static final String WRITE_VM_METADATA_PATH = "/nfsprimarystorage/vm/metadata/write"; @@ -643,6 +651,11 @@ private void createNormalVolumeFromSnapshot(VolumeSnapshotInventory sp, String v cmd.setWorkspaceInstallPath(volPath); cmd.setUuid(inv.getUuid()); cmd.setVolumeUuid(sp.getVolumeUuid()); + Boolean encrypted = Q.New(VolumeVO.class).eq(VolumeVO_.uuid, volumeUuid).select(VolumeVO_.encrypted).findValue(); + if (Boolean.TRUE.equals(encrypted)) { + cmd.setEncryptLuksSecretMaterialFilePath( + volumeEncryptedSecretHelper.prepareLuksSecretMaterialFileOnHost(host.getUuid(), volumeUuid)); + } new KvmCommandSender(host.getUuid()).send(cmd, MERGE_SNAPSHOT_PATH, new KvmCommandFailureChecker() { @Override @@ -685,6 +698,11 @@ private void createIncrementalVolumeFromSnapshot(VolumeSnapshotInventory sp, Str if (volumeSize != null && volumeSize != 0) { cmd.setVirtualSize(volumeSize); } + Boolean encrypted = Q.New(VolumeVO.class).eq(VolumeVO_.uuid, volumeUuid).select(VolumeVO_.encrypted).findValue(); + if (Boolean.TRUE.equals(encrypted)) { + cmd.setEncryptLuksSecretMaterialFilePath( + volumeEncryptedSecretHelper.prepareLuksSecretMaterialFileOnHost(host.getUuid(), volumeUuid)); + } new KvmCommandSender(host.getUuid()).send(cmd, CREATE_VOLUME_WITH_BACKING_PATH, wrapper -> { CreateVolumeWithBackingRsp rsp = wrapper.getResponse(CreateVolumeWithBackingRsp.class); @@ -1098,7 +1116,8 @@ public List call() { } @Override - public void instantiateVolume(final PrimaryStorageInventory pinv, HostInventory hostInventory, final VolumeInventory volume, final ReturnValueCompletion complete) { + public void instantiateVolume(final PrimaryStorageInventory pinv, HostInventory hostInventory, final VolumeInventory volume, + VolumeLuksAgentSpec volumeLuksAgentSpec, final ReturnValueCompletion complete) { String accounUuid = acntMgr.getOwnerAccountUuidOfResource(volume.getUuid()); final CreateEmptyVolumeCmd cmd = new CreateEmptyVolumeCmd(); @@ -1120,6 +1139,10 @@ public void instantiateVolume(final PrimaryStorageInventory pinv, HostInventory throw new CloudRuntimeException(String.format("unknown volume type %s", volume.getType())); } + if (volumeLuksAgentSpec != null && volumeLuksAgentSpec.isComplete()) { + cmd.setEncryptLuksSecretMaterialFilePath(volumeLuksAgentSpec.getEncryptLuksSecretMaterialFilePath()); + } + if (volume.getType().equals(VolumeType.Memory.toString())) { cmd.setWithoutVolume(true); } @@ -1299,6 +1322,10 @@ public void revertVolumeFromSnapshot(VolumeSnapshotInventory sinv, VolumeInvento RevertVolumeFromSnapshotCmd cmd = new RevertVolumeFromSnapshotCmd(); cmd.setSnapshotInstallPath(sinv.getPrimaryStorageInstallPath()); cmd.setUuid(sinv.getPrimaryStorageUuid()); + if (Boolean.TRUE.equals(vol.getEncrypted())) { + cmd.setEncryptLuksSecretMaterialFilePath( + volumeEncryptedSecretHelper.prepareLuksSecretMaterialFileOnHost(host.getUuid(), vol.getUuid())); + } KVMHostAsyncHttpCallMsg msg = new KVMHostAsyncHttpCallMsg(); msg.setCommand(cmd); @@ -1336,6 +1363,15 @@ public void resetRootVolumeFromImage(final VolumeInventory vol, final HostInvent cmd.setImagePath(NfsPrimaryStorageKvmHelper.makeCachedImageInstallUrlFromImageUuidForTemplate(psInv, vol.getRootImageUuid())); cmd.setVolumePath(NfsPrimaryStorageKvmHelper.makeRootVolumeInstallUrl(psInv, vol)); cmd.setUuid(vol.getPrimaryStorageUuid()); + if (Boolean.TRUE.equals(vol.getEncrypted())) { + String secretMaterialFilePath = prepareVolumeSecretMaterialPath(host.getUuid(), vol); + if (StringUtils.isBlank(secretMaterialFilePath)) { + completion.fail(operr("cannot prepare LUKS secret for encrypted volume[uuid:%s] reimage on host[uuid:%s]", + vol.getUuid(), host.getUuid())); + return; + } + cmd.setEncryptLuksSecretMaterialFilePath(secretMaterialFilePath); + } KVMHostAsyncHttpCallMsg msg = new KVMHostAsyncHttpCallMsg(); msg.setCommand(cmd); @@ -1364,7 +1400,8 @@ public void run(MessageReply reply) { @Override public void createVolumeFromImageCache(final PrimaryStorageInventory primaryStorage, final ImageInventory image, final ImageCacheInventory imageCache, - final VolumeInventory volume, final ReturnValueCompletion completion) { + final VolumeInventory volume, VolumeLuksAgentSpec volumeLuksAgentSpec, + final ReturnValueCompletion completion) { HostInventory host = nfsFactory.getConnectedHostForOperation(primaryStorage).get(0); final String installPath = StringUtils.isNotEmpty(volume.getInstallPath()) ? volume.getInstallPath() : @@ -1380,6 +1417,9 @@ public void createVolumeFromImageCache(final PrimaryStorageInventory primaryStor if (image.getSize() < volume.getSize()) { cmd.setVirtualSize(volume.getSize()); } + if (volumeLuksAgentSpec != null && volumeLuksAgentSpec.isComplete()) { + cmd.setEncryptLuksSecretMaterialFilePath(volumeLuksAgentSpec.getEncryptLuksSecretMaterialFilePath()); + } KVMHostAsyncHttpCallMsg msg = new KVMHostAsyncHttpCallMsg(); msg.setCommand(cmd); @@ -1410,24 +1450,36 @@ public void run(MessageReply reply) { } @Override - public void createImageCacheFromVolumeResource(PrimaryStorageInventory primaryStorage, String volumeResource, ImageInventory image, ReturnValueCompletion completion) { + public void createImageCacheFromVolumeResource(PrimaryStorageInventory primaryStorage, String volumeResource, + ImageInventory image, String snapshotUuid, Boolean encrypted, + ReturnValueCompletion completion) { final String installPath = NfsPrimaryStorageKvmHelper.makeCachedImageInstallUrl(primaryStorage, image); - doCreateTemplateFromVolume(installPath, primaryStorage, volumeResource, image, completion); + HostInventory host = nfsFactory.getConnectedHostForOperation(primaryStorage).get(0); + VolumeLuksAgentSpec volumeLuksAgentSpec = snapshotEncryptionHelper.prepareTemporarySnapshotImageSecretMaterial( + host.getUuid(), snapshotUuid, image.getUuid(), encrypted); + doCreateTemplateFromVolume(installPath, primaryStorage, volumeResource, image, volumeLuksAgentSpec, host, completion); } @Override public void createTemplateFromVolume(final PrimaryStorageInventory primaryStorage, final VolumeInventory volume, final ImageInventory image, final ReturnValueCompletion completion) { final String installPath = NfsPrimaryStorageKvmHelper.makeTemplateFromVolumeInWorkspacePath(primaryStorage, image.getUuid()); - doCreateTemplateFromVolume(installPath, primaryStorage, volume.getInstallPath(), image, completion); + doCreateTemplateFromVolume(installPath, primaryStorage, volume.getInstallPath(), image, null, null, completion); } - private void doCreateTemplateFromVolume(final String installPath, final PrimaryStorageInventory primaryStorage, final String volumeResourceInstallPath, final ImageInventory image, final ReturnValueCompletion completion) { - final HostInventory destHost = nfsFactory.getConnectedHostForOperation(primaryStorage).get(0); + private void doCreateTemplateFromVolume(final String installPath, final PrimaryStorageInventory primaryStorage, + final String volumeResourceInstallPath, final ImageInventory image, + VolumeLuksAgentSpec volumeLuksAgentSpec, HostInventory selectedHost, + final ReturnValueCompletion completion) { + final HostInventory destHost = selectedHost != null ? selectedHost : + nfsFactory.getConnectedHostForOperation(primaryStorage).get(0); CreateTemplateFromVolumeCmd cmd = new CreateTemplateFromVolumeCmd(); cmd.setInstallPath(installPath); cmd.setVolumePath(volumeResourceInstallPath); cmd.setUuid(primaryStorage.getUuid()); + if (volumeLuksAgentSpec != null && volumeLuksAgentSpec.isComplete()) { + cmd.setEncryptLuksSecretMaterialFilePath(volumeLuksAgentSpec.getEncryptLuksSecretMaterialFilePath()); + } KVMHostAsyncHttpCallMsg msg = new KVMHostAsyncHttpCallMsg(); msg.setCommand(cmd); @@ -1495,6 +1547,7 @@ public void mergeSnapshotToVolume(final PrimaryStorageInventory pinv, VolumeSnap cmd.setSrcPath(snapshot != null ? snapshot.getPrimaryStorageInstallPath() : null); cmd.setDestPath(volume.getInstallPath()); cmd.setUuid(pinv.getUuid()); + cmd.setEncryptedDek(prepareVolumeEncryptedDek(host.getUuid(), volume, Boolean.TRUE.equals(volume.getEncrypted()))); KVMHostAsyncHttpCallMsg msg = new KVMHostAsyncHttpCallMsg(); msg.setCommand(cmd); @@ -1918,6 +1971,20 @@ public void beforeTakeSnapshot(KVMHostInventory host, TakeSnapshotOnHypervisorMs scmd.setVolumeUuid(cmd.getVolumeUuid()); scmd.setInstallUrl(cmd.getInstallPath()); scmd.setBackingFile(cmd.getVolumeInstallPath()); + String secretMaterialFilePath = null; + VolumeVO volume = dbf.findByUuid(msg.getVolume().getUuid(), VolumeVO.class); + if (volume != null && volume.isEncrypted()) { + secretMaterialFilePath = + volumeEncryptedSecretHelper.prepareLuksSecretMaterialFileOnHost(host.getUuid(), volume.getUuid()); + if (cmd.getVolume() != null) { + cmd.getVolume().setLuksSecretUuid(volumeEncryptedSecretHelper.resolveOrDefineSecretForVolume( + host.getUuid(), volume.getVmInstanceUuid(), volume.getUuid())); + } + } + if (StringUtils.isNotBlank(secretMaterialFilePath)) { + scmd.setEncryptLuksSecretMaterialFilePath(secretMaterialFilePath); + scmd.setVirtualSize(msg.getVolume().getSize()); + } KVMHostAsyncHttpCallMsg smsg = new KVMHostAsyncHttpCallMsg(); smsg.setCommand(scmd); @@ -1992,6 +2059,14 @@ public void fail(ErrorCode errorCode) { }); } + private String prepareVolumeSecretMaterialPath(String hostUuid, VolumeInventory volume) { + if (volume == null || !Boolean.TRUE.equals(volume.getEncrypted())) { + return null; + } + + return volumeEncryptedSecretHelper.prepareLuksSecretMaterialFileOnHost(hostUuid, volume.getUuid()); + } + @Override public void pullSnapshot(PullVolumeSnapshotOnPrimaryStorageMsg msg, String hostUuid, ReturnValueCompletion completion) { PullVolumeSnapshotOnPrimaryStorageReply reply = new PullVolumeSnapshotOnPrimaryStorageReply(); @@ -2001,6 +2076,7 @@ public void pullSnapshot(PullVolumeSnapshotOnPrimaryStorageMsg msg, String hostU cmd.setDestPath(msg.getDstSnapshot().getPrimaryStorageInstallPath()); cmd.setUuid(msg.getVolume().getPrimaryStorageUuid()); cmd.setFullRebase(cmd.getSrcPath() == null); + cmd.setEncryptedDek(prepareVolumeEncryptedDek(hostUuid, msg.getVolume(), Boolean.TRUE.equals(msg.getVolume().getEncrypted()))); KVMHostAsyncHttpCallMsg kmsg = new KVMHostAsyncHttpCallMsg(); kmsg.setCommand(cmd); @@ -2034,6 +2110,7 @@ public void commitSnapshot(CommitVolumeSnapshotOnPrimaryStorageMsg msg, String h cmd.base = msg.getDstSnapshot().getPrimaryStorageInstallPath(); cmd.topChildrenInstallPathInDb = msg.getSrcChildrenInstallPathInDb(); cmd.setUuid(msg.getVolume().getPrimaryStorageUuid()); + cmd.encryptedDek = prepareVolumeEncryptedDek(hostUuid, msg.getVolume(), Boolean.TRUE.equals(msg.getVolume().getEncrypted())); KVMHostAsyncHttpCallMsg kmsg = new KVMHostAsyncHttpCallMsg(); kmsg.setCommand(cmd); @@ -2230,4 +2307,82 @@ public void run(MessageReply reply) { } }); } + + @Override + public void handle(EncryptVolumeBitsOnPrimaryStorageMsg msg, ReturnValueCompletion completion) { + EncryptVolumeBitsCmd cmd = new EncryptVolumeBitsCmd(); + cmd.setUuid(msg.getPrimaryStorageUuid()); + cmd.installPath = msg.getInstallPath(); + cmd.encryptLuksSecretMaterialFilePath = msg.getEncryptLuksSecretMaterialFilePath(); + + KVMHostAsyncHttpCallMsg hmsg = new KVMHostAsyncHttpCallMsg(); + hmsg.setCommand(cmd); + hmsg.setPath(ENCRYPT_VOLUME_BITS_PATH); + hmsg.setHostUuid(msg.getHostUuid()); + bus.makeTargetServiceIdByResourceUuid(hmsg, HostConstant.SERVICE_ID, msg.getHostUuid()); + bus.send(hmsg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + completion.fail(reply.getError()); + return; + } + + EncryptVolumeBitsRsp rsp = ((KVMHostAsyncHttpCallReply) reply).toResponse(EncryptVolumeBitsRsp.class); + if (!rsp.isSuccess()) { + completion.fail(operr("failed to encrypt volume[uuid:%s] bits at path[%s] on host[uuid:%s]: %s", + msg.getVolumeUuid(), msg.getInstallPath(), msg.getHostUuid(), rsp.getError())); + return; + } + + completion.success(new EncryptVolumeBitsOnPrimaryStorageReply()); + } + }); + } + + @Override + public void handle(PrimaryStorageInventory inv, ConvertVolumeEncryptionOnPrimaryStorageMsg msg, + ReturnValueCompletion completion) { + HostInventory host = nfsFactory.getConnectedHostForOperation(inv).get(0); + ConvertVolumeEncryptionCmd cmd = new ConvertVolumeEncryptionCmd(); + cmd.setUuid(inv.getUuid()); + cmd.volumeUuid = msg.getVolume().getUuid(); + cmd.targetEncrypted = msg.isTargetEncrypted(); + cmd.items = msg.getItems(); + boolean needSecret = Boolean.TRUE.equals(msg.getVolume().getEncrypted()) || msg.isTargetEncrypted(); + cmd.encryptedDek = prepareVolumeEncryptedDek(host.getUuid(), msg.getVolume(), needSecret); + + KVMHostAsyncHttpCallMsg hmsg = new KVMHostAsyncHttpCallMsg(); + hmsg.setCommand(cmd); + hmsg.setPath(CONVERT_VOLUME_ENCRYPTION_PATH); + hmsg.setHostUuid(host.getUuid()); + bus.makeTargetServiceIdByResourceUuid(hmsg, HostConstant.SERVICE_ID, host.getUuid()); + bus.send(hmsg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + completion.fail(reply.getError()); + return; + } + + ConvertVolumeEncryptionRsp rsp = ((KVMHostAsyncHttpCallReply) reply).toResponse(ConvertVolumeEncryptionRsp.class); + if (!rsp.isSuccess()) { + completion.fail(operr("failed to convert volume[uuid:%s] encryption on NFS primary storage[uuid:%s] via host[uuid:%s]: %s", + msg.getVolume().getUuid(), inv.getUuid(), host.getUuid(), rsp.getError())); + return; + } + + ConvertVolumeEncryptionOnPrimaryStorageReply r = new ConvertVolumeEncryptionOnPrimaryStorageReply(); + r.setActualSizes(rsp.actualSizes); + completion.success(r); + } + }); + } + + private String prepareVolumeEncryptedDek(String hostUuid, VolumeInventory volume, boolean required) { + if (!required) { + return null; + } + return volumeEncryptedSecretHelper.prepareLuksEnvelopeDekOnHost(hostUuid, volume.getUuid()); + } } diff --git a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackendCommands.java b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackendCommands.java index 0901af4cadb..5883c9a3ea4 100755 --- a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackendCommands.java +++ b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackendCommands.java @@ -2,8 +2,10 @@ import org.zstack.header.HasThreadContext; import org.zstack.header.core.validation.Validation; -import org.zstack.header.volume.VolumeConstant; +import org.zstack.header.log.NoLogging; import org.zstack.header.storage.primary.VmMetadataScanEntry; +import org.zstack.header.storage.primary.ConvertVolumeEncryptionOnPrimaryStorageMsg; +import org.zstack.header.volume.VolumeConstant; import org.zstack.kvm.KVMAgentCommands; import org.zstack.kvm.KVMAgentCommands.AgentCommand; import org.zstack.kvm.KVMAgentCommands.AgentResponse; @@ -145,6 +147,7 @@ public static class UnmountResponse extends NfsPrimaryStorageAgentResponse { public static class CreateTemplateFromVolumeCmd extends NfsPrimaryStorageAgentCommand implements HasThreadContext{ private String installPath; private String rootVolumePath; + private String encryptLuksSecretMaterialFilePath; public String getInstallPath() { return installPath; @@ -159,6 +162,12 @@ public String getRootVolumePath() { public void setVolumePath(String rootVolumePath) { this.rootVolumePath = rootVolumePath; } + public String getEncryptLuksSecretMaterialFilePath() { + return encryptLuksSecretMaterialFilePath; + } + public void setEncryptLuksSecretMaterialFilePath(String encryptLuksSecretMaterialFilePath) { + this.encryptLuksSecretMaterialFilePath = encryptLuksSecretMaterialFilePath; + } } public static class CreateTemplateFromVolumeRsp extends NfsPrimaryStorageAgentResponse { private long size; @@ -366,6 +375,16 @@ public long getVirtualSize() { public void setVirtualSize(long virtualSize) { this.virtualSize = virtualSize; } + + private String encryptLuksSecretMaterialFilePath; + + public String getEncryptLuksSecretMaterialFilePath() { + return encryptLuksSecretMaterialFilePath; + } + + public void setEncryptLuksSecretMaterialFilePath(String encryptLuksSecretMaterialFilePath) { + this.encryptLuksSecretMaterialFilePath = encryptLuksSecretMaterialFilePath; + } } public static class CreateRootVolumeFromTemplateCmd extends CreateVolumeCmd { @@ -453,6 +472,26 @@ public static class CreateEmptyVolumeResponse extends NfsPrimaryStorageAgentResp public Long size; } + public static class EncryptVolumeBitsCmd extends NfsPrimaryStorageAgentCommand { + public String installPath; + public String encryptLuksSecretMaterialFilePath; + } + + public static class EncryptVolumeBitsRsp extends NfsPrimaryStorageAgentResponse { + } + + public static class ConvertVolumeEncryptionCmd extends NfsPrimaryStorageAgentCommand { + public String volumeUuid; + public boolean targetEncrypted; + public List items; + @NoLogging + public String encryptedDek; + } + + public static class ConvertVolumeEncryptionRsp extends NfsPrimaryStorageAgentResponse { + public Map actualSizes; + } + public static class DeleteCmd extends NfsPrimaryStorageAgentCommand { private boolean folder; private String installPath; @@ -512,6 +551,7 @@ public void setPaths(List paths) { public static class RevertVolumeFromSnapshotCmd extends NfsPrimaryStorageAgentCommand { private String snapshotInstallPath; + private String encryptLuksSecretMaterialFilePath; public String getSnapshotInstallPath() { return snapshotInstallPath; @@ -520,6 +560,14 @@ public String getSnapshotInstallPath() { public void setSnapshotInstallPath(String snapshotInstallPath) { this.snapshotInstallPath = snapshotInstallPath; } + + public String getEncryptLuksSecretMaterialFilePath() { + return encryptLuksSecretMaterialFilePath; + } + + public void setEncryptLuksSecretMaterialFilePath(String encryptLuksSecretMaterialFilePath) { + this.encryptLuksSecretMaterialFilePath = encryptLuksSecretMaterialFilePath; + } } public static class RevertVolumeFromSnapshotResponse extends NfsPrimaryStorageAgentResponse { @@ -549,6 +597,7 @@ public void setSize(long size) { public static class ReInitImageCmd extends NfsPrimaryStorageAgentCommand { private String imagePath; private String volumePath; + private String encryptLuksSecretMaterialFilePath; public String getImagePath() { return imagePath; @@ -565,6 +614,14 @@ public String getVolumePath() { public void setVolumePath(String volumePath) { this.volumePath = volumePath; } + + public String getEncryptLuksSecretMaterialFilePath() { + return encryptLuksSecretMaterialFilePath; + } + + public void setEncryptLuksSecretMaterialFilePath(String encryptLuksSecretMaterialFilePath) { + this.encryptLuksSecretMaterialFilePath = encryptLuksSecretMaterialFilePath; + } } public static class ReInitImageRsp extends NfsPrimaryStorageAgentResponse { @@ -639,6 +696,7 @@ public static class MergeSnapshotCmd extends NfsPrimaryStorageAgentCommand imple private String volumeUuid; private String snapshotInstallPath; private String workspaceInstallPath; + private String encryptLuksSecretMaterialFilePath; public String getVolumeUuid() { return volumeUuid; @@ -663,6 +721,14 @@ public String getWorkspaceInstallPath() { public void setWorkspaceInstallPath(String workspaceInstallPath) { this.workspaceInstallPath = workspaceInstallPath; } + + public String getEncryptLuksSecretMaterialFilePath() { + return encryptLuksSecretMaterialFilePath; + } + + public void setEncryptLuksSecretMaterialFilePath(String encryptLuksSecretMaterialFilePath) { + this.encryptLuksSecretMaterialFilePath = encryptLuksSecretMaterialFilePath; + } } public static class MergeSnapshotResponse extends NfsPrimaryStorageAgentResponse { @@ -765,6 +831,7 @@ public static class OfflineMergeSnapshotCmd extends NfsPrimaryStorageAgentComman private String srcPath; private String destPath; private boolean fullRebase; + private String encryptedDek; public boolean isFullRebase() { return fullRebase; @@ -789,6 +856,14 @@ public String getDestPath() { public void setDestPath(String destPath) { this.destPath = destPath; } + + public String getEncryptedDek() { + return encryptedDek; + } + + public void setEncryptedDek(String encryptedDek) { + this.encryptedDek = encryptedDek; + } } public static class OfflineMergeSnapshotRsp extends NfsPrimaryStorageAgentResponse { @@ -807,6 +882,7 @@ public static class OfflineCommitSnapshotCmd extends NfsPrimaryStorageAgentComma public String top; public String base; public List topChildrenInstallPathInDb = new ArrayList<>(); + public String encryptedDek; } public static class OfflineCommitSnapshotRsp extends NfsPrimaryStorageAgentResponse { diff --git a/sdk/src/main/java/org/zstack/sdk/ChangeVolumeEncryptionAction.java b/sdk/src/main/java/org/zstack/sdk/ChangeVolumeEncryptionAction.java new file mode 100644 index 00000000000..b3d3099eb9c --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/ChangeVolumeEncryptionAction.java @@ -0,0 +1,104 @@ +package org.zstack.sdk; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class ChangeVolumeEncryptionAction extends AbstractAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.ChangeVolumeEncryptionResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s]", error.code, error.description, error.details) + ); + } + + return this; + } + } + + @Param(required = true, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String uuid; + + @Param(required = true, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public boolean encrypted = false; + + @Param(required = false) + public java.util.List systemTags; + + @Param(required = false) + public java.util.List userTags; + + @Param(required = false) + public String sessionId; + + @Param(required = false) + public String accessKeyId; + + @Param(required = false) + public String accessKeySecret; + + @Param(required = false) + public String requestIp; + + @NonAPIParam + public long timeout = -1; + + @NonAPIParam + public long pollingInterval = -1; + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.ChangeVolumeEncryptionResult value = res.getResult(org.zstack.sdk.ChangeVolumeEncryptionResult.class); + ret.value = value == null ? new org.zstack.sdk.ChangeVolumeEncryptionResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "PUT"; + info.path = "/volumes/{uuid}/actions"; + info.needSession = true; + info.needPoll = true; + info.parameterName = "changeVolumeEncryption"; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/ChangeVolumeEncryptionResult.java b/sdk/src/main/java/org/zstack/sdk/ChangeVolumeEncryptionResult.java new file mode 100644 index 00000000000..f25890f0757 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/ChangeVolumeEncryptionResult.java @@ -0,0 +1,14 @@ +package org.zstack.sdk; + +import org.zstack.sdk.VolumeInventory; + +public class ChangeVolumeEncryptionResult { + public VolumeInventory inventory; + public void setInventory(VolumeInventory inventory) { + this.inventory = inventory; + } + public VolumeInventory getInventory() { + return this.inventory; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/CreateVmInstanceAction.java b/sdk/src/main/java/org/zstack/sdk/CreateVmInstanceAction.java index f475d3c4761..65f3790e53b 100644 --- a/sdk/src/main/java/org/zstack/sdk/CreateVmInstanceAction.java +++ b/sdk/src/main/java/org/zstack/sdk/CreateVmInstanceAction.java @@ -117,6 +117,9 @@ public Result throwExceptionIfError() { @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) public java.lang.Boolean virtio; + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.Boolean encrypted; + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) public java.lang.String allocatorStrategy; diff --git a/sdk/src/main/java/org/zstack/sdk/CreateVmInstanceFromVolumeAction.java b/sdk/src/main/java/org/zstack/sdk/CreateVmInstanceFromVolumeAction.java index 666c30f9303..744cdd51dd5 100644 --- a/sdk/src/main/java/org/zstack/sdk/CreateVmInstanceFromVolumeAction.java +++ b/sdk/src/main/java/org/zstack/sdk/CreateVmInstanceFromVolumeAction.java @@ -76,6 +76,9 @@ public Result throwExceptionIfError() { @Param(required = false, validValues = {"InstantStart","CreateStopped"}, nonempty = false, nullElements = false, emptyString = true, noTrim = false) public java.lang.String strategy = "InstantStart"; + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.Boolean encrypted; + @Param(required = false) public java.lang.String resourceUuid; diff --git a/sdk/src/main/java/org/zstack/sdk/CreateVmInstanceFromVolumeSnapshotAction.java b/sdk/src/main/java/org/zstack/sdk/CreateVmInstanceFromVolumeSnapshotAction.java index 1e42ee6fda2..da944ede461 100644 --- a/sdk/src/main/java/org/zstack/sdk/CreateVmInstanceFromVolumeSnapshotAction.java +++ b/sdk/src/main/java/org/zstack/sdk/CreateVmInstanceFromVolumeSnapshotAction.java @@ -79,6 +79,9 @@ public Result throwExceptionIfError() { @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) public java.util.List rootVolumeSystemTags; + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.Boolean encrypted; + @Param(required = false) public java.lang.String resourceUuid; diff --git a/sdk/src/main/java/org/zstack/sdk/CreateVmInstanceFromVolumeSnapshotGroupAction.java b/sdk/src/main/java/org/zstack/sdk/CreateVmInstanceFromVolumeSnapshotGroupAction.java index 7f477d9b0fb..6adf60a3c22 100644 --- a/sdk/src/main/java/org/zstack/sdk/CreateVmInstanceFromVolumeSnapshotGroupAction.java +++ b/sdk/src/main/java/org/zstack/sdk/CreateVmInstanceFromVolumeSnapshotGroupAction.java @@ -79,9 +79,15 @@ public Result throwExceptionIfError() { @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) public java.util.Map dataVolumeSystemTags; + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.util.List volumeSnapshotEncryptions; + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) public java.lang.Boolean resetTpm; + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.Boolean encrypted; + @Param(required = false) public java.lang.String resourceUuid; diff --git a/sdk/src/main/java/org/zstack/sdk/GetTrashOnPrimaryStorageAction.java b/sdk/src/main/java/org/zstack/sdk/GetTrashOnPrimaryStorageAction.java index 8ad8fb10332..764229becda 100644 --- a/sdk/src/main/java/org/zstack/sdk/GetTrashOnPrimaryStorageAction.java +++ b/sdk/src/main/java/org/zstack/sdk/GetTrashOnPrimaryStorageAction.java @@ -34,7 +34,7 @@ public Result throwExceptionIfError() { @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) public java.lang.String resourceType; - @Param(required = false, validValues = {"MigrateVolume","MigrateVolumeSnapshot","RevertVolume","VolumeSnapshot"}, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + @Param(required = false, validValues = {"MigrateVolume","MigrateVolumeSnapshot","ConvertVolumeEncryption","ConvertVolumeSnapshotEncryption","RevertVolume","VolumeSnapshot","ReimageVolume"}, nonempty = false, nullElements = false, emptyString = true, noTrim = false) public java.lang.String trashType; @Param(required = false) diff --git a/sdk/src/main/java/org/zstack/sdk/VmInstanceInventory.java b/sdk/src/main/java/org/zstack/sdk/VmInstanceInventory.java index 611d16b2ab7..2974f9c33e9 100644 --- a/sdk/src/main/java/org/zstack/sdk/VmInstanceInventory.java +++ b/sdk/src/main/java/org/zstack/sdk/VmInstanceInventory.java @@ -180,6 +180,30 @@ public java.sql.Timestamp getLastOpDate() { return this.lastOpDate; } + public java.lang.Boolean encrypted; + public void setEncrypted(java.lang.Boolean encrypted) { + this.encrypted = encrypted; + } + public java.lang.Boolean getEncrypted() { + return this.encrypted; + } + + public java.lang.Boolean hasEncryptionResource; + public void setHasEncryptionResource(java.lang.Boolean hasEncryptionResource) { + this.hasEncryptionResource = hasEncryptionResource; + } + public java.lang.Boolean getHasEncryptionResource() { + return this.hasEncryptionResource; + } + + public java.sql.Timestamp keyLastOpDate; + public void setKeyLastOpDate(java.sql.Timestamp keyLastOpDate) { + this.keyLastOpDate = keyLastOpDate; + } + public java.sql.Timestamp getKeyLastOpDate() { + return this.keyLastOpDate; + } + public java.lang.String state; public void setState(java.lang.String state) { this.state = state; diff --git a/sdk/src/main/java/org/zstack/sdk/VolumeInventory.java b/sdk/src/main/java/org/zstack/sdk/VolumeInventory.java index da7652e9ca4..badf2b7954e 100644 --- a/sdk/src/main/java/org/zstack/sdk/VolumeInventory.java +++ b/sdk/src/main/java/org/zstack/sdk/VolumeInventory.java @@ -188,4 +188,12 @@ public java.lang.String getProtocol() { return this.protocol; } + public java.lang.Boolean encrypted; + public void setEncrypted(java.lang.Boolean encrypted) { + this.encrypted = encrypted; + } + public java.lang.Boolean getEncrypted() { + return this.encrypted; + } + } diff --git a/sdk/src/main/java/org/zstack/sdk/VolumeSnapshotInventory.java b/sdk/src/main/java/org/zstack/sdk/VolumeSnapshotInventory.java index e025505f0e5..7f5f88e8bf9 100644 --- a/sdk/src/main/java/org/zstack/sdk/VolumeSnapshotInventory.java +++ b/sdk/src/main/java/org/zstack/sdk/VolumeSnapshotInventory.java @@ -108,6 +108,14 @@ public java.lang.Long getSize() { return this.size; } + public java.lang.Boolean encrypted; + public void setEncrypted(java.lang.Boolean encrypted) { + this.encrypted = encrypted; + } + public java.lang.Boolean getEncrypted() { + return this.encrypted; + } + public int distance; public void setDistance(int distance) { this.distance = distance; diff --git a/sdk/src/main/java/org/zstack/sdk/VolumeTO.java b/sdk/src/main/java/org/zstack/sdk/VolumeTO.java index e81815484a8..d350b386023 100644 --- a/sdk/src/main/java/org/zstack/sdk/VolumeTO.java +++ b/sdk/src/main/java/org/zstack/sdk/VolumeTO.java @@ -164,4 +164,12 @@ public int getControllerIndex() { return this.controllerIndex; } + public java.lang.String luksSecretUuid; + public void setLuksSecretUuid(java.lang.String luksSecretUuid) { + this.luksSecretUuid = luksSecretUuid; + } + public java.lang.String getLuksSecretUuid() { + return this.luksSecretUuid; + } + } diff --git a/storage/pom.xml b/storage/pom.xml old mode 100755 new mode 100644 index 678ec872184..17d8ee34d52 --- a/storage/pom.xml +++ b/storage/pom.xml @@ -3,7 +3,7 @@ zstack org.zstack - 5.0.0 + 5.0.0 .. storage @@ -54,6 +54,11 @@ longjob ${project.version} + + org.zstack + kvm + ${project.version} + diff --git a/storage/src/main/java/org/zstack/storage/encrypt/DummyVolumeEncryptedResourceKeyBackend.java b/storage/src/main/java/org/zstack/storage/encrypt/DummyVolumeEncryptedResourceKeyBackend.java new file mode 100644 index 00000000000..77166fa0393 --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/DummyVolumeEncryptedResourceKeyBackend.java @@ -0,0 +1,103 @@ +package org.zstack.storage.encrypt; + +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +/** + * OSS / no-premium-crypto: no-op volume key-provider persistence, same role as + * {@link org.zstack.compute.vm.devices.DummyTpmEncryptedResourceKeyBackend}. + */ +public class DummyVolumeEncryptedResourceKeyBackend implements VolumeEncryptedResourceKeyBackend { + private static final CLogger logger = Utils.getLogger(DummyVolumeEncryptedResourceKeyBackend.class); + + @Override + public void attachKeyProviderToVolume(String volumeUuid, String keyProviderUuid) { + logger.debug(String.format("ignore attach key provider to volume[uuid:%s] keyProviderUuid:%s", + volumeUuid, keyProviderUuid)); + } + + @Override + public void detachKeyProviderFromVolume(String volumeUuid) { + logger.debug(String.format("ignore detach key provider from volume[uuid:%s]", volumeUuid)); + } + + @Override + public void detachKeyProviderFromSnapshot(String snapshotUuid) { + logger.debug(String.format("ignore detach key provider from snapshot[uuid:%s]", snapshotUuid)); + } + + @Override + public void detachKeyProviderFromTemporarySnapshotImage(String imageUuid) { + logger.debug(String.format("ignore detach key provider from temporary snapshot image[uuid:%s]", imageUuid)); + } + + @Override + public String findKeyProviderUuidByVolume(String volumeUuid) { + return null; + } + + @Override + public boolean checkVolumeKeyProviderAttached(String volumeUuid) { + return false; + } + + @Override + public boolean checkSnapshotKeyProviderAttached(String snapshotUuid) { + return false; + } + + @Override + public boolean checkTemporarySnapshotImageKeyProviderAttached(String imageUuid) { + return false; + } + + @Override + public void copyVolumeKeyToSnapshot(String volumeUuid, String snapshotUuid) { + logger.debug(String.format("ignore copy volume[uuid:%s] key to snapshot[uuid:%s]", volumeUuid, snapshotUuid)); + } + + @Override + public void copySnapshotKeyToVolume(String snapshotUuid, String volumeUuid) { + logger.debug(String.format("ignore copy snapshot[uuid:%s] key to volume[uuid:%s]", snapshotUuid, volumeUuid)); + } + + @Override + public void copyVolumeKeyToVolume(String srcVolumeUuid, String dstVolumeUuid) { + logger.debug(String.format("ignore copy volume[uuid:%s] key to volume[uuid:%s]", srcVolumeUuid, dstVolumeUuid)); + } + + @Override + public void copySnapshotKeyToTemporarySnapshotImage(String snapshotUuid, String imageUuid) { + logger.debug(String.format("ignore copy snapshot[uuid:%s] key to temporary snapshot image[uuid:%s]", snapshotUuid, imageUuid)); + } + + @Override + public void copyTemporarySnapshotImageKeyToVolume(String imageUuid, String volumeUuid) { + logger.debug(String.format("ignore copy temporary snapshot image[uuid:%s] key to volume[uuid:%s]", imageUuid, volumeUuid)); + } + + @Override + public String defaultKeyProviderUuid() { + return null; + } + + @Override + public String findKeyProviderUuidBySnapshot(String snapshotUuid) { + return null; + } + + @Override + public String findKeyProviderUuidByTemporarySnapshotImage(String imageUuid) { + return null; + } + + @Override + public Integer findKeyVersionByVolume(String volumeUuid) { + return null; + } + + @Override + public Integer findKeyVersionBySnapshot(String snapshotUuid) { + return null; + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/SnapshotGroupRevertVolumeEncryptionHelper.java b/storage/src/main/java/org/zstack/storage/encrypt/SnapshotGroupRevertVolumeEncryptionHelper.java new file mode 100644 index 00000000000..edffb520a7d --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/SnapshotGroupRevertVolumeEncryptionHelper.java @@ -0,0 +1,110 @@ +package org.zstack.storage.encrypt; + +import org.springframework.beans.factory.annotation.Autowire; +import org.springframework.beans.factory.annotation.Configurable; +import org.zstack.core.Platform; +import org.zstack.core.db.Q; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.storage.snapshot.VolumeSnapshotVO; +import org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupRefVO; +import org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupRefVO_; +import org.zstack.header.vm.APICreateVmInstanceFromVolumeSnapshotGroupMsg; +import org.zstack.header.vm.CreateVmInstanceMsg; +import org.zstack.header.volume.CreateDataVolumeFromVolumeSnapshotMsg; +import org.zstack.header.volume.VolumeType; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +@Configurable(preConstruction = true, autowire = Autowire.BY_TYPE) +public class SnapshotGroupRevertVolumeEncryptionHelper { + public ErrorCode validateVolumeSnapshotEncryption(APICreateVmInstanceFromVolumeSnapshotGroupMsg msg, + List effectiveSnapshots) { + Map volumeSnapshotEncryption; + try { + volumeSnapshotEncryption = getVolumeSnapshotEncryption(msg); + } catch (IllegalArgumentException e) { + return Platform.operr(e.getMessage()); + } + if (volumeSnapshotEncryption == null || volumeSnapshotEncryption.isEmpty()) { + return Platform.operr("volumeSnapshotEncryptions must specify every effective volume snapshot in volume snapshot group[uuid:%s]", + msg.getVolumeSnapshotGroupUuid()); + } + + Set effectiveSnapshotUuids = new HashSet<>(); + for (VolumeSnapshotVO snapshot : effectiveSnapshots) { + effectiveSnapshotUuids.add(snapshot.getUuid()); + } + + Set unexpectedSnapshotUuids = new HashSet<>(volumeSnapshotEncryption.keySet()); + unexpectedSnapshotUuids.removeAll(effectiveSnapshotUuids); + if (!unexpectedSnapshotUuids.isEmpty()) { + return Platform.operr("volumeSnapshotEncryptions contain volume snapshot(s)%s that do not belong to volume snapshot group[uuid:%s]", + unexpectedSnapshotUuids, msg.getVolumeSnapshotGroupUuid()); + } + + Set missingSnapshotUuids = new HashSet<>(effectiveSnapshotUuids); + missingSnapshotUuids.removeAll(volumeSnapshotEncryption.keySet()); + if (!missingSnapshotUuids.isEmpty()) { + return Platform.operr("volumeSnapshotEncryptions miss effective volume snapshot(s)%s in volume snapshot group[uuid:%s]", + missingSnapshotUuids, msg.getVolumeSnapshotGroupUuid()); + } + + for (VolumeSnapshotVO snapshot : effectiveSnapshots) { + if (snapshot.isEncrypted() && !Boolean.TRUE.equals(volumeSnapshotEncryption.get(snapshot.getUuid()))) { + return Platform.operr("volume snapshot[uuid:%s] in volume snapshot group[uuid:%s] is encrypted, cannot create an unencrypted volume from it", + snapshot.getUuid(), msg.getVolumeSnapshotGroupUuid()); + } + } + + return null; + } + + public void setupRootVolumeFromApi(APICreateVmInstanceFromVolumeSnapshotGroupMsg apiMsg, + CreateVmInstanceMsg cmsg) { + Map volumeSnapshotEncryption = getVolumeSnapshotEncryption(apiMsg); + if (volumeSnapshotEncryption == null || volumeSnapshotEncryption.isEmpty()) { + return; + } + if (cmsg.getRootDisk() == null) { + return; + } + + String rootSnapshotUuid = Q.New(VolumeSnapshotGroupRefVO.class).select(VolumeSnapshotGroupRefVO_.volumeSnapshotUuid) + .eq(VolumeSnapshotGroupRefVO_.volumeType, VolumeType.Root.toString()) + .eq(VolumeSnapshotGroupRefVO_.volumeSnapshotGroupUuid, apiMsg.getVolumeSnapshotGroupUuid()) + .findValue(); + cmsg.getRootDisk().setEncrypted(volumeSnapshotEncryption.get(rootSnapshotUuid)); + } + + public void setupDataVolumeFromApi(APICreateVmInstanceFromVolumeSnapshotGroupMsg apiMsg, + VolumeSnapshotVO snapshot, + CreateDataVolumeFromVolumeSnapshotMsg cmsg) { + Map volumeSnapshotEncryption = getVolumeSnapshotEncryption(apiMsg); + cmsg.setEncrypted(volumeSnapshotEncryption == null ? null : volumeSnapshotEncryption.get(snapshot.getUuid())); + } + + private Map getVolumeSnapshotEncryption(APICreateVmInstanceFromVolumeSnapshotGroupMsg msg) { + List volumeSnapshotEncryptions = + msg.getVolumeSnapshotEncryptions(); + if (volumeSnapshotEncryptions == null || volumeSnapshotEncryptions.isEmpty()) { + return null; + } + + Map ret = new HashMap<>(); + for (APICreateVmInstanceFromVolumeSnapshotGroupMsg.VolumeSnapshotEncryption volumeSnapshotEncryption : + volumeSnapshotEncryptions) { + if (volumeSnapshotEncryption.getVolumeSnapshotUuid() == null || volumeSnapshotEncryption.getEncrypted() == null) { + throw new IllegalArgumentException(String.format( + "invalid volumeSnapshotEncryptions item[%s], expected {\"volumeSnapshotUuid\":\"snapshotUuid\",\"encrypted\":true}", + volumeSnapshotEncryption)); + } + ret.put(volumeSnapshotEncryption.getVolumeSnapshotUuid(), volumeSnapshotEncryption.getEncrypted()); + } + + return ret; + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedAttachExtension.java b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedAttachExtension.java new file mode 100644 index 00000000000..59b3c9cea8c --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedAttachExtension.java @@ -0,0 +1,74 @@ +package org.zstack.storage.encrypt; + +import org.springframework.beans.factory.annotation.Autowire; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Configurable; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.vm.VmInstanceInventory; +import org.zstack.header.volume.VolumeInventory; +import org.zstack.kvm.KVMAgentCommands.AttachDataVolumeCmd; +import org.zstack.kvm.KVMAttachVolumeExtensionPoint; +import org.zstack.kvm.KVMHostInventory; +import org.zstack.kvm.VolumeTO; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.util.Map; + +/** + * On hot-attach of a LUKS-encrypted data volume, ensure the per-volume libvirt + * secret exists on the destination host and stamp its UUID onto the + * {@link VolumeTO} so {@code vm_plugin.filebased_volume} (the attach builder + * at {@code vm_plugin.py:3185}) can emit + * {@code }. + * + *

Without this the agent issues a {@code blockdev-add} without + * {@code encrypt.key-secret} and qemu aborts with + * {@code "Parameter 'encrypt.key-secret' is required for cipher"}. + * + *

Lives in the storage module (where the helper sits) and registers via + * {@link KVMAttachVolumeExtensionPoint} (the existing hook KVMHost.attachVolume + * already fires) to avoid creating a storage -> kvm reverse dep. + */ +@Configurable(preConstruction = true, autowire = Autowire.BY_TYPE) +public class VolumeEncryptedAttachExtension implements KVMAttachVolumeExtensionPoint { + + private static final CLogger logger = Utils.getLogger(VolumeEncryptedAttachExtension.class); + + @Autowired + private VolumeEncryptedSecretHelper secretHelper; + + @Override + public VolumeTO convertVolumeIfNeed(KVMHostInventory host, VolumeInventory inventory, VolumeTO to) { + return to; + } + + @Override + public void beforeAttachVolume(KVMHostInventory host, VmInstanceInventory vm, VolumeInventory volume, + AttachDataVolumeCmd cmd, Map data) { + if (!Boolean.TRUE.equals(volume.getEncrypted())) { + return; + } + String hostUuid = host.getUuid(); + String vmUuid = vm.getUuid(); + String volUuid = volume.getUuid(); + String secretUuid = secretHelper.resolveOrDefineSecretForVolume(hostUuid, vmUuid, volUuid); + VolumeTO to = cmd.getVolume(); + if (to != null) { + to.setLuksSecretUuid(secretUuid); + } + logger.debug(String.format( + "LUKS-ATTACH-EXT stamped secret %s onto attach cmd for volume[uuid:%s] on host[uuid:%s]", + secretUuid, volUuid, hostUuid)); + } + + @Override + public void afterAttachVolume(KVMHostInventory host, VmInstanceInventory vm, VolumeInventory volume, + AttachDataVolumeCmd cmd) { + } + + @Override + public void attachVolumeFailed(KVMHostInventory host, VmInstanceInventory vm, VolumeInventory volume, + AttachDataVolumeCmd cmd, ErrorCode err, Map data) { + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedExpungeExtension.java b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedExpungeExtension.java new file mode 100644 index 00000000000..d70e8b51626 --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedExpungeExtension.java @@ -0,0 +1,151 @@ +package org.zstack.storage.encrypt; + +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowire; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Configurable; +import org.zstack.core.db.Q; +import org.zstack.header.vm.VmInstanceVO; +import org.zstack.header.vm.VmInstanceVO_; +import org.zstack.header.volume.VolumeInventory; +import org.zstack.header.volume.VolumeJustBeforeDeleteFromDbExtensionPoint; +import org.zstack.header.volume.VolumeVO; +import org.zstack.storage.volume.VolumeSystemTags; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.util.List; + +/** + * Volume expunge cleanup for LUKS-encrypted volumes. + * + *

Mirrors the {@code VmJustBeforeDeleteFromDb} / {@code VmAfterExpunge} + * pattern used by vTPM in {@code KvmTpmExtensions}, but at the volume layer: + * + *

    + *
  • Trigger: {@link VolumeJustBeforeDeleteFromDbExtensionPoint#volumeJustBeforeDeleteFromDb} + * — runs at the tail of {@code VolumeBase} expunge, after the agent + * has destroyed the on-disk bits, just before the {@code VolumeVO} + * row is removed.
    • + *
  • Not triggered on: VM destroy with {@code Delay} policy (volume + * lives on, parked in recycle bin); VM destroy with {@code KeepVolume} + * (data volume preserved); volume detach without delete. In all these + * cases the encryption metadata must stay so the volume can be + * attached / restored later.
    • + *
+ * + *

Cleanup actions, in order: + *

    + *
  1. Look up {@code keyVersion} + host while the binding is still in DB.
    2. + *
  2. Best-effort delete the libvirt secret on the host (if we can locate + * one — bound only when the volume is attached to a VM whose + * host/lastHost is known).
    3. + *
  3. Delete the {@code EncryptedResourceKeyRefVO} row — this is the + * persistent piece, and the one that absolutely must be cleared so + * a future volume reusing the same uuid doesn't inherit a stale + * binding.
    4. + *
+ * + *

Order matters: keyVersion lives on the ref row, so we must read it + * before {@code detachKeyProviderFromVolume} wipes the row. + */ +@Configurable(preConstruction = true, autowire = Autowire.BY_TYPE) +public class VolumeEncryptedExpungeExtension implements VolumeJustBeforeDeleteFromDbExtensionPoint { + + private static final CLogger logger = Utils.getLogger(VolumeEncryptedExpungeExtension.class); + + @Autowired + private VolumeEncryptedResourceKeyBackend volumeEncryptedResourceKeyBackend; + @Autowired + private VolumeEncryptedSecretHelper secretHelper; + + @Override + public void volumeJustBeforeDeleteFromDb(VolumeInventory inv) { + if (inv == null || !Boolean.TRUE.equals(inv.getEncrypted())) { + return; + } + String volUuid = inv.getUuid(); + + // Snapshot keyVersion BEFORE detaching the binding, otherwise the + // host-side cleanup loses the key it needs to identify the secret. + Integer keyVersion = volumeEncryptedResourceKeyBackend.findKeyVersionByVolume(volUuid); + + String hostUuid = resolveHostUuidFromTag(volUuid); + if (StringUtils.isBlank(hostUuid)) { + String vmUuid = StringUtils.defaultIfBlank(inv.getVmInstanceUuid(), inv.getLastVmInstanceUuid()); + hostUuid = resolveHostUuidFromVm(vmUuid); + } + + // vmUuid passed to the key-agent's DeleteSecret RPC is a contractual + // leftover (the per-volume secret usage name doesn't actually need it + // post Method-D); pass whatever we can reconstruct, or fall back to + // the volume uuid as a stable placeholder so validation passes. + String vmUuidForRpc = StringUtils.defaultIfBlank( + StringUtils.defaultIfBlank(inv.getVmInstanceUuid(), inv.getLastVmInstanceUuid()), + volUuid); + + if (StringUtils.isNotBlank(hostUuid) && keyVersion != null) { + try { + secretHelper.deleteSecretOnHostBestEffort(hostUuid, vmUuidForRpc, volUuid, keyVersion); + } catch (RuntimeException e) { + // helper is best-effort, but guard against unchecked throws + // so we still get to the DB cleanup below. + logger.warn(String.format( + "ignoring failure to delete libvirt LUKS secret for volume[uuid:%s] on host[uuid:%s]: %s", + volUuid, hostUuid, e.getMessage())); + } + } else { + logger.debug(String.format( + "skip host-side libvirt secret cleanup for volume[uuid:%s]:" + + " hostUuid=%s keyVersion=%s", + volUuid, hostUuid, keyVersion)); + } + + try { + volumeEncryptedResourceKeyBackend.detachKeyProviderFromVolume(volUuid); + } catch (RuntimeException e) { + logger.warn(String.format( + "failed to detach EncryptedResourceKeyRefVO for volume[uuid:%s] on expunge: %s", + volUuid, e.getMessage())); + } + } + + /** + * Primary host lookup: read the {@code VOLUME_LIBVIRT_SECRET_HOST} systemtag + * that {@code VolumeEncryptedSecretHelper.defineLibvirtSecretOnHost} stamps + * after every successful libvirt secret define. The tag is non-inherent and + * lives on the {@code VolumeVO} row, so it survives anything short of the + * volume itself being deleted. + */ + private String resolveHostUuidFromTag(String volUuid) { + List tags = VolumeSystemTags.VOLUME_LIBVIRT_SECRET_HOST.getTags(volUuid, VolumeVO.class); + if (tags == null || tags.isEmpty()) { + return null; + } + return VolumeSystemTags.VOLUME_LIBVIRT_SECRET_HOST.getTokenByTag( + tags.get(0), VolumeSystemTags.VOLUME_LIBVIRT_SECRET_HOST_TOKEN); + } + + /** + * Fallback host lookup for volumes created before the + * {@code VOLUME_LIBVIRT_SECRET_HOST} tag mechanism existed. Walks the + * vmInstanceUuid / lastVmInstanceUuid → VmInstanceVO.hostUuid / + * lastHostUuid chain. + */ + private String resolveHostUuidFromVm(String vmUuid) { + if (StringUtils.isBlank(vmUuid)) { + return null; + } + String host = Q.New(VmInstanceVO.class) + .eq(VmInstanceVO_.uuid, vmUuid) + .select(VmInstanceVO_.hostUuid) + .findValue(); + if (StringUtils.isNotBlank(host)) { + return host; + } + return Q.New(VmInstanceVO.class) + .eq(VmInstanceVO_.uuid, vmUuid) + .select(VmInstanceVO_.lastHostUuid) + .findValue(); + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedInitialExtension.java b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedInitialExtension.java new file mode 100644 index 00000000000..24c6b9b78b6 --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedInitialExtension.java @@ -0,0 +1,168 @@ +package org.zstack.storage.encrypt; + +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowire; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Configurable; +import org.zstack.core.db.DatabaseFacade; +import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.header.host.HostInventory; +import org.zstack.header.keyprovider.EncryptedResourceKeyManager; +import org.zstack.header.storage.primary.InstantiateVolumeOnPrimaryStorageMsg; +import org.zstack.header.volume.AfterInstantiateVolumeExtensionPoint; +import org.zstack.header.volume.CreateDataVolumeExtensionPoint; +import org.zstack.header.volume.InstantiateVolumeMsg; +import org.zstack.header.volume.InstantiateTemporaryRootVolumeMsg; +import org.zstack.header.volume.PreInstantiateVolumeExtensionPoint; +import org.zstack.header.volume.VolumeCreateMessage; +import org.zstack.header.volume.VolumeInventory; +import org.zstack.header.volume.VolumeLuksAgentSpec; +import org.zstack.header.volume.VolumeVO; + +import static org.zstack.core.Platform.operr; + +/** + * Encrypted volume instantiate: {@link #preInstantiateVolume} prepares host LUKS secret material file; + * {@link #afterInstantiateVolume} defines the libvirt secret on the host (needs DEK again after async PS step). + */ +@Configurable(preConstruction = true, autowire = Autowire.BY_TYPE) +public class VolumeEncryptedInitialExtension implements PreInstantiateVolumeExtensionPoint, + AfterInstantiateVolumeExtensionPoint, CreateDataVolumeExtensionPoint { + @Autowired + private DatabaseFacade dbf; + @Autowired + private VolumeEncryptedResourceKeyBackend volumeEncryptedResourceKeyBackend; + @Autowired + private VolumeEncryptedSecretHelper secretHelper; + @Autowired + private VolumeSnapshotEncryptionHelper snapshotEncryptionHelper; + + @Override + public void preInstantiateVolume(InstantiateVolumeMsg msg) { + String hostUuid = msg.getHostUuid(); + if (StringUtils.isBlank(hostUuid)) { + return; + } + + VolumeLuksAgentSpec spec = new VolumeLuksAgentSpec(); + msg.setVolumeLuksAgentSpec(spec); + + String volUuid = msg.getVolumeUuid(); + VolumeVO volume = dbf.findByUuid(volUuid, VolumeVO.class); + + if (volume != null && volume.isEncrypted()) { + // Temporary root images created from snapshots own the image cache before + // the final root volume exists. Persisting the cache key on ImageVO lets + // the root volume inherit the same key, so its top layer can match the + // encrypted image cache. + snapshotEncryptionHelper.inheritFromTemporarySnapshotImageKeyIfPossible(volume); + inheritTemporaryRootVolumeKeyFromOrigin(msg, volume); + String kpUuid = volumeEncryptedResourceKeyBackend.findKeyProviderUuidByVolume(volUuid); + if (StringUtils.isBlank(kpUuid)) { + kpUuid = volumeEncryptedResourceKeyBackend.defaultKeyProviderUuid(); + if (StringUtils.isBlank(kpUuid)) { + throw new OperationFailureException(operr( + "encrypted volume[uuid:%s] has no key provider binding and no default key provider configured", + volUuid)); + } + volumeEncryptedResourceKeyBackend.attachKeyProviderToVolume(volUuid, kpUuid); + } + + EncryptedResourceKeyManager.ResourceKeyResult keyResult = secretHelper.materializeDek(volUuid, kpUuid); + String dekBase64 = keyResult.getDekBase64(); + if (StringUtils.isBlank(dekBase64)) { + throw new OperationFailureException(operr( + "encrypted volume[uuid:%s]: key manager returned empty DEK after materialization", + volUuid)); + } + + String secFilePath = secretHelper.ensureLuksSecretFileOnHost(hostUuid, volUuid, dekBase64); + spec.setEncryptLuksSecretMaterialFilePath(secFilePath); + } + } + + private void inheritTemporaryRootVolumeKeyFromOrigin(InstantiateVolumeMsg msg, VolumeVO volume) { + if (!(msg instanceof InstantiateTemporaryRootVolumeMsg) || volume == null || !volume.isEncrypted()) { + return; + } + + String originVolumeUuid = ((InstantiateTemporaryRootVolumeMsg) msg).getOriginVolumeUuid(); + if (StringUtils.isBlank(originVolumeUuid)) { + return; + } + + VolumeVO originVolume = dbf.findByUuid(originVolumeUuid, VolumeVO.class); + if (originVolume == null || !originVolume.isEncrypted()) { + return; + } + + if (!volumeEncryptedResourceKeyBackend.checkVolumeKeyProviderAttached(originVolumeUuid)) { + throw new OperationFailureException(operr( + "encrypted origin root volume[uuid:%s] has no key provider binding for temporary root volume[uuid:%s]", + originVolumeUuid, volume.getUuid())); + } + + volumeEncryptedResourceKeyBackend.copyVolumeKeyToVolume(originVolumeUuid, volume.getUuid()); + } + + @Override + public void afterInstantiateVolume(InstantiateVolumeOnPrimaryStorageMsg msg) { + VolumeInventory volInv = msg.getVolume(); + if (volInv == null || !Boolean.TRUE.equals(volInv.getEncrypted())) { + return; + } + String volUuid = volInv.getUuid(); + VolumeLuksAgentSpec spec = msg.getVolumeLuksAgentSpec(); + if (spec == null || StringUtils.isBlank(spec.getEncryptLuksSecretMaterialFilePath())) { + return; + } + if (StringUtils.isNotBlank(spec.getEncryptSecretUuid())) { + return; + } + HostInventory destHost = msg.getDestHost(); + if (destHost == null || StringUtils.isBlank(destHost.getUuid())) { + return; + } + + VolumeVO volume = dbf.findByUuid(volUuid, VolumeVO.class); + if (volume == null || !volume.isEncrypted()) { + return; + } + // VolumeInventory already carries vmInstanceUuid when present, so we + // skip the extra select(VolumeVO_.vmInstanceUuid) round-trip. + // VmInstantiateOtherDiskFlow's "create empty data volume" path + // (setupCreateVolumeFromDiskSizeFlows) does NOT set + // VolumeVO.vmInstanceUuid before this hook fires — vmInstanceUuid is + // backfilled only after the volume is attached, well after this + // afterInstantiateVolume runs. Without vmUuid we cannot key the + // libvirt secret (SecretHostDefineMsg requires it), so skip the + // early-define here; VolumeEncryptedStartExtension on the start_vm + // path will define the secret then, when vmUuid is known. + String vmInstanceUuid = volInv.getVmInstanceUuid(); + if (StringUtils.isBlank(vmInstanceUuid)) { + return; + } + String kpUuid = volumeEncryptedResourceKeyBackend.findKeyProviderUuidByVolume(volUuid); + String libvirtSecretUuid = secretHelper.defineSecretFromBinding( + destHost.getUuid(), vmInstanceUuid, volUuid, kpUuid); + + spec.setEncryptSecretUuid(libvirtSecretUuid); + } + + @Override + public void preCreateVolume(VolumeCreateMessage msg) { + } + + @Override + public void beforeCreateVolume(VolumeInventory volume) { + } + + @Override + public void afterCreateVolume(VolumeVO volume) { + } + + @Override + public void afterCreateVolume(VolumeVO volume, String snapshotUuid) { + snapshotEncryptionHelper.inheritFromRelatedSnapshotKeyIfPossible(volume, snapshotUuid); + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedMigrateVmExtension.java b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedMigrateVmExtension.java new file mode 100644 index 00000000000..1d739410f5f --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedMigrateVmExtension.java @@ -0,0 +1,157 @@ +package org.zstack.storage.encrypt; + +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowire; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Configurable; +import org.zstack.header.core.Completion; +import org.zstack.header.core.NoErrorCompletion; +import org.zstack.core.db.Q; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.vm.VmInstanceInventory; +import org.zstack.header.vm.VmInstanceMigrateExtensionPoint; +import org.zstack.header.vm.VmMigrationType; +import org.zstack.header.vm.VmPreMigrationExtensionPoint; +import org.zstack.header.volume.VolumeAO_; +import org.zstack.header.volume.VolumeInventory; +import org.zstack.header.volume.VolumeVO; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.zstack.core.Platform.operr; + +/** + * Ensures LUKS secrets are pre-defined on the destination host before live migration, + * and cleaned up on the source host after migration (or on the destination host on failure). + * + * Only handles HostMigration. PrimaryStorageMigration keeps the VM on the same host, + * so the existing libvirt secrets remain valid; VolumeEncryptedStartExtension handles + * the next start_vm on any new host. + */ +@Configurable(preConstruction = true, autowire = Autowire.BY_TYPE) +public class VolumeEncryptedMigrateVmExtension + implements VmPreMigrationExtensionPoint, VmInstanceMigrateExtensionPoint { + + private static final CLogger logger = Utils.getLogger(VolumeEncryptedMigrateVmExtension.class); + + @Autowired + private VolumeEncryptedSecretHelper secretHelper; + @Autowired + private VolumeEncryptedResourceKeyBackend keyBackend; + + @Override + public void preVmMigration(VmInstanceInventory vm, VmMigrationType type, String dstHostUuid, Completion completion) { + if (type != VmMigrationType.HostMigration) { + completion.success(); + return; + } + if (vm == null || StringUtils.isBlank(dstHostUuid)) { + completion.success(); + return; + } + + List encryptedVols = collectEncryptedVolumes(vm); + if (encryptedVols.isEmpty()) { + completion.success(); + return; + } + + String vmUuid = vm.getUuid(); + String srcHostUuid = getSourceHostUuid(vm, vm.getHostUuid()); + if (StringUtils.isBlank(srcHostUuid)) { + completion.success(); + return; + } + + try { + for (VolumeInventory vol : encryptedVols) { + secretHelper.resolveOrDefineSecretForVolumeMigration(srcHostUuid, dstHostUuid, vmUuid, vol.getUuid()); + } + completion.success(); + } catch (Exception e) { + completion.fail(operr("failed to pre-define LUKS secret for encrypted VM[uuid:%s] on host[uuid:%s]: %s", + vmUuid, dstHostUuid, e.getMessage())); + } + } + + @Override + public void afterMigrateVm(VmInstanceInventory inv, String srcHostUuid, NoErrorCompletion completion) { + if (inv == null) { + completion.done(); + return; + } + + String vmUuid = inv.getUuid(); + String sourceHostUuid = getSourceHostUuid(inv, srcHostUuid); + if (StringUtils.isBlank(sourceHostUuid)) { + completion.done(); + return; + } + + String destHostUuid = inv.getHostUuid(); + if (StringUtils.isBlank(destHostUuid) || sourceHostUuid.equals(destHostUuid)) { + completion.done(); + return; + } + + for (VolumeInventory vol : collectEncryptedVolumes(inv)) { + Integer keyVersion = keyBackend.findKeyVersionByVolume(vol.getUuid()); + secretHelper.deleteSecretOnHostBestEffort(sourceHostUuid, vmUuid, vol.getUuid(), keyVersion); + } + completion.done(); + } + + @Override + public void failedToMigrateVm(VmInstanceInventory inv, String destHostUuid, ErrorCode reason, NoErrorCompletion completion) { + if (inv == null || StringUtils.isBlank(destHostUuid)) { + completion.done(); + return; + } + + String vmUuid = inv.getUuid(); + for (VolumeInventory vol : collectEncryptedVolumes(inv)) { + Integer keyVersion = keyBackend.findKeyVersionByVolume(vol.getUuid()); + secretHelper.deleteSecretOnHostBestEffort(destHostUuid, vmUuid, vol.getUuid(), keyVersion); + } + completion.done(); + } + + private String getSourceHostUuid(VmInstanceInventory inv, String srcHostUuid) { + if (StringUtils.isNotBlank(srcHostUuid)) { + return srcHostUuid; + } + return inv == null ? null : inv.getLastHostUuid(); + } + + private List collectEncryptedVolumes(VmInstanceInventory vm) { + List result = new ArrayList<>(); + Set seen = new HashSet<>(); + VolumeInventory root = vm.getRootVolume(); + if (root != null && Boolean.TRUE.equals(root.getEncrypted())) { + result.add(root); + seen.add(root.getUuid()); + } + if (vm.getAllDiskVolumes() != null) { + for (VolumeInventory v : vm.getAllDiskVolumes()) { + if (v != null && Boolean.TRUE.equals(v.getEncrypted()) && seen.add(v.getUuid())) { + result.add(v); + } + } + } + + List dbVolumes = Q.New(VolumeVO.class) + .eq(VolumeAO_.vmInstanceUuid, vm.getUuid()) + .list(); + for (VolumeVO v : dbVolumes) { + if (v.isEncrypted() && seen.add(v.getUuid())) { + result.add(VolumeInventory.valueOf(v)); + } + } + return result; + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedMigrateVmWithStorageExtension.java b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedMigrateVmWithStorageExtension.java new file mode 100644 index 00000000000..ca7a5b7a824 --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedMigrateVmWithStorageExtension.java @@ -0,0 +1,78 @@ +package org.zstack.storage.encrypt; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.host.HostInventory; +import org.zstack.header.host.MigrateVmOnHypervisorMsg; +import org.zstack.header.storage.migration.KvmMigrateVmWithStorageExtensionPoint; +import org.zstack.header.vm.VmInstanceInventory; +import org.zstack.header.volume.VolumeInventory; +import org.zstack.header.volume.VolumeLuksAgentSpec; +import org.zstack.header.volume.VolumeVO; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.zstack.core.Platform.operr; + +public class VolumeEncryptedMigrateVmWithStorageExtension implements KvmMigrateVmWithStorageExtensionPoint { + @Autowired + private VolumeEncryptedSecretHelper volumeEncryptedSecretHelper; + + @Override + public ErrorCode beforeMigrateVmWithStorage(HostInventory dstHost, + VmInstanceInventory vm, + List volumesToMigrate, + Map volumeMappingDict, + MigrateVmOnHypervisorMsg msg) { + Map volumeLuksSecrets = new HashMap<>(); + + try { + for (VolumeVO sourceVolume : volumesToMigrate) { + if (!sourceVolume.isEncrypted()) { + continue; + } + + String targetVolumeUuid = volumeMappingDict.get(sourceVolume.getUuid()); + if (targetVolumeUuid == null || targetVolumeUuid.trim().isEmpty()) { + return operr("missing live migration temporary volume mapping for encrypted source volume[uuid:%s]", + sourceVolume.getUuid()); + } + + volumeLuksSecrets.put(targetVolumeUuid, + volumeEncryptedSecretHelper.resolveOrDefineSecretForVolume( + dstHost.getUuid(), vm.getUuid(), targetVolumeUuid)); + } + } catch (Exception e) { + return operr("failed to prepare LUKS secrets for live storage migration vm[uuid:%s]: %s", + vm.getUuid(), e.getMessage()); + } + + if (!volumeLuksSecrets.isEmpty()) { + msg.setVolumeLuksSecrets(volumeLuksSecrets); + } + return null; + } + + @Override + public VolumeLuksAgentSpec prepareVolumeLuksAgentSpec(String hostUuid, VolumeInventory volume) { + if (volume == null || !Boolean.TRUE.equals(volume.getEncrypted())) { + return null; + } + + VolumeLuksAgentSpec spec = new VolumeLuksAgentSpec(); + spec.setEncryptLuksSecretMaterialFilePath( + volumeEncryptedSecretHelper.prepareLuksSecretMaterialFileOnHost(hostUuid, volume.getUuid())); + return spec; + } + + @Override + public String prepareVolumeEncryptedDek(String hostUuid, VolumeInventory volume) { + if (volume == null || !Boolean.TRUE.equals(volume.getEncrypted())) { + return null; + } + + return volumeEncryptedSecretHelper.prepareLuksEnvelopeDekOnHost(hostUuid, volume.getUuid()); + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedResourceKeyBackend.java b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedResourceKeyBackend.java new file mode 100644 index 00000000000..bed784a9c12 --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedResourceKeyBackend.java @@ -0,0 +1,73 @@ +package org.zstack.storage.encrypt; + +/** + * Handles {@link org.zstack.header.volume.VolumeVO} rows in {@link org.zstack.header.keyprovider.EncryptedResourceKeyRefVO} + * (key provider binding for LUKS volumes), analogous to {@link org.zstack.compute.vm.devices.TpmEncryptedResourceKeyBackend} + * for TPM. + */ +public interface VolumeEncryptedResourceKeyBackend { + + /** + * Link a volume to a key provider (placeholder ref row). Non-async. + */ + void attachKeyProviderToVolume(String volumeUuid, String keyProviderUuid); + + /** + * Remove key-provider binding for the volume. Non-async. + */ + void detachKeyProviderFromVolume(String volumeUuid); + + /** + * Remove key-provider binding for the snapshot. Non-async. + */ + void detachKeyProviderFromSnapshot(String snapshotUuid); + + void detachKeyProviderFromTemporarySnapshotImage(String imageUuid); + + /** + * @return provider uuid or null when not bound / crypto not installed + */ + String findKeyProviderUuidByVolume(String volumeUuid); + + /** + * Whether an {@code EncryptedResourceKeyRefVO} row exists for this volume. + */ + boolean checkVolumeKeyProviderAttached(String volumeUuid); + + boolean checkSnapshotKeyProviderAttached(String snapshotUuid); + + boolean checkTemporarySnapshotImageKeyProviderAttached(String imageUuid); + + void copyVolumeKeyToSnapshot(String volumeUuid, String snapshotUuid); + + void copySnapshotKeyToVolume(String snapshotUuid, String volumeUuid); + + void copyVolumeKeyToVolume(String srcVolumeUuid, String dstVolumeUuid); + + void copySnapshotKeyToTemporarySnapshotImage(String snapshotUuid, String imageUuid); + + void copyTemporarySnapshotImageKeyToVolume(String imageUuid, String volumeUuid); + + /** + * Global default key provider uuid, or null (e.g. NONE / crypto not installed). + */ + String defaultKeyProviderUuid(); + + String findKeyProviderUuidBySnapshot(String snapshotUuid); + + String findKeyProviderUuidByTemporarySnapshotImage(String imageUuid); + + /** + * Current key version (DEK rotation generation) bound to this volume's + * {@code EncryptedResourceKeyRefVO} row, or {@code null} when no row exists + * (e.g. volume not yet bound to a key provider, or crypto not installed). + * + *

Mirrors {@link org.zstack.compute.vm.devices.TpmEncryptedResourceKeyBackend#findKeyVersionByTpm}. + * Used by the start_vm path to derive the libvirt secret identity tuple + * ({@code vmUuid, purpose="volume", keyVersion, usageInstance}) without + * re-materializing the DEK. + */ + Integer findKeyVersionByVolume(String volumeUuid); + + Integer findKeyVersionBySnapshot(String snapshotUuid); +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedSecretHelper.java b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedSecretHelper.java new file mode 100644 index 00000000000..dfc1f3629e7 --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedSecretHelper.java @@ -0,0 +1,478 @@ +package org.zstack.storage.encrypt; + +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowire; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Configurable; +import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.componentloader.PluginRegistry; +import org.zstack.core.db.DatabaseFacade; +import org.zstack.core.db.Q; +import org.zstack.header.core.ReturnValueCompletion; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.header.host.HostConstant; +import org.zstack.header.host.HostKeyIdentityVO; +import org.zstack.header.keyprovider.EncryptedResourceKeyManager; +import org.zstack.header.message.MessageReply; +import org.zstack.header.secret.SecretHostDefineMsg; +import org.zstack.header.secret.SecretHostDefineReply; +import org.zstack.header.secret.SecretHostDeleteMsg; +import org.zstack.header.secret.SecretHostGetMsg; +import org.zstack.header.secret.SecretHostGetReply; +import org.zstack.header.secret.SecretHostEnsureLuksSecretFileMsg; +import org.zstack.header.secret.SecretHostEnsureLuksSecretFileReply; +import org.zstack.header.volume.VolumeVO; +import org.zstack.header.volume.VolumeVO_; +import org.zstack.kvm.HostKeyIdentityHelper; +import org.zstack.kvm.HostSecretEnvelopeCryptoExtensionPoint; +import org.zstack.kvm.KVMAgentCommands; +import org.zstack.kvm.KVMConstant; +import org.zstack.kvm.KVMHostAsyncHttpCallMsg; +import org.zstack.kvm.KVMHostAsyncHttpCallReply; +import org.zstack.storage.volume.VolumeSystemTags; +import org.zstack.tag.SystemTagCreator; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.util.Base64; +import java.util.Collections; +import java.util.List; + +import static org.zstack.core.Platform.operr; + +/** + * Shared helpers for the volume LUKS secret lifecycle on a KVM host: + * + *

    + *
  • {@link #materializeDek} — unseal/get-or-create the DEK for a volume's + * bound key provider. Idempotent; safe to call on every start_vm.
    • + *
  • {@link #defineLibvirtSecretOnHost} — define+set-value the libvirt secret + * on the destination host (RAM-only). Idempotent in key-agent + * (EnsureSecret keyed on {@code vmUuid, purpose, keyVersion, usageInstance}).
    • + *
  • {@link #getSecretOnHost} — ask the host whether a previously-defined + * libvirt secret is still resident; returns {@code null} on miss so + * callers can decide to re-define.
    • + *
+ * + *

Used by the create path ({@link VolumeEncryptedInitialExtension}), the + * start path ({@link VolumeEncryptedStartExtension}) and the hot-attach path + * ({@link VolumeEncryptedAttachExtension}). All cloudbus calls are synchronous + * via {@link CloudBus#call(org.zstack.header.message.NeedReplyMessage)}; + * timeouts are enforced by the underlying KVMHost handlers (HTTP layer has its + * own {@code ENVELOPE_KEY_HTTP_TIMEOUT_SEC}). {@link EncryptedResourceKeyManager#getOrCreateKey} + * is itself a synchronous DB / NKP / KMS call on the management server — we use + * a one-shot capturing {@link ReturnValueCompletion} to read the result without + * pulling in {@code FutureReturnValueCompletion}'s wait/notify and timeout layer. + */ +@Configurable(preConstruction = true, autowire = Autowire.BY_TYPE) +public class VolumeEncryptedSecretHelper { + private static final CLogger logger = Utils.getLogger(VolumeEncryptedSecretHelper.class); + + @Autowired + private CloudBus bus; + @Autowired + private EncryptedResourceKeyManager encryptedResourceKeyManager; + @Autowired + private VolumeEncryptedResourceKeyBackend volumeEncryptedResourceKeyBackend; + @Autowired + private DatabaseFacade dbf; + @Autowired + private PluginRegistry pluginRegistry; + + public EncryptedResourceKeyManager.ResourceKeyResult materializeDek(String volUuid, String kpUuid) { + EncryptedResourceKeyManager.GetOrCreateResourceKeyContext ctx = + new EncryptedResourceKeyManager.GetOrCreateResourceKeyContext(); + ctx.setResourceUuid(volUuid); + ctx.setResourceType(VolumeVO.class.getSimpleName()); + ctx.setKeyProviderUuid(kpUuid); + ctx.setPurpose("instantiate-volume"); + + // getOrCreateKey is synchronous in EncryptedResourceKeyManagerImpl — + // the completion fires on this thread before the call returns. + final EncryptedResourceKeyManager.ResourceKeyResult[] resultRef = + new EncryptedResourceKeyManager.ResourceKeyResult[1]; + final ErrorCode[] errorRef = new ErrorCode[1]; + encryptedResourceKeyManager.getOrCreateKey(ctx, + new ReturnValueCompletion(null) { + @Override + public void success(EncryptedResourceKeyManager.ResourceKeyResult r) { + resultRef[0] = r; + } + + @Override + public void fail(ErrorCode err) { + errorRef[0] = err; + } + }); + if (errorRef[0] != null) { + throw new OperationFailureException(operr( + "failed to materialize encryption key for volume[uuid:%s]", volUuid) + .withCause(errorRef[0])); + } + return resultRef[0]; + } + + public String ensureLuksSecretFileOnHost(String hostUuid, String resourceUuid, String dekBase64) { + SecretHostEnsureLuksSecretFileMsg ensureMsg = new SecretHostEnsureLuksSecretFileMsg(); + ensureMsg.setHostUuid(hostUuid); + ensureMsg.setDekBase64(dekBase64); + bus.makeTargetServiceIdByResourceUuid(ensureMsg, HostConstant.SERVICE_ID, hostUuid); + + MessageReply reply = bus.call(ensureMsg); + if (!reply.isSuccess()) { + throw new OperationFailureException(operr( + "failed to prepare secret material file for encrypted resource[uuid:%s] on host[uuid:%s]", + resourceUuid, hostUuid).withCause(reply.getError())); + } + SecretHostEnsureLuksSecretFileReply r = reply.castReply(); + if (StringUtils.isBlank(r.getSecFilePath())) { + throw new OperationFailureException(operr( + "ensure LUKS secret file on host succeeded but secFilePath is empty, host[uuid:%s]", + hostUuid)); + } + return r.getSecFilePath(); + } + + public String prepareLuksSecretMaterialFileOnHost(String hostUuid, String volumeUuid) { + if (StringUtils.isBlank(hostUuid) || StringUtils.isBlank(volumeUuid)) { + throw new OperationFailureException(operr( + "prepare LUKS secret material file requires non-blank hostUuid and volumeUuid")); + } + + String kpUuid = volumeEncryptedResourceKeyBackend.findKeyProviderUuidByVolume(volumeUuid); + if (StringUtils.isBlank(kpUuid)) { + throw new OperationFailureException(operr( + "volume[uuid:%s] requires LUKS secret material but has no key provider binding", + volumeUuid)); + } + + EncryptedResourceKeyManager.ResourceKeyResult keyResult = materializeDek(volumeUuid, kpUuid); + String dekBase64 = keyResult.getDekBase64(); + if (StringUtils.isBlank(dekBase64)) { + throw new OperationFailureException(operr( + "encrypted volume[uuid:%s]: key manager returned empty DEK for LUKS secret material file", + volumeUuid)); + } + + return ensureLuksSecretFileOnHost(hostUuid, volumeUuid, dekBase64); + } + + public String prepareLuksEnvelopeDekOnHost(String hostUuid, String volumeUuid) { + if (StringUtils.isBlank(hostUuid) || StringUtils.isBlank(volumeUuid)) { + throw new OperationFailureException(operr( + "prepare LUKS envelope DEK requires non-blank hostUuid and volumeUuid")); + } + + String kpUuid = volumeEncryptedResourceKeyBackend.findKeyProviderUuidByVolume(volumeUuid); + if (StringUtils.isBlank(kpUuid)) { + throw new OperationFailureException(operr( + "volume[uuid:%s] requires LUKS secret material but has no key provider binding", + volumeUuid)); + } + + EncryptedResourceKeyManager.ResourceKeyResult keyResult = materializeDek(volumeUuid, kpUuid); + String dekBase64 = keyResult.getDekBase64(); + if (StringUtils.isBlank(dekBase64)) { + throw new OperationFailureException(operr( + "encrypted volume[uuid:%s]: key manager returned empty DEK for LUKS envelope", + volumeUuid)); + } + + HostKeyIdentityVO identity = HostKeyIdentityHelper.getHostKeyIdentity(dbf, hostUuid); + String pubKey = identity != null ? StringUtils.trimToNull(identity.getPublicKey()) : null; + Boolean verifyOk = identity != null ? identity.getVerified() : null; + if (pubKey == null) { + throw new OperationFailureException(operr("no public key for host[uuid:%s], connect/reconnect did not sync key", hostUuid)); + } + + String storedFingerprint = StringUtils.trimToNull(identity.getFingerprint()); + String computed = HostKeyIdentityHelper.fingerprintFromPublicKey(pubKey); + if (storedFingerprint == null || !StringUtils.equals(storedFingerprint, computed)) { + throw new OperationFailureException(operr( + "host[uuid:%s] public key fingerprint mismatch, key may be corrupted or tampered", + hostUuid)); + } + + if (!Boolean.TRUE.equals(verifyOk)) { + throw new OperationFailureException(operr("host[uuid:%s] secret key verify not ok, not synced", hostUuid)); + } + + byte[] dekRaw = null; + byte[] envelope = null; + try { + dekRaw = Base64.getDecoder().decode(dekBase64.trim()); + if (dekRaw.length == 0) { + throw new OperationFailureException(operr("dekBase64 decoded to empty")); + } + if (dekRaw.length > KVMConstant.MAX_DEK_BYTES) { + throw new OperationFailureException(operr("dekBase64 decoded payload is too large")); + } + + byte[] pubKeyBytes = Base64.getDecoder().decode(pubKey); + if (pubKeyBytes.length != 32) { + throw new OperationFailureException(operr("host[uuid:%s] public key must be 32 bytes (X25519)", hostUuid)); + } + + List sealers = + pluginRegistry.getExtensionList(HostSecretEnvelopeCryptoExtensionPoint.class); + if (sealers == null || sealers.isEmpty()) { + throw new OperationFailureException(operr( + "host secret envelope sealer not available (premium crypto module required)")); + } + + envelope = sealers.get(0).seal(pubKeyBytes, dekRaw); + return Base64.getEncoder().encodeToString(envelope); + } catch (IllegalArgumentException e) { + throw new OperationFailureException(operr("invalid base64 while preparing LUKS envelope DEK: %s", e.getMessage())); + } catch (OperationFailureException e) { + throw e; + } catch (Exception e) { + throw new OperationFailureException(operr("HPKE seal failed: %s", e.getMessage())); + } finally { + if (dekRaw != null) { + for (int i = 0; i < dekRaw.length; i++) { + dekRaw[i] = 0; + } + } + if (envelope != null) { + for (int i = 0; i < envelope.length; i++) { + envelope[i] = 0; + } + } + } + } + + /** + * Define a per-volume libvirt secret on {@code hostUuid}. Returns the + * libvirt secret UUID. Throws on failure / blank reply. + */ + public String defineLibvirtSecretOnHost(String hostUuid, String vmUuid, String volUuid, + String dekBase64, Integer keyVersion) { + return defineLibvirtSecretOnHost(hostUuid, vmUuid, volUuid, dekBase64, keyVersion, null); + } + + public String defineLibvirtSecretOnHost(String hostUuid, String vmUuid, String volUuid, + String dekBase64, Integer keyVersion, String secretUuid) { + if (StringUtils.isBlank(hostUuid) || StringUtils.isBlank(volUuid) || + StringUtils.isBlank(dekBase64) || keyVersion == null) { + throw new OperationFailureException(operr( + "defineLibvirtSecretOnHost requires non-blank hostUuid, volUuid, dekBase64 and a non-null keyVersion")); + } + SecretHostDefineMsg defineMsg = new SecretHostDefineMsg(); + defineMsg.setHostUuid(hostUuid); + defineMsg.setVmUuid(vmUuid); + defineMsg.setDekBase64(dekBase64); + defineMsg.setPurpose("volume"); + defineMsg.setKeyVersion(keyVersion); + defineMsg.setUsageInstance(KVMConstant.volumeSecretUsageInstance(volUuid)); + if (StringUtils.isNotBlank(secretUuid)) { + defineMsg.setSecretUuid(secretUuid); + } + defineMsg.setDescription(String.format("LUKS DEK for volume %s", volUuid)); + bus.makeTargetServiceIdByResourceUuid(defineMsg, HostConstant.SERVICE_ID, hostUuid); + + MessageReply reply = bus.call(defineMsg); + if (!reply.isSuccess()) { + throw new OperationFailureException(operr( + "failed to ensure libvirt secret for encrypted volume[uuid:%s] on host[uuid:%s]", + volUuid, hostUuid).withCause(reply.getError())); + } + SecretHostDefineReply r = reply.castReply(); + if (StringUtils.isBlank(r.getSecretUuid())) { + throw new OperationFailureException(operr( + "ensure volume LUKS secret on host succeeded but secretUuid is empty, host[uuid:%s]", + hostUuid)); + } + + // Remember which host now owns this volume's libvirt secret so that + // expunge can clean it up later, even if the owning VM is gone by then. + // recreate=true overwrites any stale tag from a previous host. + try { + SystemTagCreator tc = VolumeSystemTags.VOLUME_LIBVIRT_SECRET_HOST.newSystemTagCreator(volUuid); + tc.setTagByTokens(Collections.singletonMap( + VolumeSystemTags.VOLUME_LIBVIRT_SECRET_HOST_TOKEN, hostUuid)); + tc.inherent = false; + tc.recreate = true; + tc.create(); + } catch (RuntimeException tagEx) { + // Tag write failure must not break the actual secret define -- the + // define already succeeded, the tag is for cleanup bookkeeping only. + logger.warn(String.format( + "failed to stamp VOLUME_LIBVIRT_SECRET_HOST tag on volume[uuid:%s] for host[uuid:%s]: %s", + volUuid, hostUuid, tagEx.getMessage())); + } + + return r.getSecretUuid(); + } + + public String lookupVmInstanceUuid(String volumeUuid) { + return Q.New(VolumeVO.class) + .eq(VolumeVO_.uuid, volumeUuid) + .select(VolumeVO_.vmInstanceUuid) + .findValue(); + } + + /** + * Materialize the DEK for {@code volUuid} under the binding in + * {@code kpUuid}, then define+set-value the libvirt secret on + * {@code hostUuid}. Used by both the create-time path and the start-time + * fallback when the host's libvirt secret value was lost. + */ + public String defineSecretFromBinding(String hostUuid, String vmUuid, String volUuid, String kpUuid) { + return defineSecretFromBinding(hostUuid, vmUuid, volUuid, kpUuid, null); + } + + public String defineSecretFromBinding(String hostUuid, String vmUuid, String volUuid, String kpUuid, String secretUuid) { + if (StringUtils.isBlank(kpUuid)) { + throw new OperationFailureException(operr( + "encrypted volume[uuid:%s] has no key provider binding; cannot define libvirt secret on host[uuid:%s]", + volUuid, hostUuid)); + } + EncryptedResourceKeyManager.ResourceKeyResult keyResult = materializeDek(volUuid, kpUuid); + String dekBase64 = keyResult.getDekBase64(); + if (StringUtils.isBlank(dekBase64)) { + throw new OperationFailureException(operr( + "encrypted volume[uuid:%s]: key manager returned empty DEK for libvirt secret", + volUuid)); + } + return defineLibvirtSecretOnHost(hostUuid, vmUuid, volUuid, dekBase64, keyResult.getKeyVersion(), secretUuid); + } + + /** + * Ask {@code hostUuid} for the libvirt secret UUID identified by the + * (vm, volume, keyVersion) tuple. Returns null on SECRET_NOT_FOUND so + * callers can fall back to {@link #defineSecretFromBinding}; throws on + * any other failure. + */ + public String getSecretOnHost(String hostUuid, String vmUuid, String volUuid, Integer keyVersion) { + SecretHostGetMsg msg = new SecretHostGetMsg(); + msg.setHostUuid(hostUuid); + msg.setVmUuid(vmUuid); + msg.setPurpose("volume"); + msg.setKeyVersion(keyVersion); + msg.setUsageInstance(KVMConstant.volumeSecretUsageInstance(volUuid)); + bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, hostUuid); + + MessageReply reply = bus.call(msg); + if (reply.isSuccess()) { + SecretHostGetReply r = reply.castReply(); + return r.getSecretUuid(); + } + ErrorCode err = reply.getError(); + if (SecretHostGetReply.isSecretNotFound(err)) { + return null; + } + throw new OperationFailureException(operr( + "failed to get libvirt LUKS secret on host[uuid:%s] vm[uuid:%s] volume[uuid:%s] keyVersion[%s]: %s", + hostUuid, vmUuid, volUuid, keyVersion, err)); + } + + public void deleteSecretOnHostBestEffort(String hostUuid, String vmUuid, String volUuid, Integer keyVersion) { + if (StringUtils.isBlank(hostUuid) || StringUtils.isBlank(vmUuid) + || StringUtils.isBlank(volUuid) || keyVersion == null) { + return; + } + SecretHostDeleteMsg msg = new SecretHostDeleteMsg(); + msg.setHostUuid(hostUuid); + msg.setVmUuid(vmUuid); + msg.setPurpose("volume"); + msg.setKeyVersion(keyVersion); + msg.setUsageInstance(KVMConstant.volumeSecretUsageInstance(volUuid)); + bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, hostUuid); + + MessageReply reply = bus.call(msg); + if (!reply.isSuccess()) { + logger.warn(String.format( + "best-effort delete libvirt LUKS secret failed for volume[uuid:%s] on host[uuid:%s] vm[uuid:%s]: %s", + volUuid, hostUuid, vmUuid, reply.getError())); + } + } + + public String resolveOrDefineSecretForVolume(String hostUuid, String vmUuid, String volUuid) { + Integer keyVersion = volumeEncryptedResourceKeyBackend.findKeyVersionByVolume(volUuid); + if (keyVersion == null) { + throw new OperationFailureException(operr( + "encrypted volume[uuid:%s] has no key version bound (EncryptedResourceKeyRefVO missing);" + + " cannot resolve libvirt LUKS secret on host[uuid:%s] for vm[uuid:%s]", + volUuid, hostUuid, vmUuid)); + } + String secretUuid = getSecretOnHost(hostUuid, vmUuid, volUuid, keyVersion); + if (StringUtils.isNotBlank(secretUuid)) { + return secretUuid; + } + String kpUuid = volumeEncryptedResourceKeyBackend.findKeyProviderUuidByVolume(volUuid); + return defineSecretFromBinding(hostUuid, vmUuid, volUuid, kpUuid); + } + + private String resolveVolumeLibvirtSecretUuidFromDomainXml(String hostUuid, String vmUuid, String volUuid) { + KVMAgentCommands.ResolveVolumeLibvirtSecretCmd cmd = new KVMAgentCommands.ResolveVolumeLibvirtSecretCmd(); + cmd.setVmUuid(vmUuid); + cmd.setVolumeUuid(volUuid); + + KVMHostAsyncHttpCallMsg msg = new KVMHostAsyncHttpCallMsg(); + msg.setHostUuid(hostUuid); + msg.setPath(KVMConstant.KVM_VOLUME_RESOLVE_LIBVIRT_SECRET_UUID_PATH); + msg.setCommand(cmd); + bus.makeTargetServiceIdByResourceUuid(msg, HostConstant.SERVICE_ID, hostUuid); + + MessageReply reply = bus.call(msg); + if (!reply.isSuccess()) { + logger.warn(String.format( + "failed to resolve volume LUKS secret UUID from domain XML, host[uuid:%s], vm[uuid:%s], volume[uuid:%s]: %s", + hostUuid, vmUuid, volUuid, reply.getError())); + return null; + } + + KVMHostAsyncHttpCallReply kReply = reply.castReply(); + KVMAgentCommands.ResolveVolumeLibvirtSecretResponse rsp = + kReply.toResponse(KVMAgentCommands.ResolveVolumeLibvirtSecretResponse.class); + if (rsp != null && rsp.isSuccess() && StringUtils.isNotBlank(rsp.getSecretUuid())) { + return rsp.getSecretUuid(); + } + + String err = rsp == null ? "empty agent response" : rsp.getError(); + logger.warn(String.format( + "volume LUKS secret UUID is not available from domain XML, host[uuid:%s], vm[uuid:%s], volume[uuid:%s]: %s", + hostUuid, vmUuid, volUuid, err)); + return null; + } + + public String resolveOrDefineSecretForVolumeMigration(String srcHostUuid, String dstHostUuid, String vmUuid, String volUuid) { + if (StringUtils.isBlank(srcHostUuid) || StringUtils.isBlank(dstHostUuid) + || StringUtils.isBlank(vmUuid) || StringUtils.isBlank(volUuid)) { + throw new OperationFailureException(operr( + "resolve migration LUKS secret requires non-blank srcHostUuid, dstHostUuid, vmUuid and volUuid")); + } + + Integer keyVersion = volumeEncryptedResourceKeyBackend.findKeyVersionByVolume(volUuid); + if (keyVersion == null) { + throw new OperationFailureException(operr( + "encrypted volume[uuid:%s] has no key version bound (EncryptedResourceKeyRefVO missing);" + + " cannot resolve migration LUKS secret for vm[uuid:%s]", + volUuid, vmUuid)); + } + + String sourceSecretUuid = resolveVolumeLibvirtSecretUuidFromDomainXml(srcHostUuid, vmUuid, volUuid); + if (StringUtils.isNotBlank(sourceSecretUuid)) { + logger.info(String.format( + "resolved source volume LUKS secret UUID from domain XML before migration, vm[uuid:%s], volume[uuid:%s], host[uuid:%s], secretUuid:%s", + vmUuid, volUuid, srcHostUuid, sourceSecretUuid)); + } + if (StringUtils.isBlank(sourceSecretUuid)) { + sourceSecretUuid = getSecretOnHost(srcHostUuid, vmUuid, volUuid, keyVersion); + } + if (StringUtils.isBlank(sourceSecretUuid)) { + String kpUuid = volumeEncryptedResourceKeyBackend.findKeyProviderUuidByVolume(volUuid); + sourceSecretUuid = defineSecretFromBinding(srcHostUuid, vmUuid, volUuid, kpUuid); + } + + String destSecretUuid = getSecretOnHost(dstHostUuid, vmUuid, volUuid, keyVersion); + if (StringUtils.equals(destSecretUuid, sourceSecretUuid)) { + return destSecretUuid; + } + + String kpUuid = volumeEncryptedResourceKeyBackend.findKeyProviderUuidByVolume(volUuid); + return defineSecretFromBinding(dstHostUuid, vmUuid, volUuid, kpUuid, sourceSecretUuid); + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedStartExtension.java b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedStartExtension.java new file mode 100644 index 00000000000..db68ec8dc22 --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedStartExtension.java @@ -0,0 +1,110 @@ +package org.zstack.storage.encrypt; + +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowire; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Configurable; +import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.header.host.HostInventory; +import org.zstack.header.vm.VmBeforeCreateOnHypervisorExtensionPoint; +import org.zstack.header.vm.VmBeforeStartOnHypervisorExtensionPoint; +import org.zstack.header.vm.VmInstanceSpec; +import org.zstack.header.volume.VolumeInventory; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.zstack.core.Platform.operr; + +/** + * Per start_vm: resolve the libvirt LUKS secret UUID for every encrypted volume + * on the destination host and stash the mapping into + * {@link VmInstanceSpec#putExtensionData} under {@link #EXT_DATA_KEY} so + * {@code KVMHost.handleStart} can inline it into the {@code VolumeTO} that + * ships with {@code StartVmCmd}. + * + *

Why every start (not persisted)? libvirt secret values are RAM-only; + * libvirtd restart / host reboot / live-migrate-to-new-host all wipe them. The + * UUID itself would persist but without the value the secret is useless to qemu. + * So on every start_vm we delegate to + * {@link VolumeEncryptedSecretHelper#resolveOrDefineSecretForVolume}, which + * first asks the host (idempotent {@code SecretHostGetMsg} → key-agent + * {@code GetSecret}) and falls back to materialize-DEK + define on miss. + */ +@Configurable(preConstruction = true, autowire = Autowire.BY_TYPE) +public class VolumeEncryptedStartExtension + implements VmBeforeStartOnHypervisorExtensionPoint, VmBeforeCreateOnHypervisorExtensionPoint { + + private static final CLogger logger = Utils.getLogger(VolumeEncryptedStartExtension.class); + + /** {@code Map} consumed by KVMHost.handleStart. */ + public static final String EXT_DATA_KEY = "VolumeLuksSecrets"; + + @Autowired + private VolumeEncryptedSecretHelper secretHelper; + + @Override + public void beforeStartVmOnHypervisor(VmInstanceSpec spec) { + HostInventory destHost = spec.getDestHost(); + if (destHost == null || StringUtils.isBlank(destHost.getUuid())) { + return; + } + + List encryptedVolumes = collectEncryptedVolumes(spec); + if (encryptedVolumes.isEmpty()) { + return; + } + + String hostUuid = destHost.getUuid(); + String vmUuid = spec.getVmInventory().getUuid(); + Map resolved = new HashMap<>(); + + for (VolumeInventory vol : encryptedVolumes) { + String volUuid = vol.getUuid(); + String secretUuid = secretHelper.resolveOrDefineSecretForVolume(hostUuid, vmUuid, volUuid); + if (StringUtils.isBlank(secretUuid)) { + throw new OperationFailureException(operr( + "failed to resolve libvirt LUKS secret for encrypted volume[uuid:%s] on host[uuid:%s]", + volUuid, hostUuid)); + } + resolved.put(volUuid, secretUuid); + } + + if (!resolved.isEmpty()) { + spec.putExtensionData(EXT_DATA_KEY, resolved); + logger.debug(String.format("LUKS-START-EXT stashed %d secrets into spec.extensionData[%s]: %s", + resolved.size(), EXT_DATA_KEY, resolved)); + } + } + + /** + * "Create VM" (provisioning) path uses {@link org.zstack.compute.vm.VmCreateOnHypervisorFlow} + * which fires {@link VmBeforeCreateOnHypervisorExtensionPoint}, NOT the start-vm hook. + * Delegate to the same logic: encrypted root volumes need their libvirt secret + * resolved on the destination host before the agent receives StartVmCmd. + */ + @Override + public void beforeCreateVmOnHypervisor(VmInstanceSpec spec) { + beforeStartVmOnHypervisor(spec); + } + + private List collectEncryptedVolumes(VmInstanceSpec spec) { + List result = new ArrayList<>(); + VolumeInventory root = spec.getDestRootVolume(); + if (root != null && Boolean.TRUE.equals(root.getEncrypted())) { + result.add(root); + } + if (spec.getDestDataVolumes() != null) { + for (VolumeInventory v : spec.getDestDataVolumes()) { + if (v != null && Boolean.TRUE.equals(v.getEncrypted())) { + result.add(v); + } + } + } + return result; + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedStorageLiveMigrationPrepareBlockMigrationExtension.java b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedStorageLiveMigrationPrepareBlockMigrationExtension.java new file mode 100644 index 00000000000..2f317cf37fd --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedStorageLiveMigrationPrepareBlockMigrationExtension.java @@ -0,0 +1,52 @@ +package org.zstack.storage.encrypt; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.host.HostInventory; +import org.zstack.header.storage.migration.KvmBlockLiveMigrationExtensionPoint; +import org.zstack.header.vm.VmInstanceInventory; +import org.zstack.header.volume.VolumeVO; + +import java.util.List; +import java.util.Map; + +import static org.zstack.core.Platform.operr; + +public class VolumeEncryptedStorageLiveMigrationPrepareBlockMigrationExtension implements KvmBlockLiveMigrationExtensionPoint { + @Autowired + private VolumeEncryptedSecretHelper volumeEncryptedSecretHelper; + + @Override + public ErrorCode beforeBlockLiveMigration(HostInventory dstHost, + VmInstanceInventory vm, + List volumesToMigrate, + Map volumeMappingDict, + Map targetVolumeLuksSecrets) { + if (volumeMappingDict == null || volumeMappingDict.isEmpty()) { + return null; + } + + try { + for (VolumeVO sourceVolume : volumesToMigrate) { + if (!sourceVolume.isEncrypted()) { + continue; + } + + String targetVolumeUuid = volumeMappingDict.get(sourceVolume.getUuid()); + if (targetVolumeUuid == null || targetVolumeUuid.trim().isEmpty()) { + return operr("missing target volume uuid for encrypted source volume[uuid:%s]", + sourceVolume.getUuid()); + } + + targetVolumeLuksSecrets.put(targetVolumeUuid, + volumeEncryptedSecretHelper.resolveOrDefineSecretForVolume( + dstHost.getUuid(), vm.getUuid(), targetVolumeUuid)); + } + } catch (Exception e) { + return operr("failed to prepare LUKS secret for block live migration vm[uuid:%s]: %s", + vm.getUuid(), e.getMessage()); + } + + return null; + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedStorageLiveMigrationPrepareTargetVolumeExtension.java b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedStorageLiveMigrationPrepareTargetVolumeExtension.java new file mode 100644 index 00000000000..5fca42a0c6a --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedStorageLiveMigrationPrepareTargetVolumeExtension.java @@ -0,0 +1,166 @@ +package org.zstack.storage.encrypt; + +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.header.core.workflow.Flow; +import org.zstack.header.core.workflow.FlowTrigger; +import org.zstack.header.core.workflow.NoRollbackFlow; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.header.host.HostConstant; +import org.zstack.header.message.MessageReply; +import org.zstack.header.storage.migration.KvmStorageLiveMigrationExtensionPoint; +import org.zstack.header.storage.primary.InstantiateTemporaryVolumeOnPrimaryStorageMsg; +import org.zstack.header.volume.CreateVolumeMsg; +import org.zstack.header.volume.VolumeAO; +import org.zstack.header.volume.VolumeInventory; +import org.zstack.header.volume.VolumeLuksAgentSpec; +import org.zstack.header.volume.VolumeVO; +import org.zstack.kvm.KVMConstant; +import org.zstack.kvm.KVMAgentCommands; +import org.zstack.kvm.KVMHostSyncHttpCallMsg; +import org.zstack.kvm.KVMHostSyncHttpCallReply; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.zstack.core.Platform.operr; + +public class VolumeEncryptedStorageLiveMigrationPrepareTargetVolumeExtension implements KvmStorageLiveMigrationExtensionPoint { + private static final CLogger logger = Utils.getLogger(VolumeEncryptedStorageLiveMigrationPrepareTargetVolumeExtension.class); + + @Autowired + private CloudBus bus; + @Autowired + private VolumeEncryptedResourceKeyBackend volumeEncryptedResourceKeyBackend; + @Autowired + private VolumeEncryptedSecretHelper volumeEncryptedSecretHelper; + + @Override + public List getBeforeCreateTemporaryVolumeFlows(String vmUuid, + String srcHostUuid, + List volumesToMigrate, + Map sourceVolumeVirtualSizes) { + return Collections.singletonList(new NoRollbackFlow() { + String __name__ = "query-active-volume-size-for-encrypted-vm-" + vmUuid; + + @Override + public void run(FlowTrigger trigger, Map data) { + List encryptedVolumes = volumesToMigrate.stream() + .filter(VolumeVO::isEncrypted) + .collect(Collectors.toList()); + if (encryptedVolumes.isEmpty()) { + trigger.next(); + return; + } + + if (StringUtils.isBlank(srcHostUuid)) { + trigger.fail(operr("cannot query active volume size for encrypted vm[uuid:%s], source host uuid is empty", + vmUuid)); + return; + } + + KVMAgentCommands.GetActiveVolumeSizeCmd cmd = new KVMAgentCommands.GetActiveVolumeSizeCmd(); + cmd.setVmUuid(vmUuid); + cmd.setInstallPaths(encryptedVolumes.stream().map(VolumeAO::getInstallPath).collect(Collectors.toList())); + + KVMHostSyncHttpCallMsg hmsg = new KVMHostSyncHttpCallMsg(); + hmsg.setCommand(cmd); + hmsg.setHostUuid(srcHostUuid); + hmsg.setPath(KVMConstant.KVM_GET_ACTIVE_VOLUME_SIZE_PATH); + bus.makeTargetServiceIdByResourceUuid(hmsg, HostConstant.SERVICE_ID, srcHostUuid); + bus.send(hmsg, new CloudBusCallBack(trigger) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + trigger.fail(reply.getError()); + return; + } + + KVMHostSyncHttpCallReply r = reply.castReply(); + KVMAgentCommands.GetActiveVolumeSizeRsp rsp = + r.toResponse(KVMAgentCommands.GetActiveVolumeSizeRsp.class); + if (!rsp.isSuccess()) { + trigger.fail(operr("failed to query active volume size for encrypted vm[uuid:%s] on host[uuid:%s], %s", + vmUuid, srcHostUuid, rsp.getError())); + return; + } + + Map volumeSizes = rsp.getVolumeSizes(); + for (VolumeVO volume : encryptedVolumes) { + Long size = volumeSizes == null ? null : volumeSizes.get(volume.getInstallPath()); + if (size == null || size <= 0) { + trigger.fail(operr("failed to query active volume[uuid:%s, path:%s] virtual size for encrypted vm[uuid:%s] on host[uuid:%s]", + volume.getUuid(), volume.getInstallPath(), vmUuid, srcHostUuid)); + return; + } + + sourceVolumeVirtualSizes.put(volume.getInstallPath(), size); + logger.debug(String.format("use active volume[uuid:%s, path:%s] virtual size[%s] for live storage migration temporary volume", + volume.getUuid(), volume.getInstallPath(), size)); + } + + trigger.next(); + } + }); + } + }); + } + + @Override + public ErrorCode beforeCreateTemporaryVolume(CreateVolumeMsg msg, + VolumeVO sourceVolume, + Map sourceVolumeVirtualSizes) { + if (!sourceVolume.isEncrypted()) { + return null; + } + + msg.setEncrypted(true); + msg.setSize(sourceVolumeVirtualSizes.getOrDefault(sourceVolume.getInstallPath(), sourceVolume.getSize())); + return null; + } + + @Override + public ErrorCode afterCreateTemporaryVolume(VolumeVO sourceVolume, VolumeInventory temporaryVolume) { + if (!sourceVolume.isEncrypted()) { + return null; + } + + try { + volumeEncryptedResourceKeyBackend.copyVolumeKeyToVolume(sourceVolume.getUuid(), temporaryVolume.getUuid()); + temporaryVolume.setEncrypted(true); + return null; + } catch (Exception e) { + return operr("failed to copy encryption key ref from volume[uuid:%s] to live migration temporary volume[uuid:%s]: %s", + sourceVolume.getUuid(), temporaryVolume.getUuid(), e.getMessage()); + } + } + + @Override + public ErrorCode beforeInstantiateTemporaryVolume(String hostUuid, + VolumeInventory temporaryVolume, + InstantiateTemporaryVolumeOnPrimaryStorageMsg msg) { + if (!Boolean.TRUE.equals(temporaryVolume.getEncrypted())) { + return null; + } + + try { + VolumeLuksAgentSpec spec = new VolumeLuksAgentSpec(); + spec.setEncryptLuksSecretMaterialFilePath( + volumeEncryptedSecretHelper.prepareLuksSecretMaterialFileOnHost(hostUuid, temporaryVolume.getUuid())); + msg.setVolumeLuksAgentSpec(spec); + return null; + } catch (OperationFailureException e) { + return e.getErrorCode(); + } catch (Exception e) { + return operr("failed to prepare LUKS secret material for live migration temporary volume[uuid:%s] on host[uuid:%s]: %s", + temporaryVolume.getUuid(), hostUuid, e.getMessage()); + } + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedTrashCleanupHelper.java b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedTrashCleanupHelper.java new file mode 100644 index 00000000000..7d1cb7f6e0b --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/VolumeEncryptedTrashCleanupHelper.java @@ -0,0 +1,68 @@ +package org.zstack.storage.encrypt; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Autowire; +import org.springframework.beans.factory.annotation.Configurable; +import org.zstack.core.db.Q; +import org.zstack.header.core.trash.InstallPathRecycleInventory; +import org.zstack.header.storage.snapshot.VolumeSnapshotVO; +import org.zstack.header.storage.snapshot.VolumeSnapshotVO_; +import org.zstack.header.volume.VolumeVO; +import org.zstack.header.volume.VolumeVO_; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +/** + * Trash can keep old encrypted bits after the live resource has already moved to + * a new install path or encryption state. Key refs must therefore survive until + * the trash entry that still needs those bits is actually cleaned. + */ +@Configurable(preConstruction = true, autowire = Autowire.BY_TYPE) +public class VolumeEncryptedTrashCleanupHelper { + private static final CLogger logger = Utils.getLogger(VolumeEncryptedTrashCleanupHelper.class); + + @Autowired + private VolumeEncryptedResourceKeyBackend volumeEncryptedResourceKeyBackend; + + public void cleanupKeyRefAfterTrashDeleted(InstallPathRecycleInventory inv) { + if (inv == null || inv.getResourceUuid() == null || inv.getResourceType() == null) { + return; + } + + try { + if (VolumeVO.class.getSimpleName().equals(inv.getResourceType())) { + cleanupVolumeKeyRef(inv.getResourceUuid()); + } else if (VolumeSnapshotVO.class.getSimpleName().equals(inv.getResourceType())) { + cleanupSnapshotKeyRef(inv.getResourceUuid()); + } + } catch (RuntimeException e) { + logger.warn(String.format( + "failed to cleanup encrypted key ref after deleting trash[trashId:%s, resourceType:%s, resourceUuid:%s]: %s", + inv.getTrashId(), inv.getResourceType(), inv.getResourceUuid(), e.getMessage())); + } + } + + private void cleanupVolumeKeyRef(String volumeUuid) { + Boolean encrypted = Q.New(VolumeVO.class) + .select(VolumeVO_.encrypted) + .eq(VolumeVO_.uuid, volumeUuid) + .findValue(); + if (Boolean.TRUE.equals(encrypted)) { + return; + } + + volumeEncryptedResourceKeyBackend.detachKeyProviderFromVolume(volumeUuid); + } + + private void cleanupSnapshotKeyRef(String snapshotUuid) { + Boolean encrypted = Q.New(VolumeSnapshotVO.class) + .select(VolumeSnapshotVO_.encrypted) + .eq(VolumeSnapshotVO_.uuid, snapshotUuid) + .findValue(); + if (Boolean.TRUE.equals(encrypted)) { + return; + } + + volumeEncryptedResourceKeyBackend.detachKeyProviderFromSnapshot(snapshotUuid); + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/VolumeSnapshotEncryptionExtension.java b/storage/src/main/java/org/zstack/storage/encrypt/VolumeSnapshotEncryptionExtension.java new file mode 100644 index 00000000000..03ab7a2ddaf --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/VolumeSnapshotEncryptionExtension.java @@ -0,0 +1,244 @@ +package org.zstack.storage.encrypt; + +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.core.db.DatabaseFacade; +import org.zstack.header.core.Completion; +import org.zstack.header.core.workflow.Flow; +import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.header.host.TakeSnapshotOnHypervisorMsg; +import org.zstack.header.storage.snapshot.BeforeTakeLiveSnapshotsOnVolumes; +import org.zstack.header.storage.snapshot.ConsistentType; +import org.zstack.header.storage.snapshot.CreateVolumesSnapshotOverlayInnerMsg; +import org.zstack.header.storage.snapshot.TakeSnapshotsOnKvmJobStruct; +import org.zstack.header.storage.snapshot.TakeVolumesSnapshotOnKvmMsg; +import org.zstack.header.storage.snapshot.TakeVolumesSnapshotOnKvmReply; +import org.zstack.header.storage.snapshot.VolumeSnapshotAfterDeleteExtensionPoint; +import org.zstack.header.storage.snapshot.VolumeSnapshotCreationExtensionPoint; +import org.zstack.header.storage.snapshot.VolumeSnapshotInventory; +import org.zstack.header.storage.snapshot.VolumeSnapshotVO; +import org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupInventory; +import org.zstack.header.volume.CreateVolumeSnapshotGroupMessage; +import org.zstack.header.volume.VolumeVO; +import org.zstack.kvm.KVMAgentCommands; +import org.zstack.kvm.KVMTakeSnapshotExtensionPoint; +import org.zstack.kvm.KVMHostInventory; +import org.zstack.kvm.VolumeTO; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.util.List; +import java.util.Map; + +import static org.zstack.core.Platform.operr; + +public class VolumeSnapshotEncryptionExtension implements KVMTakeSnapshotExtensionPoint, + BeforeTakeLiveSnapshotsOnVolumes, VolumeSnapshotCreationExtensionPoint, + VolumeSnapshotAfterDeleteExtensionPoint { + private static final CLogger logger = Utils.getLogger(VolumeSnapshotEncryptionExtension.class); + + @Autowired + private DatabaseFacade dbf; + @Autowired + private VolumeSnapshotEncryptionHelper snapshotEncryptionHelper; + @Autowired + private VolumeEncryptedResourceKeyBackend volumeEncryptedResourceKeyBackend; + @Autowired + private VolumeEncryptedSecretHelper volumeEncryptedSecretHelper; + + @Override + public void beforeTakeSnapshot(KVMHostInventory host, TakeSnapshotOnHypervisorMsg msg, + KVMAgentCommands.TakeSnapshotCmd cmd, Completion completion) { + try { + VolumeVO volume = findVolume(msg.getVolume().getUuid()); + if (!volume.isEncrypted()) { + completion.success(); + return; + } + + VolumeSnapshotInventory snapshot = findSnapshot(msg.getSnapshotName()); + snapshotEncryptionHelper.inheritVolumeKeyToSnapshot(volume, snapshot); + if (!cmd.isOnline()) { + String envelopeDek = volumeEncryptedSecretHelper.prepareLuksEnvelopeDekOnHost(host.getUuid(), volume.getUuid()); + if (StringUtils.isNotBlank(envelopeDek)) { + cmd.setEncryptedDek(envelopeDek); + } + } + + completion.success(); + } catch (OperationFailureException e) { + completion.fail(e.getErrorCode()); + } catch (RuntimeException e) { + completion.fail(operr("failed to prepare encrypted volume snapshot[uuid:%s] on host[uuid:%s]: %s", + msg.getSnapshotName(), host.getUuid(), e.getMessage())); + } + } + + @Override + public void beforeTakeLiveSnapshotsOnVolumes(CreateVolumesSnapshotOverlayInnerMsg msg, + TakeVolumesSnapshotOnKvmMsg tmsg, + Map flowData, + Completion completion) { + try { + if (tmsg == null || tmsg.getSnapshotJobs() == null) { + completion.success(); + return; + } + + for (TakeSnapshotsOnKvmJobStruct job : tmsg.getSnapshotJobs()) { + if (job.isMemory()) { + continue; + } + + VolumeVO volume = findVolume(job.getVolumeUuid()); + if (!volume.isEncrypted()) { + continue; + } + + VolumeSnapshotInventory snapshot = findSnapshot(job.getSnapshotUuid()); + snapshotEncryptionHelper.inheritVolumeKeyToSnapshot(volume, snapshot); + + String secretPath = + volumeEncryptedSecretHelper.prepareLuksSecretMaterialFileOnHost(tmsg.getHostUuid(), volume.getUuid()); + // This extension runs in a chain with storage-specific implementations; + // keep any secret path already prepared by a storage backend. + boolean needFillEncryptSecret = StringUtils.isBlank(job.getEncryptLuksSecretMaterialFilePath()); + if (needFillEncryptSecret && StringUtils.isNotBlank(secretPath)) { + job.setEncryptLuksSecretMaterialFilePath(secretPath); + } + + if (job.getVolume() instanceof VolumeTO) { + ((VolumeTO) job.getVolume()).setLuksSecretUuid( + volumeEncryptedSecretHelper.resolveOrDefineSecretForVolume( + tmsg.getHostUuid(), volume.getVmInstanceUuid(), volume.getUuid())); + } + } + + completion.success(); + } catch (OperationFailureException e) { + completion.fail(e.getErrorCode()); + } catch (RuntimeException e) { + completion.fail(operr("failed to prepare encrypted live volume snapshots: %s", e.getMessage())); + } + } + + @Override + public void afterVolumeSnapshotCreated(VolumeSnapshotInventory snapshot, Completion completion) { + try { + if (snapshot == null) { + completion.success(); + return; + } + + VolumeVO volume = findVolume(snapshot.getVolumeUuid()); + if (!volume.isEncrypted()) { + completion.success(); + return; + } + + snapshotEncryptionHelper.completeTakeSnapshot(volume, snapshot); + completion.success(); + } catch (OperationFailureException e) { + completion.fail(e.getErrorCode()); + } catch (RuntimeException e) { + completion.fail(operr("failed to complete encrypted volume snapshot[uuid:%s]: %s", + snapshot == null ? null : snapshot.getUuid(), e.getMessage())); + } + } + + @Override + public void afterTakeSnapshot(KVMHostInventory host, TakeSnapshotOnHypervisorMsg msg, + KVMAgentCommands.TakeSnapshotCmd cmd, + KVMAgentCommands.TakeSnapshotResponse rsp) { + } + + @Override + public void afterTakeSnapshotFailed(KVMHostInventory host, TakeSnapshotOnHypervisorMsg msg, + KVMAgentCommands.TakeSnapshotCmd cmd, + KVMAgentCommands.TakeSnapshotResponse rsp, + org.zstack.header.errorcode.ErrorCode err) { + } + + @Override + public void afterVolumeLiveSnapshotGroupCreatedOnBackend(CreateVolumesSnapshotOverlayInnerMsg msg, + TakeVolumesSnapshotOnKvmReply treply, + Completion completion) { + completion.success(); + } + + @Override + public void afterVolumeLiveSnapshotGroupCreationFailsOnBackend(CreateVolumesSnapshotOverlayInnerMsg msg, + TakeVolumesSnapshotOnKvmReply treply) { + } + + @Override + public void afterVolumeSnapshotGroupCreated(VolumeSnapshotGroupInventory snapshotGroup, + ConsistentType consistentType, + Completion completion) { + completion.success(); + } + + @Override + public List beforeCreateVolumeSnapshotFlow(CreateVolumeSnapshotGroupMessage msg) { + return null; + } + + @Override + public void volumeSnapshotAfterDeleteExtensionPoint(VolumeSnapshotInventory snapshot, Completion completion) { + completion.success(); + } + + @Override + public void volumeSnapshotAfterFailedDeleteExtensionPoint(VolumeSnapshotInventory snapshot) { + } + + @Override + public void volumeSnapshotAfterCleanUpExtensionPoint(String volumeUuid, List snapshots) { + if (snapshots == null || snapshots.isEmpty()) { + return; + } + + for (VolumeSnapshotInventory snapshot : snapshots) { + if (snapshot == null || StringUtils.isBlank(snapshot.getUuid())) { + continue; + } + if (!Boolean.TRUE.equals(snapshot.getEncrypted())) { + continue; + } + if (dbf.isExist(snapshot.getUuid(), VolumeSnapshotVO.class)) { + continue; + } + try { + volumeEncryptedResourceKeyBackend.detachKeyProviderFromSnapshot(snapshot.getUuid()); + } catch (RuntimeException e) { + logger.warn(String.format( + "failed to detach EncryptedResourceKeyRefVO for volume snapshot[uuid:%s] on delete cleanup: %s", + snapshot.getUuid(), e.getMessage())); + } + } + } + + private VolumeVO findVolume(String volumeUuid) { + if (StringUtils.isBlank(volumeUuid)) { + throw new OperationFailureException(operr("volume uuid is required for encrypted snapshot preparation")); + } + + VolumeVO volume = dbf.findByUuid(volumeUuid, VolumeVO.class); + if (volume == null) { + throw new OperationFailureException(operr("volume[uuid:%s] not found", volumeUuid)); + } + return volume; + } + + private VolumeSnapshotInventory findSnapshot(String snapshotUuid) { + if (StringUtils.isBlank(snapshotUuid)) { + throw new OperationFailureException(operr("snapshot uuid is required for encrypted snapshot preparation")); + } + + VolumeSnapshotVO snapshot = dbf.findByUuid(snapshotUuid, VolumeSnapshotVO.class); + if (snapshot == null) { + throw new OperationFailureException(operr("volume snapshot[uuid:%s] not found", snapshotUuid)); + } + return VolumeSnapshotInventory.valueOf(snapshot); + } +} diff --git a/storage/src/main/java/org/zstack/storage/encrypt/VolumeSnapshotEncryptionHelper.java b/storage/src/main/java/org/zstack/storage/encrypt/VolumeSnapshotEncryptionHelper.java new file mode 100644 index 00000000000..912d1caf960 --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/encrypt/VolumeSnapshotEncryptionHelper.java @@ -0,0 +1,276 @@ +package org.zstack.storage.encrypt; + +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowire; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Configurable; +import org.zstack.core.db.DatabaseFacade; +import org.zstack.core.db.Q; +import org.zstack.header.errorcode.OperationFailureException; +import org.zstack.header.image.ImageConstant; +import org.zstack.header.image.ImageVO; +import org.zstack.header.image.ImageVO_; +import org.zstack.header.keyprovider.EncryptedResourceKeyManager; +import org.zstack.header.storage.snapshot.VolumeSnapshotInventory; +import org.zstack.header.storage.snapshot.VolumeSnapshotVO; +import org.zstack.header.storage.snapshot.VolumeSnapshotVO_; +import org.zstack.header.volume.VolumeLuksAgentSpec; +import org.zstack.header.volume.VolumeVO; + +import static org.zstack.core.Platform.operr; + +@Configurable(preConstruction = true, autowire = Autowire.BY_TYPE) +public class VolumeSnapshotEncryptionHelper { + @Autowired + private DatabaseFacade dbf; + @Autowired + private VolumeEncryptedResourceKeyBackend keyBackend; + @Autowired + private VolumeEncryptedSecretHelper secretHelper; + @Autowired + private EncryptedResourceKeyManager encryptedResourceKeyManager; + + public void completeTakeSnapshot(VolumeVO volume, VolumeSnapshotInventory snapshot) { + if (volume == null || snapshot == null || !volume.isEncrypted()) { + return; + } + + snapshot.setEncrypted(true); + Boolean encrypted = Q.New(VolumeSnapshotVO.class) + .eq(VolumeSnapshotVO_.uuid, snapshot.getUuid()) + .select(VolumeSnapshotVO_.encrypted) + .findValue(); + if (!Boolean.TRUE.equals(encrypted)) { + VolumeSnapshotVO vo = dbf.findByUuid(snapshot.getUuid(), VolumeSnapshotVO.class); + if (vo != null) { + vo.setEncrypted(true); + dbf.update(vo); + } + } + } + + public void inheritVolumeKeyToSnapshot(VolumeVO volume, VolumeSnapshotInventory snapshot) { + if (volume == null || snapshot == null || !volume.isEncrypted()) { + return; + } + keyBackend.copyVolumeKeyToSnapshot(volume.getUuid(), snapshot.getUuid()); + } + + public void inheritFromRelatedSnapshotKeyIfPossible(VolumeVO volume, String snapshotUuid) { + if (volume == null || StringUtils.isBlank(snapshotUuid) + || keyBackend.checkVolumeKeyProviderAttached(volume.getUuid())) { + return; + } + + Boolean snapshotEncrypted = Q.New(VolumeSnapshotVO.class) + .eq(VolumeSnapshotVO_.uuid, snapshotUuid) + .select(VolumeSnapshotVO_.encrypted) + .findValue(); + if (!Boolean.TRUE.equals(snapshotEncrypted)) { + if (volume.isEncrypted()) { + String kpUuid = keyBackend.defaultKeyProviderUuid(); + if (StringUtils.isBlank(kpUuid)) { + throw new OperationFailureException(operr( + "encrypted volume[uuid:%s] has no key provider binding and no default key provider configured", + volume.getUuid())); + } + keyBackend.attachKeyProviderToVolume(volume.getUuid(), kpUuid); + materializeVolumeKey(volume.getUuid(), kpUuid, "create-encrypted-volume-from-plain-snapshot"); + } + return; + } + if (!keyBackend.checkSnapshotKeyProviderAttached(snapshotUuid)) { + throw new OperationFailureException(operr( + "encrypted snapshot[uuid:%s] has no key provider binding; cannot inherit key for volume[uuid:%s]", + snapshotUuid, volume.getUuid())); + } + + keyBackend.copySnapshotKeyToVolume(snapshotUuid, volume.getUuid()); + if (!volume.isEncrypted()) { + volume.setEncrypted(true); + dbf.update(volume); + } + } + + public void inheritFromTemporarySnapshotImageKeyIfPossible(VolumeVO volume) { + if (volume == null || StringUtils.isBlank(volume.getRootImageUuid())) { + return; + } + + String imageUrl = Q.New(ImageVO.class) + .eq(ImageVO_.uuid, volume.getRootImageUuid()) + .select(ImageVO_.url) + .findValue(); + if (StringUtils.isBlank(imageUrl)) { + return; + } + + if (imageUrl.startsWith("volume://")) { + String srcVolumeUuid = imageUrl.substring("volume://".length()); + if (!keyBackend.checkVolumeKeyProviderAttached(volume.getUuid())) { + if (keyBackend.checkVolumeKeyProviderAttached(srcVolumeUuid)) { + keyBackend.copyVolumeKeyToVolume(srcVolumeUuid, volume.getUuid()); + } + } + return; + } + + if (!imageUrl.startsWith(ImageConstant.IMAGE_FROM_SNAPSHOT_SCHEMA) + && !imageUrl.startsWith(ImageConstant.SNAPSHOT_REUSE_IMAGE_SCHEMA)) { + return; + } + + if (keyBackend.checkTemporarySnapshotImageKeyProviderAttached(volume.getRootImageUuid())) { + if (!keyBackend.checkVolumeKeyProviderAttached(volume.getUuid())) { + keyBackend.copyTemporarySnapshotImageKeyToVolume(volume.getRootImageUuid(), volume.getUuid()); + } + return; + } + + String snapshotUuid = getSnapshotUuidFromImageUrl(imageUrl); + inheritFromRelatedSnapshotKeyIfPossible(volume, snapshotUuid); + } + + public boolean hasTemporarySnapshotImageKey(String imageUuid) { + return StringUtils.isNotBlank(imageUuid) && keyBackend.checkTemporarySnapshotImageKeyProviderAttached(imageUuid); + } + + private String getSnapshotUuidFromImageUrl(String imageUrl) { + String snapshotUuid; + if (imageUrl.startsWith(ImageConstant.IMAGE_FROM_SNAPSHOT_SCHEMA)) { + snapshotUuid = imageUrl.substring(ImageConstant.IMAGE_FROM_SNAPSHOT_SCHEMA.length()); + } else if (imageUrl.startsWith(ImageConstant.SNAPSHOT_REUSE_IMAGE_SCHEMA)) { + snapshotUuid = imageUrl.substring(ImageConstant.SNAPSHOT_REUSE_IMAGE_SCHEMA.length()); + } else { + return null; + } + return snapshotUuid.length() >= 32 ? snapshotUuid.substring(0, 32) : snapshotUuid; + } + + private EncryptedResourceKeyManager.ResourceKeyResult materializeVolumeKey(String volumeUuid, + String keyProviderUuid, + String purpose) { + EncryptedResourceKeyManager.GetOrCreateResourceKeyContext ctx = + new EncryptedResourceKeyManager.GetOrCreateResourceKeyContext(); + ctx.setResourceUuid(volumeUuid); + ctx.setResourceType(VolumeVO.class.getSimpleName()); + ctx.setKeyProviderUuid(keyProviderUuid); + ctx.setPurpose(purpose); + + final EncryptedResourceKeyManager.ResourceKeyResult[] resultRef = + new EncryptedResourceKeyManager.ResourceKeyResult[1]; + final org.zstack.header.errorcode.ErrorCode[] errorRef = + new org.zstack.header.errorcode.ErrorCode[1]; + encryptedResourceKeyManager.getOrCreateKey(ctx, + new org.zstack.header.core.ReturnValueCompletion(null) { + @Override + public void success(EncryptedResourceKeyManager.ResourceKeyResult returnValue) { + resultRef[0] = returnValue; + } + + @Override + public void fail(org.zstack.header.errorcode.ErrorCode errorCode) { + errorRef[0] = errorCode; + } + }); + + if (errorRef[0] != null) { + throw new OperationFailureException(operr( + "failed to materialize encryption key for volume[uuid:%s]", volumeUuid).withCause(errorRef[0])); + } + return resultRef[0]; + } + + public VolumeLuksAgentSpec prepareTemporarySnapshotImageSecretMaterial(String hostUuid, + String snapshotUuid, + String imageUuid, + Boolean encrypted) { + if (StringUtils.isBlank(hostUuid) || StringUtils.isBlank(snapshotUuid) || StringUtils.isBlank(imageUuid) + || !Boolean.TRUE.equals(encrypted)) { + return null; + } + + EncryptedResourceKeyManager.ResourceKeyResult keyResult; + Boolean snapshotEncrypted = Q.New(VolumeSnapshotVO.class) + .eq(VolumeSnapshotVO_.uuid, snapshotUuid) + .select(VolumeSnapshotVO_.encrypted) + .findValue(); + if (Boolean.TRUE.equals(snapshotEncrypted)) { + keyBackend.copySnapshotKeyToTemporarySnapshotImage(snapshotUuid, imageUuid); + keyResult = getTemporarySnapshotImageKey(imageUuid); + } else { + keyResult = createTemporarySnapshotImageKey(imageUuid); + } + + VolumeLuksAgentSpec spec = new VolumeLuksAgentSpec(); + spec.setEncryptLuksSecretMaterialFilePath( + secretHelper.ensureLuksSecretFileOnHost(hostUuid, imageUuid, keyResult.getDekBase64())); + return spec; + } + + private EncryptedResourceKeyManager.ResourceKeyResult getTemporarySnapshotImageKey(String imageUuid) { + String kpUuid = keyBackend.findKeyProviderUuidByTemporarySnapshotImage(imageUuid); + if (StringUtils.isBlank(kpUuid)) { + throw new OperationFailureException(operr( + "encrypted temporary snapshot image[uuid:%s] has no key provider binding", imageUuid)); + } + + EncryptedResourceKeyManager.GetOrCreateResourceKeyContext ctx = + new EncryptedResourceKeyManager.GetOrCreateResourceKeyContext(); + ctx.setResourceUuid(imageUuid); + ctx.setResourceType(ImageVO.class.getSimpleName()); + ctx.setKeyProviderUuid(kpUuid); + ctx.setPurpose("prepare-temporary-snapshot-image-secret-material"); + + EncryptedResourceKeyManager.ResourceKeyResult keyResult = encryptedResourceKeyManager.getKey(ctx); + if (StringUtils.isBlank(keyResult.getDekBase64())) { + throw new OperationFailureException(operr( + "key manager returned empty DEK for encrypted temporary snapshot image[uuid:%s]", imageUuid)); + } + return keyResult; + } + + private EncryptedResourceKeyManager.ResourceKeyResult createTemporarySnapshotImageKey(String imageUuid) { + String kpUuid = keyBackend.defaultKeyProviderUuid(); + if (StringUtils.isBlank(kpUuid)) { + throw new OperationFailureException(operr( + "encrypted temporary snapshot image[uuid:%s] has no default key provider configured", imageUuid)); + } + + EncryptedResourceKeyManager.GetOrCreateResourceKeyContext ctx = + new EncryptedResourceKeyManager.GetOrCreateResourceKeyContext(); + ctx.setResourceUuid(imageUuid); + ctx.setResourceType(ImageVO.class.getSimpleName()); + ctx.setKeyProviderUuid(kpUuid); + ctx.setPurpose("prepare-temporary-snapshot-image-secret-material"); + + final EncryptedResourceKeyManager.ResourceKeyResult[] resultRef = + new EncryptedResourceKeyManager.ResourceKeyResult[1]; + final org.zstack.header.errorcode.ErrorCode[] errorRef = + new org.zstack.header.errorcode.ErrorCode[1]; + encryptedResourceKeyManager.getOrCreateKey(ctx, + new org.zstack.header.core.ReturnValueCompletion(null) { + @Override + public void success(EncryptedResourceKeyManager.ResourceKeyResult returnValue) { + resultRef[0] = returnValue; + } + + @Override + public void fail(org.zstack.header.errorcode.ErrorCode errorCode) { + errorRef[0] = errorCode; + } + }); + + if (errorRef[0] != null) { + throw new OperationFailureException(operr( + "failed to materialize encryption key for temporary snapshot image[uuid:%s]", + imageUuid).withCause(errorRef[0])); + } + if (resultRef[0] == null || StringUtils.isBlank(resultRef[0].getDekBase64())) { + throw new OperationFailureException(operr( + "key manager returned empty DEK for encrypted temporary snapshot image[uuid:%s]", imageUuid)); + } + return resultRef[0]; + } + +} diff --git a/storage/src/main/java/org/zstack/storage/primary/PrimaryStorageBase.java b/storage/src/main/java/org/zstack/storage/primary/PrimaryStorageBase.java index d54bc3d9c85..e94bf01ec58 100755 --- a/storage/src/main/java/org/zstack/storage/primary/PrimaryStorageBase.java +++ b/storage/src/main/java/org/zstack/storage/primary/PrimaryStorageBase.java @@ -52,6 +52,7 @@ import org.zstack.header.vm.*; import org.zstack.header.vm.metadata.*; import org.zstack.header.volume.*; +import org.zstack.storage.encrypt.VolumeEncryptedTrashCleanupHelper; import org.zstack.storage.volume.VolumeUtils; import org.zstack.utils.CollectionDSL; import org.zstack.utils.DebugUtils; @@ -100,6 +101,8 @@ public abstract class PrimaryStorageBase extends AbstractPrimaryStorage { @Autowired protected PrimaryStoragePhysicalCapacityManager physicalCapacityMgr; @Autowired + private VolumeEncryptedTrashCleanupHelper volumeEncryptedTrashCleanupHelper; + @Autowired private PluginRegistry pluginRgty; public PrimaryStorageBase() { @@ -178,6 +181,13 @@ public void setNewAdded(boolean newAdded) { protected abstract void handle(GetVolumeSnapshotEncryptedOnPrimaryStorageMsg msg); + protected void handle(ConvertVolumeEncryptionOnPrimaryStorageMsg msg) { + ConvertVolumeEncryptionOnPrimaryStorageReply reply = new ConvertVolumeEncryptionOnPrimaryStorageReply(); + reply.setError(operr("primary storage[type:%s, uuid:%s] does not support volume encryption conversion", + self.getType(), self.getUuid())); + bus.reply(msg, reply); + } + public PrimaryStorageBase(PrimaryStorageVO self) { this.self = self; } @@ -190,7 +200,9 @@ protected String getSyncId() { return String.format("primaryStorage-%s", self.getUuid()); } - protected static List trashLists = CollectionDSL.list(TrashType.MigrateVolume, TrashType.MigrateVolumeSnapshot, TrashType.RevertVolume, TrashType.VolumeSnapshot, TrashType.ReimageVolume); + protected static List trashLists = CollectionDSL.list(TrashType.MigrateVolume, TrashType.MigrateVolumeSnapshot, + TrashType.ConvertVolumeEncryption, TrashType.ConvertVolumeSnapshotEncryption, + TrashType.RevertVolume, TrashType.VolumeSnapshot, TrashType.ReimageVolume); protected void fireDisconnectedCanonicalEvent(ErrorCode reason) { PrimaryStorageCanonicalEvent.DisconnectedData data = new PrimaryStorageCanonicalEvent.DisconnectedData(); @@ -417,6 +429,8 @@ protected void handleLocalMessage(Message msg) { handle((UnlinkBitsOnPrimaryStorageMsg) msg); } else if (msg instanceof GetVolumeSnapshotEncryptedOnPrimaryStorageMsg) { handle((GetVolumeSnapshotEncryptedOnPrimaryStorageMsg) msg); + } else if (msg instanceof ConvertVolumeEncryptionOnPrimaryStorageMsg) { + handle((ConvertVolumeEncryptionOnPrimaryStorageMsg) msg); } else if (msg instanceof DeleteVolumeChainOnPrimaryStorageMsg) { handle((DeleteVolumeChainOnPrimaryStorageMsg) msg); } else if (msg instanceof CleanUpStorageTrashOnPrimaryStorageMsg) { @@ -1095,6 +1109,7 @@ public void run(MessageReply reply) { imsg.setAllocatedInstallUrl(allocatedInstallUrl); bus.makeTargetServiceIdByResourceUuid(imsg, PrimaryStorageConstant.SERVICE_ID, self.getUuid()); bus.send(imsg); + volumeEncryptedTrashCleanupHelper.cleanupKeyRefAfterTrashDeleted(inv); trash.removeFromDb(trashId); logger.info(String.format("Returned space[size:%s] to PS %s after volume migration", inv.getSize(), self.getUuid())); completion.success(new TrashCleanupResult(inv.getResourceUuid(), inv.getTrashId(), inv.getSize())); diff --git a/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotManagerImpl.java b/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotManagerImpl.java index 1f936ca6141..e39c8c74b9c 100755 --- a/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotManagerImpl.java +++ b/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotManagerImpl.java @@ -737,6 +737,7 @@ private VolumeSnapshotStruct getVolumeSnapshotStruct(CreateVolumeSnapshotMsg msg vo.setDescription(msg.getDescription()); vo.setVolumeUuid(msg.getVolumeUuid()); vo.setFormat(vol.getFormat()); + vo.setEncrypted(vol.isEncrypted()); vo.setState(VolumeSnapshotState.Enabled); vo.setStatus(VolumeSnapshotStatus.Creating); vo.setVolumeType(vol.getType().toString()); @@ -991,6 +992,9 @@ public void handle(Map data) { markSnapshotTreeCompleted(snapshot); if (volumeNewInstallPath != null) { vol.setInstallPath(volumeNewInstallPath); + if (Boolean.TRUE.equals(snapshot.getEncrypted())) { + vol.setEncrypted(true); + } dbf.update(vol); } @@ -1000,6 +1004,7 @@ public void handle(Map data) { svo.setPrimaryStorageInstallPath(snapshot.getPrimaryStorageInstallPath()); svo.setStatus(VolumeSnapshotStatus.Ready); svo.setSize(snapshot.getSize()); + svo.setEncrypted(Boolean.TRUE.equals(snapshot.getEncrypted())); if (snapshot.getFormat() != null) { svo.setFormat(snapshot.getFormat()); } diff --git a/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotTreeBase.java b/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotTreeBase.java index beb8b044d1a..ad6ca7520ac 100755 --- a/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotTreeBase.java +++ b/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotTreeBase.java @@ -1842,6 +1842,7 @@ private void createImageCache(final CreateImageCacheFromVolumeSnapshotMsg msg, f CreateImageCacheFromVolumeSnapshotOnPrimaryStorageMsg cmsg = new CreateImageCacheFromVolumeSnapshotOnPrimaryStorageMsg(); cmsg.setImageInventory(image); cmsg.setVolumeSnapshot(VolumeSnapshotInventory.valueOf(currentRoot)); + cmsg.setEncrypted(msg.getEncrypted()); cmsg.setSystemTags(msg.getSystemTags()); bus.makeTargetServiceIdByResourceUuid(cmsg, PrimaryStorageConstant.SERVICE_ID, cmsg.getPrimaryStorageUuid()); bus.send(cmsg, new CloudBusCallBack(msg, completion) { diff --git a/storage/src/main/java/org/zstack/storage/volume/VolumeApiInterceptor.java b/storage/src/main/java/org/zstack/storage/volume/VolumeApiInterceptor.java index 09e41c229f7..3c6b85a2c2b 100755 --- a/storage/src/main/java/org/zstack/storage/volume/VolumeApiInterceptor.java +++ b/storage/src/main/java/org/zstack/storage/volume/VolumeApiInterceptor.java @@ -54,6 +54,7 @@ import org.zstack.header.volume.APIAttachDataVolumeToHostMsg; import org.zstack.header.volume.APIAttachDataVolumeToVmMsg; import org.zstack.header.volume.APIBackupDataVolumeMsg; +import org.zstack.header.volume.APIChangeVolumeEncryptionMsg; import org.zstack.header.volume.APIChangeVolumeStateMsg; import org.zstack.header.volume.APICreateDataVolumeFromVolumeTemplateMsg; import org.zstack.header.volume.APICreateDataVolumeMsg; @@ -147,6 +148,8 @@ public APIMessage intercept(APIMessage msg) throws ApiMessageInterceptionExcepti validate((APIDetachDataVolumeFromHostMsg) msg); } else if (msg instanceof APIFlattenVolumeMsg) { validate((APIFlattenVolumeMsg) msg); + } else if (msg instanceof APIChangeVolumeEncryptionMsg) { + validate((APIChangeVolumeEncryptionMsg) msg); } else if (msg instanceof APIUndoSnapshotCreationMsg) { validate((APIUndoSnapshotCreationMsg) msg); } else if (msg instanceof APICreateVmInstanceMsg) { @@ -605,6 +608,44 @@ private void validate(APIFlattenVolumeMsg msg) { } } + private void validate(APIChangeVolumeEncryptionMsg msg) { + Tuple t = Q.New(VolumeVO.class) + .select(VolumeVO_.status, VolumeVO_.state, VolumeVO_.vmInstanceUuid) + .eq(VolumeVO_.uuid, msg.getVolumeUuid()) + .findTuple(); + VolumeStatus status = (VolumeStatus) t.get(0); + VolumeState state = (VolumeState) t.get(1); + String vmUuid = (String) t.get(2); + + if (status != VolumeStatus.Ready) { + throw new ApiMessageInterceptionException(operr( + "volume[uuid:%s] is not in status Ready, current is %s, cannot change encryption", + msg.getVolumeUuid(), status)); + } + if (state != VolumeState.Enabled) { + throw new ApiMessageInterceptionException(operr( + "volume[uuid:%s] is not in state Enabled, current is %s, cannot change encryption", + msg.getVolumeUuid(), state)); + } + if (Q.New(VolumeHostRefVO.class).eq(VolumeHostRefVO_.volumeUuid, msg.getVolumeUuid()).isExists()) { + throw new ApiMessageInterceptionException(operr( + "volume[uuid:%s] is attached to a host, cannot change encryption", msg.getVolumeUuid())); + } + if (vmUuid == null) { + return; + } + + VmInstanceState vmState = Q.New(VmInstanceVO.class) + .select(VmInstanceVO_.state) + .eq(VmInstanceVO_.uuid, vmUuid) + .findValue(); + if (vmState != VmInstanceState.Stopped) { + throw new ApiMessageInterceptionException(operr( + "volume[uuid:%s] is attached to vm[uuid:%s] whose state is %s, only Stopped is allowed to change encryption", + msg.getVolumeUuid(), vmUuid, vmState)); + } + } + private void validate(APIUndoSnapshotCreationMsg msg) { String currentTreeUuid = Q.New(VolumeSnapshotTreeVO.class) .select(VolumeSnapshotTreeVO_.uuid) diff --git a/storage/src/main/java/org/zstack/storage/volume/VolumeBase.java b/storage/src/main/java/org/zstack/storage/volume/VolumeBase.java index f0cd6d4a511..e65d729d9cc 100755 --- a/storage/src/main/java/org/zstack/storage/volume/VolumeBase.java +++ b/storage/src/main/java/org/zstack/storage/volume/VolumeBase.java @@ -1,5 +1,6 @@ package org.zstack.storage.volume; +import org.apache.commons.lang.StringUtils; import org.springframework.beans.factory.annotation.Autowire; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Configurable; @@ -51,6 +52,8 @@ import org.zstack.storage.primary.EstimateVolumeTemplateSizeOnPrimaryStorageMsg; import org.zstack.storage.primary.EstimateVolumeTemplateSizeOnPrimaryStorageReply; import org.zstack.storage.primary.PrimaryStorageGlobalConfig; +import org.zstack.storage.encrypt.VolumeEncryptedResourceKeyBackend; +import org.zstack.storage.encrypt.VolumeEncryptedSecretHelper; import org.zstack.storage.snapshot.group.VolumeSnapshotGroupOperationValidator; import org.zstack.storage.snapshot.reference.VolumeSnapshotReferenceUtils; import org.zstack.tag.SystemTagCreator; @@ -63,6 +66,7 @@ import org.zstack.utils.path.PathUtil; import javax.persistence.TypedQuery; +import java.io.File; import java.util.*; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -103,6 +107,12 @@ public class VolumeBase extends AbstractVolume implements Volume { private VmInstanceResourceMetadataManager vidm; @Autowired private StorageTrash trash; + @Autowired + private VolumeInPlaceEncryptor volumeInPlaceEncryptor; + @Autowired + private VolumeEncryptedResourceKeyBackend volumeEncryptedResourceKeyBackend; + @Autowired + private VolumeEncryptedSecretHelper volumeEncryptedSecretHelper; public VolumeBase(VolumeVO vo) { self = vo; @@ -170,6 +180,10 @@ private void handleLocalMessage(Message msg) { handle((FlattenVolumeMsg) msg); } else if (msg instanceof CancelFlattenVolumeMsg) { handle((CancelFlattenVolumeMsg) msg); + } else if (msg instanceof EncryptVolumeMsg) { + handle((EncryptVolumeMsg) msg); + } else if (msg instanceof ChangeVolumeEncryptionMsg) { + handle((ChangeVolumeEncryptionMsg) msg); } else { bus.dealWithUnknownMessage(msg); } @@ -270,6 +284,7 @@ public void run(final FlowTrigger trigger, Map data) { rmsg.setVolume(rootVolumeInventory); rmsg.setOriginSize(originSize); rmsg.setAllocatedInstallUrl(allocatedInstallUrl); + rmsg.setHostUuid(msg.getHostUuid()); bus.makeTargetServiceIdByResourceUuid(rmsg, PrimaryStorageConstant.SERVICE_ID, rootVolumeInventory.getPrimaryStorageUuid()); bus.send(rmsg, new CloudBusCallBack(trigger) { @Override @@ -609,6 +624,7 @@ private void prepareMsg(InstantiateVolumeMsg msg, InstantiateVolumeOnPrimaryStor imsg.setSystemTags(msg.getSystemTags()); imsg.setSkipIfExisting(msg.isSkipIfExisting()); imsg.setAllocatedInstallUrl(msg.getAllocatedInstallUrl()); + imsg.setVolumeLuksAgentSpec(msg.getVolumeLuksAgentSpec()); if (msg.getHostUuid() != null) { imsg.setDestHost(HostInventory.valueOf(dbf.findByUuid(msg.getHostUuid(), HostVO.class))); } @@ -2170,6 +2186,8 @@ private void handleApiMessage(APIMessage msg) { handle((APIDetachDataVolumeFromHostMsg) msg); } else if (msg instanceof APIFlattenVolumeMsg) { handle((APIFlattenVolumeMsg) msg); + } else if (msg instanceof APIChangeVolumeEncryptionMsg) { + handle((APIChangeVolumeEncryptionMsg) msg); } else if (msg instanceof APIUndoSnapshotCreationMsg) { handle((APIUndoSnapshotCreationMsg) msg); } else { @@ -3216,6 +3234,462 @@ public void run(MessageReply reply) { }); } + private void handle(APIChangeVolumeEncryptionMsg msg) { + ChangeVolumeEncryptionMsg cmsg = new ChangeVolumeEncryptionMsg(); + cmsg.setVolumeUuid(msg.getUuid()); + cmsg.setEncrypted(msg.isEncrypted()); + bus.makeTargetServiceIdByResourceUuid(cmsg, VolumeConstant.SERVICE_ID, msg.getVolumeUuid()); + bus.send(cmsg, new CloudBusCallBack(msg) { + @Override + public void run(MessageReply reply) { + APIChangeVolumeEncryptionEvent evt = new APIChangeVolumeEncryptionEvent(msg.getId()); + if (!reply.isSuccess()) { + evt.setError(reply.getError()); + } else { + ChangeVolumeEncryptionReply cr = reply.castReply(); + evt.setInventory(cr.getInventory()); + } + bus.publish(evt); + } + }); + } + + private void handle(ChangeVolumeEncryptionMsg msg) { + refreshVO(); + ChangeVolumeEncryptionReply reply = new ChangeVolumeEncryptionReply(); + if (self.isEncrypted() == msg.isEncrypted()) { + reply.setInventory(getSelfInventory()); + bus.reply(msg, reply); + return; + } + + thdf.chainSubmit(new ChainTask(msg) { + @Override + public String getSyncSignature() { + return String.format("change-volume-encryption-%s", msg.getVolumeUuid()); + } + + @Override + public void run(SyncTaskChain chain) { + changeVolumeEncryption(msg.isEncrypted(), new ReturnValueCompletion(chain) { + @Override + public void success(VolumeInventory inventory) { + reply.setInventory(inventory); + bus.reply(msg, reply); + chain.next(); + } + + @Override + public void fail(ErrorCode errorCode) { + reply.setError(errorCode); + bus.reply(msg, reply); + chain.next(); + } + }); + } + + @Override + public String getName() { + return getSyncSignature(); + } + }); + } + + private void changeVolumeEncryption(boolean targetEncrypted, ReturnValueCompletion completion) { + boolean sourceEncrypted = self.isEncrypted(); + List snapshots = Q.New(VolumeSnapshotVO.class) + .eq(VolumeSnapshotVO_.volumeUuid, self.getUuid()) + .list(); + snapshots.sort(Comparator + .comparingInt(VolumeSnapshotVO::getDistance) + .thenComparing(VolumeSnapshotVO::getUuid)); + + Map oldAndNewInstallPaths = buildEncryptionConversionInstallPaths(snapshots); + Map oldAndTrashInstallPaths = buildEncryptionConversionSourceTrashInstallPaths(snapshots); + VolumeInventory oldVolumeInventory = VolumeInventory.valueOf(self); + oldVolumeInventory.setInstallPath(oldAndTrashInstallPaths.get(self.getInstallPath())); + List oldSnapshotInventories = snapshots.stream() + .filter(snapshot -> StringUtils.isNotBlank(snapshot.getPrimaryStorageInstallPath())) + .filter(snapshot -> StringUtils.isNotBlank(oldAndTrashInstallPaths.get(snapshot.getPrimaryStorageInstallPath()))) + .map(snapshot -> { + VolumeSnapshotInventory inv = VolumeSnapshotInventory.valueOf(snapshot); + inv.setPrimaryStorageInstallPath(oldAndTrashInstallPaths.get(snapshot.getPrimaryStorageInstallPath())); + return inv; + }) + .collect(Collectors.toList()); + List items = + buildEncryptionConversionItems(snapshots, oldAndNewInstallPaths, oldAndTrashInstallPaths); + + FlowChain chain = FlowChainBuilder.newShareFlowChain(); + chain.setName(String.format("change-volume-%s-encryption-to-%s", self.getUuid(), targetEncrypted)); + chain.then(new ShareFlow() { + boolean createdKeyBinding; + + @Override + public void setup() { + flow(new Flow() { + String __name__ = "prepare-target-volume-key"; + + @Override + public void run(FlowTrigger trigger, Map data) { + createdKeyBinding = ensureVolumeKeyBindingIfNeeded(targetEncrypted); + trigger.next(); + } + + @Override + public void rollback(FlowRollback trigger, Map data) { + if (createdKeyBinding) { + volumeEncryptedResourceKeyBackend.detachKeyProviderFromVolume(self.getUuid()); + } + trigger.rollback(); + } + }); + + flow(new Flow() { + String __name__ = "convert-volume-encryption-on-primary-storage"; + + @Override + public void run(FlowTrigger trigger, Map data) { + ConvertVolumeEncryptionOnPrimaryStorageMsg cmsg = new ConvertVolumeEncryptionOnPrimaryStorageMsg(); + cmsg.setPrimaryStorageUuid(self.getPrimaryStorageUuid()); + cmsg.setVolume(VolumeInventory.valueOf(self)); + cmsg.setTargetEncrypted(targetEncrypted); + cmsg.setItems(items); + bus.makeTargetServiceIdByResourceUuid(cmsg, PrimaryStorageConstant.SERVICE_ID, self.getPrimaryStorageUuid()); + bus.send(cmsg, new CloudBusCallBack(trigger) { + @Override + public void run(MessageReply reply) { + if (reply.isSuccess()) { + ConvertVolumeEncryptionOnPrimaryStorageReply cr = reply.castReply(); + data.put("actualSizes", cr.getActualSizes()); + trigger.next(); + } else { + trigger.fail(reply.getError()); + } + } + }); + } + + @Override + public void rollback(FlowRollback trigger, Map data) { + deleteConvertedVolumeEncryptionBits(items); + trigger.rollback(); + } + }); + + flow(new Flow() { + String __name__ = "update-volume-encryption-in-db"; + + @Override + public void run(FlowTrigger trigger, Map data) { + updateVolumeEncryptionConversionInDb(targetEncrypted, snapshots, oldAndNewInstallPaths, + (Map) data.get("actualSizes")); + refreshVO(); + trigger.next(); + } + + @Override + public void rollback(FlowRollback trigger, Map data) { + trigger.rollback(); + } + }); + + flow(new NoRollbackFlow() { + String __name__ = "cleanup-old-libvirt-secret"; + + @Override + public void run(FlowTrigger trigger, Map data) { + if (!sourceEncrypted || targetEncrypted) { + trigger.next(); + return; + } + + Integer keyVersion = volumeEncryptedResourceKeyBackend.findKeyVersionByVolume(self.getUuid()); + if (keyVersion == null) { + trigger.next(); + return; + } + + String hostUuid = null; + List tags = VolumeSystemTags.VOLUME_LIBVIRT_SECRET_HOST.getTags(self.getUuid(), VolumeVO.class); + if (tags != null && !tags.isEmpty()) { + hostUuid = VolumeSystemTags.VOLUME_LIBVIRT_SECRET_HOST.getTokenByTag( + tags.get(0), VolumeSystemTags.VOLUME_LIBVIRT_SECRET_HOST_TOKEN); + } + + String vmUuid = self.getVmInstanceUuid(); + if (StringUtils.isBlank(hostUuid) && StringUtils.isNotBlank(vmUuid)) { + hostUuid = Q.New(VmInstanceVO.class) + .eq(VmInstanceVO_.uuid, vmUuid) + .select(VmInstanceVO_.hostUuid) + .findValue(); + } + if (StringUtils.isBlank(hostUuid) && StringUtils.isNotBlank(vmUuid)) { + hostUuid = Q.New(VmInstanceVO.class) + .eq(VmInstanceVO_.uuid, vmUuid) + .select(VmInstanceVO_.lastHostUuid) + .findValue(); + } + + volumeEncryptedSecretHelper.deleteSecretOnHostBestEffort( + hostUuid, StringUtils.defaultIfBlank(vmUuid, self.getUuid()), + self.getUuid(), keyVersion); + trigger.next(); + } + }); + + flow(new NoRollbackFlow() { + String __name__ = "record-old-volume-bits-in-trash"; + + @Override + public void run(FlowTrigger trigger, Map data) { + trash.createTrash(TrashType.ConvertVolumeEncryption, false, oldVolumeInventory); + for (VolumeSnapshotInventory snapshot : oldSnapshotInventories) { + trash.createTrash(TrashType.ConvertVolumeSnapshotEncryption, false, snapshot); + } + trigger.next(); + } + }); + } + }).done(new FlowDoneHandler(completion) { + @Override + public void handle(Map data) { + completion.success(getSelfInventory()); + } + }).error(new FlowErrorHandler(completion) { + @Override + public void handle(ErrorCode errCode, Map data) { + completion.fail(errCode); + } + }).start(); + } + + private boolean ensureVolumeKeyBindingIfNeeded(boolean targetEncrypted) { + if (!targetEncrypted || volumeEncryptedResourceKeyBackend.checkVolumeKeyProviderAttached(self.getUuid())) { + return false; + } + + String kpUuid = volumeEncryptedResourceKeyBackend.defaultKeyProviderUuid(); + if (StringUtils.isBlank(kpUuid)) { + throw new OperationFailureException(operr( + "cannot encrypt volume[uuid:%s]: no key provider bound and no default key provider configured", + self.getUuid())); + } + volumeEncryptedResourceKeyBackend.attachKeyProviderToVolume(self.getUuid(), kpUuid); + return true; + } + + private Map buildEncryptionConversionInstallPaths(List snapshots) { + Map paths = new LinkedHashMap<>(); + for (VolumeSnapshotVO snapshot : snapshots) { + if (StringUtils.isNotBlank(snapshot.getPrimaryStorageInstallPath())) { + paths.put(snapshot.getPrimaryStorageInstallPath(), + makeConvertedSnapshotInstallPath(snapshot.getPrimaryStorageInstallPath(), snapshot.getUuid())); + } + } + paths.put(self.getInstallPath(), makeConvertedVolumeInstallPath(self.getInstallPath(), self.getUuid())); + return paths; + } + + private Map buildEncryptionConversionSourceTrashInstallPaths(List snapshots) { + Map paths = new LinkedHashMap<>(); + Set trashPaths = new HashSet<>(); + for (VolumeSnapshotVO snapshot : snapshots) { + if (StringUtils.isNotBlank(snapshot.getPrimaryStorageInstallPath())) { + String sourceTrashPath = makeSourceSnapshotTrashInstallPath(snapshot.getPrimaryStorageInstallPath(), snapshot.getUuid(), + snapshot.isEncrypted(), trashPaths); + if (StringUtils.isNotBlank(sourceTrashPath)) { + paths.put(snapshot.getPrimaryStorageInstallPath(), sourceTrashPath); + } + } + } + paths.put(self.getInstallPath(), makeSourceTrashInstallPath(self.getInstallPath(), self.getUuid(), self.isEncrypted(), trashPaths)); + return paths; + } + + private List buildEncryptionConversionItems( + List snapshots, Map oldAndNewInstallPaths, + Map oldAndTrashInstallPaths) { + Map snapshotMap = snapshots.stream() + .collect(Collectors.toMap(VolumeSnapshotVO::getUuid, it -> it)); + List items = new ArrayList<>(); + + for (VolumeSnapshotVO snapshot : snapshots) { + if (StringUtils.isBlank(snapshot.getPrimaryStorageInstallPath())) { + continue; + } + VolumeSnapshotVO parent = snapshot.getParentUuid() == null ? null : snapshotMap.get(snapshot.getParentUuid()); + String newBackingPath = parent == null || isCephInstallPath(snapshot.getPrimaryStorageInstallPath()) ? + null : oldAndNewInstallPaths.get(parent.getPrimaryStorageInstallPath()); + // When parent is null, no target backing is set and the converted base snapshot becomes full. + items.add(makeEncryptionConversionItem(snapshot.getUuid(), VolumeSnapshotVO.class.getSimpleName(), + snapshot.getPrimaryStorageInstallPath(), oldAndNewInstallPaths.get(snapshot.getPrimaryStorageInstallPath()), + oldAndTrashInstallPaths.get(snapshot.getPrimaryStorageInstallPath()), newBackingPath)); + } + + String activeBackingPath = null; + String currentTreeUuid = Q.New(VolumeSnapshotTreeVO.class) + .select(VolumeSnapshotTreeVO_.uuid) + .eq(VolumeSnapshotTreeVO_.volumeUuid, self.getUuid()) + .eq(VolumeSnapshotTreeVO_.current, true) + .findValue(); + if (StringUtils.isNotBlank(currentTreeUuid)) { + String latestSnapshotPath = Q.New(VolumeSnapshotVO.class) + .select(VolumeSnapshotVO_.primaryStorageInstallPath) + .eq(VolumeSnapshotVO_.treeUuid, currentTreeUuid) + .eq(VolumeSnapshotVO_.latest, true) + .findValue(); + activeBackingPath = isCephInstallPath(latestSnapshotPath) ? null : oldAndNewInstallPaths.get(latestSnapshotPath); + } + items.add(makeEncryptionConversionItem(self.getUuid(), VolumeVO.class.getSimpleName(), + self.getInstallPath(), oldAndNewInstallPaths.get(self.getInstallPath()), + oldAndTrashInstallPaths.get(self.getInstallPath()), activeBackingPath)); + return items; + } + + private ConvertVolumeEncryptionOnPrimaryStorageMsg.VolumeEncryptionConversionItem makeEncryptionConversionItem( + String resourceUuid, String resourceType, String sourcePath, String targetPath, + String sourceTrashPath, String targetBackingPath) { + ConvertVolumeEncryptionOnPrimaryStorageMsg.VolumeEncryptionConversionItem item = + new ConvertVolumeEncryptionOnPrimaryStorageMsg.VolumeEncryptionConversionItem(); + item.setResourceUuid(resourceUuid); + item.setResourceType(resourceType); + item.setSourceInstallPath(sourcePath); + item.setSourceTrashInstallPath(sourceTrashPath); + item.setTargetInstallPath(targetPath); + item.setTargetBackingInstallPath(targetBackingPath); + return item; + } + + private String makeConvertedVolumeInstallPath(String installPath, String resourceUuid) { + if (isCephInstallPath(installPath)) { + return makeCephVolumeInstallPath(installPath, resourceUuid); + } + if (isSharedBlockInstallPath(installPath) || installPath.startsWith("/dev/")) { + return makeSiblingInstallPath(installPath, resourceUuid); + } + return makeSiblingInstallPath(installPath, String.format("%s.qcow2", resourceUuid)); + } + + private String makeConvertedSnapshotInstallPath(String installPath, String resourceUuid) { + if (isCephInstallPath(installPath)) { + return String.format("%s@%s", makeCephVolumeInstallPath(installPath, self.getUuid()), resourceUuid); + } + if (isSharedBlockInstallPath(installPath) || installPath.startsWith("/dev/")) { + return makeSiblingInstallPath(installPath, resourceUuid); + } + return makeSiblingInstallPath(installPath, String.format("%s.qcow2", resourceUuid)); + } + + private String makeSiblingInstallPath(String installPath, String fileName) { + if (isSharedBlockInstallPath(installPath)) { + String prefix = "sharedblock://"; + String path = installPath.substring(prefix.length()); + int vgEnd = path.indexOf('/'); + if (vgEnd < 0) { + return installPath; + } + return String.format("%s%s/%s", prefix, path.substring(0, vgEnd), fileName); + } + File file = new File(installPath); + File parent = file.getParentFile(); + return parent == null ? fileName : new File(parent, fileName).getPath(); + } + + private String makeCephVolumeInstallPath(String installPath, String resourceUuid) { + String imagePath = installPath.split("@", 2)[0]; + String prefix = "ceph://"; + String path = imagePath.substring(prefix.length()); + int poolEnd = path.indexOf('/'); + if (poolEnd < 0) { + return imagePath; + } + return String.format("%s%s/%s", prefix, path.substring(0, poolEnd), resourceUuid); + } + + private String makeSourceSnapshotTrashInstallPath(String installPath, String resourceUuid, boolean sourceEncrypted, Set trashPaths) { + if (isCephInstallPath(installPath)) { + return null; + } + return makeSourceTrashInstallPath(installPath, resourceUuid, sourceEncrypted, trashPaths); + } + + private String makeSourceTrashInstallPath(String installPath, String resourceUuid, boolean sourceEncrypted, Set trashPaths) { + String trashPath; + do { + String fileName = String.format("%s.trash.%s.%s.qcow2", + resourceUuid, sourceEncrypted ? "encrypted" : "plain", Platform.getUuid().substring(0, 6)); + if (isCephInstallPath(installPath)) { + trashPath = makeCephVolumeInstallPath(installPath, fileName); + } else { + trashPath = makeSiblingInstallPath(installPath, fileName); + } + } while (!trashPaths.add(trashPath)); + return trashPath; + } + + private boolean isCephInstallPath(String installPath) { + return installPath != null && installPath.startsWith("ceph://"); + } + + private boolean isSharedBlockInstallPath(String installPath) { + return installPath != null && installPath.startsWith("sharedblock://"); + } + + private void deleteConvertedVolumeEncryptionBits(List items) { + Set sourceInstallPaths = items.stream() + .map(ConvertVolumeEncryptionOnPrimaryStorageMsg.VolumeEncryptionConversionItem::getSourceInstallPath) + .collect(Collectors.toSet()); + for (ConvertVolumeEncryptionOnPrimaryStorageMsg.VolumeEncryptionConversionItem item : items) { + if (sourceInstallPaths.contains(item.getTargetInstallPath())) { + continue; + } + DeleteVolumeBitsOnPrimaryStorageMsg dmsg = new DeleteVolumeBitsOnPrimaryStorageMsg(); + dmsg.setPrimaryStorageUuid(self.getPrimaryStorageUuid()); + dmsg.setInstallPath(item.getTargetInstallPath()); + dmsg.setBitsUuid(item.getResourceUuid()); + dmsg.setBitsType(item.getResourceType()); + dmsg.setHypervisorType(VolumeFormat.getMasterHypervisorTypeByVolumeFormat(self.getFormat()).toString()); + bus.makeTargetServiceIdByResourceUuid(dmsg, PrimaryStorageConstant.SERVICE_ID, self.getPrimaryStorageUuid()); + bus.send(dmsg); + } + } + + @Transactional + private void updateVolumeEncryptionConversionInDb(boolean targetEncrypted, List snapshots, + Map oldAndNewInstallPaths, + Map actualSizes) { + VolumeSnapshotReferenceUtils.handleVolumeInstallUrlChange(self.getUuid(), oldAndNewInstallPaths); + + UpdateQuery q = SQL.New(VolumeVO.class) + .eq(VolumeVO_.uuid, self.getUuid()) + .set(VolumeVO_.installPath, oldAndNewInstallPaths.get(self.getInstallPath())) + .set(VolumeVO_.encrypted, targetEncrypted); + if (actualSizes != null && actualSizes.get(self.getUuid()) != null) { + q.set(VolumeVO_.actualSize, actualSizes.get(self.getUuid())); + } + q.update(); + + for (VolumeSnapshotVO snapshot : snapshots) { + String newPath = oldAndNewInstallPaths.get(snapshot.getPrimaryStorageInstallPath()); + if (StringUtils.isBlank(newPath)) { + continue; + } + SQL.New(VolumeSnapshotVO.class) + .eq(VolumeSnapshotVO_.uuid, snapshot.getUuid()) + .set(VolumeSnapshotVO_.primaryStorageInstallPath, newPath) + .set(VolumeSnapshotVO_.encrypted, targetEncrypted) + .update(); + SQL.New(VolumeSnapshotGroupRefVO.class) + .eq(VolumeSnapshotGroupRefVO_.volumeSnapshotUuid, snapshot.getUuid()) + .set(VolumeSnapshotGroupRefVO_.volumeSnapshotInstallPath, newPath) + .update(); + if (targetEncrypted) { + volumeEncryptedResourceKeyBackend.copyVolumeKeyToSnapshot(self.getUuid(), snapshot.getUuid()); + } + } + } + private void handle(FlattenVolumeMsg msg) { if (msg.isDryRun()) { FlattenVolumeReply reply = new FlattenVolumeReply(); @@ -3301,6 +3775,45 @@ public void fail(ErrorCode errorCode) { } } + /** + * Converts this volume's bits to LUKS-encrypted form in place. The heavy lifting + * (key materialization, host secret staging, PS-side LUKS conversion, DB update) + * is delegated to {@link VolumeInPlaceEncryptor} so both this message handler and + * {@code VolumeManagerImpl}'s create-data-volume-from-template flow share a single + * implementation. + */ + private void handle(EncryptVolumeMsg msg) { + EncryptVolumeReply reply = new EncryptVolumeReply(); + refreshVO(); + + if (self == null) { + reply.setError(operr("volume[uuid:%s] has been deleted", msg.getVolumeUuid())); + bus.reply(msg, reply); + return; + } + + VolumeInPlaceEncryptor.Context ctx = new VolumeInPlaceEncryptor.Context() + .setHostUuid(msg.getHostUuid()) + .setPrimaryStorageUuid(msg.getPrimaryStorageUuid()) + .setInstallPath(msg.getInstallPath()) + .setPurpose(msg.getPurpose()); + + volumeInPlaceEncryptor.encryptInPlace(self, ctx, new ReturnValueCompletion(msg) { + @Override + public void success(VolumeVO latest) { + self = latest; + reply.setInventory(getSelfInventory()); + bus.reply(msg, reply); + } + + @Override + public void fail(ErrorCode errorCode) { + reply.setError(errorCode); + bus.reply(msg, reply); + } + }); + } + @VmAttachVolumeValidatorMethod static void vmAttachVolumeValidator(VmInstanceInventory vmInv, String volumeUuid) { String vmUuid = vmInv.getUuid(); diff --git a/storage/src/main/java/org/zstack/storage/volume/VolumeInPlaceEncryptor.java b/storage/src/main/java/org/zstack/storage/volume/VolumeInPlaceEncryptor.java new file mode 100644 index 00000000000..b454fc9f55d --- /dev/null +++ b/storage/src/main/java/org/zstack/storage/volume/VolumeInPlaceEncryptor.java @@ -0,0 +1,264 @@ +package org.zstack.storage.volume; + +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Autowire; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Configurable; +import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.core.db.DatabaseFacade; +import org.zstack.header.core.ReturnValueCompletion; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.host.HostConstant; +import org.zstack.header.keyprovider.EncryptedResourceKeyManager; +import org.zstack.header.message.MessageReply; +import org.zstack.header.secret.SecretHostEnsureLuksSecretFileMsg; +import org.zstack.header.secret.SecretHostEnsureLuksSecretFileReply; +import org.zstack.header.storage.primary.EncryptVolumeBitsOnPrimaryStorageMsg; +import org.zstack.header.storage.primary.PrimaryStorageConstant; +import org.zstack.header.volume.VolumeVO; +import org.zstack.storage.encrypt.VolumeEncryptedResourceKeyBackend; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import static org.zstack.core.Platform.operr; + +/** + * Performs an in-place LUKS conversion of an existing volume's bits. + * + *

This is the single source of truth for the "encrypt-in-place" workflow that was + * previously duplicated between {@link VolumeBase#handleMessage} (for the + * {@code EncryptVolumeMsg} entry point) and + * {@link VolumeManagerImpl} (for the create-data-volume-from-template flow). + * + *

Steps performed: + *

    + *
  1. Ensure a key-provider binding exists for the volume; auto-attach the default + * provider when none is bound yet.
    2. + *
  2. Materialize a DEK via {@link EncryptedResourceKeyManager#getOrCreateKey}.
    3. + *
  3. Stage the LUKS secret material file on the target host via + * {@code SecretHostEnsureLuksSecretFileMsg}.
    4. + *
  4. Ask the primary storage backend to LUKS-convert the bits in place via + * {@code EncryptVolumeBitsOnPrimaryStorageMsg}.
    5. + *
  5. Persist {@code VolumeVO.encrypted = true} after a successful conversion.
    6. + *
+ * + *

Idempotency: when {@code volume.encrypted == true} the helper treats the volume as + * already converted and short-circuits with success. Callers must therefore not pre-mark + * the row encrypted before invoking this helper -- the encrypted flag is the single + * authoritative signal that "the bits on disk are already LUKS". + */ +@Configurable(preConstruction = true, autowire = Autowire.BY_TYPE) +public class VolumeInPlaceEncryptor { + private static final CLogger logger = Utils.getLogger(VolumeInPlaceEncryptor.class); + + @Autowired + private CloudBus bus; + @Autowired + private DatabaseFacade dbf; + @Autowired + private EncryptedResourceKeyManager encryptedResourceKeyManager; + @Autowired + private VolumeEncryptedResourceKeyBackend volumeEncryptedResourceKeyBackend; + + /** + * Inputs that don't live on {@link VolumeVO} (host to stage the secret on, overrides + * for installPath / primaryStorageUuid when the caller already knows them, etc.). + */ + public static class Context { + private String hostUuid; + /** Optional; falls back to {@code volume.getPrimaryStorageUuid()}. */ + private String primaryStorageUuid; + /** Optional; falls back to {@code volume.getInstallPath()}. */ + private String installPath; + /** Free-form purpose label for the DEK get-or-create audit trail. */ + private String purpose; + + public String getHostUuid() { + return hostUuid; + } + + public Context setHostUuid(String hostUuid) { + this.hostUuid = hostUuid; + return this; + } + + public String getPrimaryStorageUuid() { + return primaryStorageUuid; + } + + public Context setPrimaryStorageUuid(String primaryStorageUuid) { + this.primaryStorageUuid = primaryStorageUuid; + return this; + } + + public String getInstallPath() { + return installPath; + } + + public Context setInstallPath(String installPath) { + this.installPath = installPath; + return this; + } + + public String getPurpose() { + return purpose; + } + + public Context setPurpose(String purpose) { + this.purpose = purpose; + return this; + } + } + + /** + * Run the encrypt-in-place workflow. + * + * @param volume the (already-persisted) target volume; the caller is responsible + * for having a fresh row before invoking + * @param ctx host / installPath / purpose + * @param completion success returns the latest {@code VolumeVO} (encrypted row when + * the workflow actually ran, or the original row when it was a + * no-op short-circuit) + */ + public void encryptInPlace(VolumeVO volume, Context ctx, ReturnValueCompletion completion) { + if (volume == null) { + completion.fail(operr("encrypt-in-place: volume is null")); + return; + } + + // Idempotent short-circuit. The encrypted flag is the authoritative signal that + // the on-disk bits are already in LUKS form (this helper is the only place that + // flips it to true, and it does so only after a successful qemu-img convert). + if (volume.isEncrypted()) { + completion.success(volume); + return; + } + + if (StringUtils.isBlank(ctx.getHostUuid())) { + completion.fail(operr( + "cannot encrypt volume[uuid:%s] in place: hostUuid is required to stage LUKS secret", + volume.getUuid())); + return; + } + + final String installPath = StringUtils.isNotBlank(ctx.getInstallPath()) + ? ctx.getInstallPath() : volume.getInstallPath(); + if (StringUtils.isBlank(installPath)) { + completion.fail(operr( + "cannot encrypt volume[uuid:%s] in place: installPath unknown (volume not instantiated?)", + volume.getUuid())); + return; + } + + final String psUuid = StringUtils.isNotBlank(ctx.getPrimaryStorageUuid()) + ? ctx.getPrimaryStorageUuid() : volume.getPrimaryStorageUuid(); + if (StringUtils.isBlank(psUuid)) { + completion.fail(operr( + "cannot encrypt volume[uuid:%s] in place: primaryStorageUuid unknown", + volume.getUuid())); + return; + } + + // 1) Ensure a key-provider binding exists; auto-attach the default provider when missing. + // Binding is no longer eagerly performed at volume-create time, so this helper is the + // canonical attach point for the encrypt-from-template and encrypt-existing-volume paths + // (the regular instantiate path is covered by VolumeEncryptedInitialExtension). + String kpUuid = volumeEncryptedResourceKeyBackend.findKeyProviderUuidByVolume(volume.getUuid()); + if (StringUtils.isBlank(kpUuid)) { + kpUuid = volumeEncryptedResourceKeyBackend.defaultKeyProviderUuid(); + if (StringUtils.isBlank(kpUuid)) { + completion.fail(operr( + "cannot encrypt volume[uuid:%s] in place: no key provider bound and no default key provider configured", + volume.getUuid())); + return; + } + volumeEncryptedResourceKeyBackend.attachKeyProviderToVolume(volume.getUuid(), kpUuid); + } + + // 2) Materialize the DEK (idempotent: get-or-create). + EncryptedResourceKeyManager.GetOrCreateResourceKeyContext keyCtx = + new EncryptedResourceKeyManager.GetOrCreateResourceKeyContext(); + keyCtx.setResourceUuid(volume.getUuid()); + keyCtx.setResourceType(VolumeVO.class.getSimpleName()); + keyCtx.setKeyProviderUuid(kpUuid); + keyCtx.setPurpose(StringUtils.defaultIfBlank(ctx.getPurpose(), "encrypt-volume-in-place")); + + final EncryptedResourceKeyManager.ResourceKeyResult[] keyResultRef = + new EncryptedResourceKeyManager.ResourceKeyResult[1]; + final ErrorCode[] keyErrorRef = new ErrorCode[1]; + encryptedResourceKeyManager.getOrCreateKey(keyCtx, + new ReturnValueCompletion(completion) { + @Override + public void success(EncryptedResourceKeyManager.ResourceKeyResult r) { + keyResultRef[0] = r; + } + + @Override + public void fail(ErrorCode err) { + keyErrorRef[0] = err; + } + }); + if (keyErrorRef[0] != null) { + completion.fail(operr( + "failed to materialize encryption key for volume[uuid:%s]", + volume.getUuid()).withCause(keyErrorRef[0])); + return; + } + final String dekBase64 = keyResultRef[0].getDekBase64(); + if (StringUtils.isBlank(dekBase64)) { + completion.fail(operr( + "encryption key manager returned empty DEK for volume[uuid:%s]", + volume.getUuid())); + return; + } + + // 3) Stage the LUKS secret material file on the host. + SecretHostEnsureLuksSecretFileMsg ensureMsg = new SecretHostEnsureLuksSecretFileMsg(); + ensureMsg.setHostUuid(ctx.getHostUuid()); + ensureMsg.setDekBase64(dekBase64); + bus.makeTargetServiceIdByResourceUuid(ensureMsg, HostConstant.SERVICE_ID, ctx.getHostUuid()); + + MessageReply ensureReply = bus.call(ensureMsg); + if (!ensureReply.isSuccess()) { + completion.fail(operr( + "failed to prepare LUKS secret material file for volume[uuid:%s] on host[uuid:%s]", + volume.getUuid(), ctx.getHostUuid()) + .withCause(ensureReply.getError())); + return; + } + SecretHostEnsureLuksSecretFileReply er = ensureReply.castReply(); + if (StringUtils.isBlank(er.getSecFilePath())) { + completion.fail(operr( + "ensure LUKS secret file on host succeeded but secFilePath is empty, host[uuid:%s]", + ctx.getHostUuid())); + return; + } + final String secFilePath = er.getSecFilePath(); + + // 4) Ask the PS backend to LUKS-convert the bits in place. + EncryptVolumeBitsOnPrimaryStorageMsg emsg = new EncryptVolumeBitsOnPrimaryStorageMsg(); + emsg.setPrimaryStorageUuid(psUuid); + emsg.setHostUuid(ctx.getHostUuid()); + emsg.setVolumeUuid(volume.getUuid()); + emsg.setInstallPath(installPath); + emsg.setEncryptLuksSecretMaterialFilePath(secFilePath); + bus.makeTargetServiceIdByResourceUuid(emsg, PrimaryStorageConstant.SERVICE_ID, psUuid); + bus.send(emsg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply r) { + if (!r.isSuccess()) { + completion.fail(r.getError()); + return; + } + // 5) Persist encrypted=true. The short-circuit above guarantees we only + // reach here when the row was previously encrypted=false; this is the + // one and only place the flag flips, ensuring it always reflects the + // on-disk reality. + volume.setEncrypted(true); + VolumeVO latest = dbf.updateAndRefresh(volume); + completion.success(latest); + } + }); + } +} diff --git a/storage/src/main/java/org/zstack/storage/volume/VolumeManagerImpl.java b/storage/src/main/java/org/zstack/storage/volume/VolumeManagerImpl.java index 88aa63237c5..3bae878279e 100755 --- a/storage/src/main/java/org/zstack/storage/volume/VolumeManagerImpl.java +++ b/storage/src/main/java/org/zstack/storage/volume/VolumeManagerImpl.java @@ -42,6 +42,7 @@ import org.zstack.header.storage.snapshot.*; import org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupVO; import org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupVO_; +import org.zstack.storage.encrypt.VolumeSnapshotEncryptionHelper; import org.zstack.header.vm.*; import org.zstack.header.vm.devices.VmInstanceResourceMetadataManager; import org.zstack.header.volume.*; @@ -92,6 +93,10 @@ public class VolumeManagerImpl extends AbstractService implements VolumeManager, private PluginRegistry pluginRgty; @Autowired private VmInstanceResourceMetadataManager vidm; + @Autowired + private VolumeInPlaceEncryptor volumeInPlaceEncryptor; + @Autowired + private VolumeSnapshotEncryptionHelper snapshotEncryptionHelper; private Future volumeExpungeTask; @@ -239,6 +244,13 @@ private void handle(CreateDataVolumeFromVolumeTemplateMsg msg) { vol.setPrimaryStorageUuid(msg.getPrimaryStorageUuid()); vol.setAccountUuid(msg.getAccountUuid()); vol.setShareable(getShareableCapabilityFromMsg(msg)); + // Do not pre-mark encrypted here. The template bits we are about to download + // are plain; if we set encrypted=true now, encryptInPlace's idempotent + // short-circuit (volume.isEncrypted() => no-op) would skip the actual + // qemu-img convert and we'd be left with a row claiming encryption while + // the file is plain. encryptInPlace itself sets encrypted=true once the + // conversion actually succeeds. + vol.setEncrypted(false); if (msg.getSystemTags() != null) { Iterator iterators = msg.getSystemTags().iterator(); @@ -461,6 +473,57 @@ public void rollback(FlowRollback trigger, Map data) { } }); + flow(new NoRollbackFlow() { + String __name__ = String.format("encrypt data volume %s in place if needed", vol.getUuid()); + + @Override + public boolean skip(Map data) { + if (!Boolean.TRUE.equals(msg.getEncrypted())) { + return true; + } + // Like non-fast clone, template bits are already LUKS — skip encryptInPlace, + // key inheritance handled below + return isTemplateFromEncryptedSource(msg.getImageUuid()); + } + + @Override + public void run(FlowTrigger trigger, Map data) { + VolumeInPlaceEncryptor.Context ctx = new VolumeInPlaceEncryptor.Context() + .setHostUuid(msg.getHostUuid()) + .setPrimaryStorageUuid(targetPrimaryStorage.getUuid()) + .setInstallPath(primaryStorageInstallPath) + .setPurpose("create-data-volume-from-template"); + volumeInPlaceEncryptor.encryptInPlace(vol, ctx, new ReturnValueCompletion(trigger) { + @Override + public void success(VolumeVO latest) { + trigger.next(); + } + + @Override + public void fail(ErrorCode errorCode) { + trigger.fail(errorCode); + } + }); + } + }); + + flow(new NoRollbackFlow() { + String __name__ = String.format("inherit key for encrypted data volume %s from snapshot template", vol.getUuid()); + + @Override + public boolean skip(Map data) { + return !Boolean.TRUE.equals(msg.getEncrypted()) || !isTemplateFromEncryptedSource(msg.getImageUuid()); + } + + @Override + public void run(FlowTrigger trigger, Map data) { + VolumeVO latest = dbf.reload(vol); + snapshotEncryptionHelper.inheritFromTemporarySnapshotImageKeyIfPossible(latest); + SQL.New(VolumeVO.class).eq(VolumeVO_.uuid, vol.getUuid()).set(VolumeVO_.encrypted, true).update(); + trigger.next(); + } + }); + flow(new NoRollbackFlow() { String __name__ = String.format("sync volume %s size", vol.getUuid()); @@ -586,6 +649,7 @@ private VolumeInventory createVolume(CreateVolumeMsg msg) { vo.setStatus(VolumeStatus.NotInstantiated); vo.setType(VolumeType.valueOf(msg.getVolumeType())); vo.setDiskOfferingUuid(msg.getDiskOfferingUuid()); + vo.setEncrypted(Boolean.TRUE.equals(msg.getEncrypted())); if (vo.getType() == VolumeType.Root) { vo.setDeviceId(0); } @@ -638,6 +702,42 @@ protected VolumeVO scripts() { return inv; } + private boolean isTemplateFromEncryptedSource(String imageUuid) { + if (StringUtils.isBlank(imageUuid)) { + return false; + } + + String imageUrl = Q.New(ImageVO.class) + .eq(ImageVO_.uuid, imageUuid) + .select(ImageVO_.url) + .findValue(); + if (StringUtils.isBlank(imageUrl)) { + return false; + } + + if (imageUrl.startsWith("volume://")) { + String srcVolumeUuid = imageUrl.substring("volume://".length()); + return Boolean.TRUE.equals(Q.New(VolumeVO.class) + .eq(VolumeVO_.uuid, srcVolumeUuid) + .select(VolumeVO_.encrypted) + .findValue()); + } + + String snapshotUuid; + if (imageUrl.startsWith(ImageConstant.IMAGE_FROM_SNAPSHOT_SCHEMA)) { + snapshotUuid = imageUrl.substring(ImageConstant.IMAGE_FROM_SNAPSHOT_SCHEMA.length()); + } else if (imageUrl.startsWith(ImageConstant.SNAPSHOT_REUSE_IMAGE_SCHEMA)) { + snapshotUuid = imageUrl.substring(ImageConstant.SNAPSHOT_REUSE_IMAGE_SCHEMA.length()); + } else { + return false; + } + snapshotUuid = snapshotUuid.length() >= 32 ? snapshotUuid.substring(0, 32) : snapshotUuid; + return Boolean.TRUE.equals(Q.New(VolumeSnapshotVO.class) + .eq(VolumeSnapshotVO_.uuid, snapshotUuid) + .select(VolumeSnapshotVO_.encrypted) + .findValue()); + } + private void handle(CreateVolumeMsg msg) { VolumeInventory inv = createVolume(msg); CreateVolumeReply reply = new CreateVolumeReply(); @@ -667,6 +767,11 @@ private void handle(APICreateDataVolumeFromVolumeSnapshotMsg msg) { vo.setStatus(VolumeStatus.Creating); vo.setType(VolumeType.Data); vo.setSize(0); + Boolean snapshotEncrypted = Q.New(VolumeSnapshotVO.class) + .eq(VolumeSnapshotVO_.uuid, msg.getVolumeSnapshotUuid()) + .select(VolumeSnapshotVO_.encrypted) + .findValue(); + vo.setEncrypted(Boolean.TRUE.equals(snapshotEncrypted)); vo.setAccountUuid(msg.getSession().getAccountUuid()); if (msg.hasSystemTag(VolumeSystemTags.FAST_CREATE::isMatch)) { @@ -685,6 +790,10 @@ protected VolumeVO scripts() { }.execute(); new FireVolumeCanonicalEvent().fireVolumeStatusChangedEvent(null, VolumeInventory.valueOf(vvo)); + for (CreateDataVolumeExtensionPoint ext : pluginRgty.getExtensionList(CreateDataVolumeExtensionPoint.class)) { + ext.afterCreateVolume(vvo, msg.getVolumeSnapshotUuid()); + } + vvo = dbf.reload(vvo); instantiateDataVolumeFromSnapshot(vo, msg.getVolumeSnapshotUuid(), msg.getSystemTags(), new ReturnValueCompletion(evt) { @Override @@ -827,6 +936,11 @@ private void handle(CreateDataVolumeFromVolumeSnapshotMsg msg) { vo.setStatus(VolumeStatus.Creating); vo.setType(VolumeType.Data); vo.setSize(msg.getSize() != null ? msg.getSize() : 0); + Boolean snapshotEncrypted = Q.New(VolumeSnapshotVO.class) + .eq(VolumeSnapshotVO_.uuid, msg.getVolumeSnapshotUuid()) + .select(VolumeSnapshotVO_.encrypted) + .findValue(); + vo.setEncrypted(Boolean.TRUE.equals(msg.getEncrypted()) || Boolean.TRUE.equals(snapshotEncrypted)); vo.setAccountUuid(msg.getSession().getAccountUuid()); VolumeVO vvo = new SQLBatchWithReturn() { @Override @@ -848,6 +962,10 @@ protected VolumeVO scripts() { }.execute(); new FireVolumeCanonicalEvent().fireVolumeStatusChangedEvent(null, VolumeInventory.valueOf(vvo)); + for (CreateDataVolumeExtensionPoint ext : pluginRgty.getExtensionList(CreateDataVolumeExtensionPoint.class)) { + ext.afterCreateVolume(vvo, msg.getVolumeSnapshotUuid()); + } + vvo = dbf.reload(vvo); instantiateDataVolumeFromSnapshot(vo, msg.getVolumeSnapshotUuid(), msg.getSystemTags(), new ReturnValueCompletion(msg) { @Override @@ -908,7 +1026,13 @@ public void run(MessageReply reply) { new FireVolumeCanonicalEvent().fireVolumeStatusChangedEvent(VolumeStatus.Creating, VolumeInventory.valueOf(vvo)); completion.success(VolumeInventory.valueOf(vvo)); } else { - dbf.removeByPrimaryKey(vo.getUuid(), VolumeVO.class); + vvo = dbf.reload(vo); + if (vvo != null) { + VolumeInventory inventory = VolumeInventory.valueOf(vvo); + CollectionUtils.safeForEach(pluginRgty.getExtensionList(VolumeJustBeforeDeleteFromDbExtensionPoint.class), + ext -> ext.volumeJustBeforeDeleteFromDb(inventory)); + dbf.remove(vvo); + } completion.fail(reply.getError()); } } @@ -1416,4 +1540,4 @@ public void run(MessageReply reply) { }); }); } -} \ No newline at end of file +} diff --git a/storage/src/main/java/org/zstack/storage/volume/VolumeSystemTags.java b/storage/src/main/java/org/zstack/storage/volume/VolumeSystemTags.java index 996dfec92b0..ab99d063983 100644 --- a/storage/src/main/java/org/zstack/storage/volume/VolumeSystemTags.java +++ b/storage/src/main/java/org/zstack/storage/volume/VolumeSystemTags.java @@ -49,4 +49,9 @@ public class VolumeSystemTags { public static String VOLUME_QOS_TOKEN = "qos"; public static PatternedSystemTag VOLUME_QOS = new PatternedSystemTag(String.format("%s::{%s}", VOLUME_QOS_TOKEN, VOLUME_QOS_TOKEN), VolumeVO.class); + + @NonCloneable + public static String VOLUME_LIBVIRT_SECRET_HOST_TOKEN = "hostUuid"; + public static PatternedSystemTag VOLUME_LIBVIRT_SECRET_HOST = new PatternedSystemTag( + String.format("volumeLibvirtSecretHost::{%s}", VOLUME_LIBVIRT_SECRET_HOST_TOKEN), VolumeVO.class); } diff --git a/testlib/src/main/java/org/zstack/testlib/KVMSimulator.groovy b/testlib/src/main/java/org/zstack/testlib/KVMSimulator.groovy index e0ae1576003..eb8277e2e84 100755 --- a/testlib/src/main/java/org/zstack/testlib/KVMSimulator.groovy +++ b/testlib/src/main/java/org/zstack/testlib/KVMSimulator.groovy @@ -350,6 +350,12 @@ class KVMSimulator implements Simulator { return rsp } + spec.simulator(KVMConstant.KVM_WRITE_SECRET_MATERIAL_FILE_PATH) { + def rsp = new SecretHostEnsureLuksSecretFileResponse() + rsp.secFilePath = "/var/lib/zstack/luks/test.key" + return rsp + } + spec.simulator(KVMConstant.KVM_ENSURE_SECRET_PATH) { HttpEntity e -> String hostUuid = e.getHeaders().getFirst(Constants.AGENT_HTTP_HEADER_RESOURCE_UUID) SecretHostDefineCmd cmd = JSONObjectUtil.toObject(e.body, SecretHostDefineCmd.class) diff --git a/testlib/src/main/java/org/zstack/testlib/LocalStorageSpec.groovy b/testlib/src/main/java/org/zstack/testlib/LocalStorageSpec.groovy index f1dba92674a..3530492f9bb 100755 --- a/testlib/src/main/java/org/zstack/testlib/LocalStorageSpec.groovy +++ b/testlib/src/main/java/org/zstack/testlib/LocalStorageSpec.groovy @@ -295,6 +295,14 @@ class LocalStorageSpec extends PrimaryStorageSpec { return new LocalStorageKvmBackend.CreateVolumeWithBackingRsp() } + simulator(LocalStorageKvmBackend.ENCRYPT_VOLUME_BITS_PATH) { + return new LocalStorageKvmBackend.EncryptVolumeBitsRsp() + } + + simulator(LocalStorageKvmBackend.CONVERT_VOLUME_ENCRYPTION_PATH) { + return new LocalStorageKvmBackend.ConvertVolumeEncryptionRsp() + } + VFS.vfsHook(LocalStorageKvmBackend.CREATE_VOLUME_WITH_BACKING_PATH, espec) { rsp, HttpEntity e, EnvSpec spec -> def cmd = JSONObjectUtil.toObject(e.body, LocalStorageKvmBackend.CreateVolumeWithBackingCmd.class) VFS vfs = vfs(e, cmd, spec) diff --git a/testlib/src/main/java/org/zstack/testlib/NfsPrimaryStorageSpec.groovy b/testlib/src/main/java/org/zstack/testlib/NfsPrimaryStorageSpec.groovy index c99f96cc273..32ea00411d1 100755 --- a/testlib/src/main/java/org/zstack/testlib/NfsPrimaryStorageSpec.groovy +++ b/testlib/src/main/java/org/zstack/testlib/NfsPrimaryStorageSpec.groovy @@ -153,6 +153,14 @@ class NfsPrimaryStorageSpec extends PrimaryStorageSpec { return new NfsPrimaryStorageKVMBackendCommands.CreateEmptyVolumeResponse() } + simulator(NfsPrimaryStorageKVMBackend.ENCRYPT_VOLUME_BITS_PATH) { + return new NfsPrimaryStorageKVMBackendCommands.EncryptVolumeBitsRsp() + } + + simulator(NfsPrimaryStorageKVMBackend.CONVERT_VOLUME_ENCRYPTION_PATH) { + return new NfsPrimaryStorageKVMBackendCommands.ConvertVolumeEncryptionRsp() + } + VFS.vfsHook(NfsPrimaryStorageKVMBackend.CREATE_EMPTY_VOLUME_PATH, xspec) { rsp, HttpEntity e, EnvSpec spec -> def cmd = JSONObjectUtil.toObject(e.body, NfsPrimaryStorageKVMBackendCommands.CreateEmptyVolumeCmd.class) VFS vfs = vfs(cmd, spec)