diff --git a/conf/db/zsv/V5.1.0__schema.sql b/conf/db/zsv/V5.1.0__schema.sql index 532984350b0..94e9d697eea 100644 --- a/conf/db/zsv/V5.1.0__schema.sql +++ b/conf/db/zsv/V5.1.0__schema.sql @@ -314,3 +314,21 @@ CREATE TABLE IF NOT EXISTS `zstack`.`PlatformServiceInstanceVO` ( -- Feature: LicenseHistoryVO cause | ZSV-12697 CALL INSERT_COLUMN('LicenseHistoryVO', 'cause', 'varchar(32)', 0, 'UpdateLicense', 'source'); + +-- Feature: serialize VM metadata cleanup with cluster-wide metadata flushes | ZSV-11867 +CREATE TABLE IF NOT EXISTS `zstack`.`VmMetadataCleanupBarrierVO` ( + `id` varchar(32) NOT NULL, + `state` varchar(32) NOT NULL DEFAULT 'Idle', + `operationUuid` varchar(32) DEFAULT NULL, + `managementNodeUuid` varchar(32) DEFAULT NULL, + `leaseExpireDate` timestamp NULL DEFAULT NULL, + `generation` bigint NOT NULL DEFAULT 0, + `lastOpDate` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `createDate` timestamp NOT NULL DEFAULT '1999-12-31 23:59:59', + PRIMARY KEY (`id`), + CONSTRAINT `fkVmMetadataCleanupBarrierVOManagementNodeVO` + FOREIGN KEY (`managementNodeUuid`) REFERENCES `ManagementNodeVO` (`uuid`) ON DELETE SET NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +INSERT IGNORE INTO `zstack`.`VmMetadataCleanupBarrierVO` (`id`, `state`, `createDate`) +VALUES ('vm-metadata-cleanup', 'Idle', CURRENT_TIMESTAMP); diff --git a/conf/persistence.xml b/conf/persistence.xml index a720c4f9897..fa1446a9935 100755 --- a/conf/persistence.xml +++ b/conf/persistence.xml @@ -222,6 +222,7 @@ org.zstack.header.resourceattribute.entity.ResourceAttributeKeyResourceTypeVO org.zstack.header.resourceattribute.entity.ResourceAttributeConstraintVO org.zstack.softwarePackage.header.SoftwarePackageVO + org.zstack.header.vm.metadata.VmMetadataCleanupBarrierVO org.zstack.header.vm.metadata.VmMetadataDirtyVO org.zstack.header.vm.metadata.VmMetadataFlushStateVO org.zstack.log.server.LogServerVO diff --git a/conf/serviceConfig/vmInstance.xml b/conf/serviceConfig/vmInstance.xml index 730951e0ece..86db25dff1f 100755 --- a/conf/serviceConfig/vmInstance.xml +++ b/conf/serviceConfig/vmInstance.xml @@ -264,12 +264,15 @@ org.zstack.header.vm.APIDeleteTemplatedVmInstanceMsg - - org.zstack.header.vm.APICleanupVmInstanceMetadataMsg - - - org.zstack.header.vm.APIRegisterVmInstanceFromMetadataMsg - + + org.zstack.header.vm.APICleanupVmInstanceMetadataMsg + + + org.zstack.header.vm.APICleanupAllVmInstanceMetadataMsg + + + org.zstack.header.vm.APIRegisterVmInstanceFromMetadataMsg + org.zstack.header.vm.APIUpdateVmInstanceMetadataMsg diff --git a/header/src/main/java/org/zstack/header/storage/primary/CleanupAllVmMetadataOnPrimaryStorageMsg.java b/header/src/main/java/org/zstack/header/storage/primary/CleanupAllVmMetadataOnPrimaryStorageMsg.java new file mode 100644 index 00000000000..8239f7b2498 --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/primary/CleanupAllVmMetadataOnPrimaryStorageMsg.java @@ -0,0 +1,34 @@ +package org.zstack.header.storage.primary; + +import org.zstack.header.message.NeedReplyMessage; + +public class CleanupAllVmMetadataOnPrimaryStorageMsg extends NeedReplyMessage implements PrimaryStorageMessage { + private String primaryStorageUuid; + private String metadataDir; + private long metadataGeneration; + + @Override + public String getPrimaryStorageUuid() { + return primaryStorageUuid; + } + + public void setPrimaryStorageUuid(String primaryStorageUuid) { + this.primaryStorageUuid = primaryStorageUuid; + } + + public String getMetadataDir() { + return metadataDir; + } + + public void setMetadataDir(String metadataDir) { + this.metadataDir = metadataDir; + } + + public long getMetadataGeneration() { + return metadataGeneration; + } + + public void setMetadataGeneration(long metadataGeneration) { + this.metadataGeneration = metadataGeneration; + } +} diff --git a/header/src/main/java/org/zstack/header/storage/primary/CleanupAllVmMetadataOnPrimaryStorageReply.java b/header/src/main/java/org/zstack/header/storage/primary/CleanupAllVmMetadataOnPrimaryStorageReply.java new file mode 100644 index 00000000000..cc69434505a --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/primary/CleanupAllVmMetadataOnPrimaryStorageReply.java @@ -0,0 +1,24 @@ +package org.zstack.header.storage.primary; + +import org.zstack.header.message.MessageReply; + +public class CleanupAllVmMetadataOnPrimaryStorageReply extends MessageReply { + private boolean skipped; + private Long currentGeneration; + + public boolean isSkipped() { + return skipped; + } + + public void setSkipped(boolean skipped) { + this.skipped = skipped; + } + + public Long getCurrentGeneration() { + return currentGeneration; + } + + public void setCurrentGeneration(Long currentGeneration) { + this.currentGeneration = currentGeneration; + } +} diff --git a/header/src/main/java/org/zstack/header/vm/APICleanupAllVmInstanceMetadataEvent.java b/header/src/main/java/org/zstack/header/vm/APICleanupAllVmInstanceMetadataEvent.java new file mode 100644 index 00000000000..263c9058f53 --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/APICleanupAllVmInstanceMetadataEvent.java @@ -0,0 +1,33 @@ +package org.zstack.header.vm; + +import org.zstack.header.message.APIEvent; +import org.zstack.header.rest.RestResponse; + +import java.util.List; + +@RestResponse(fieldsTo = {"all"}) +public class APICleanupAllVmInstanceMetadataEvent extends APIEvent { + private List failedPrimaryStorageUuids; + + public APICleanupAllVmInstanceMetadataEvent() { + super(null); + } + + public APICleanupAllVmInstanceMetadataEvent(String apiId) { + super(apiId); + } + + public List getFailedPrimaryStorageUuids() { + return failedPrimaryStorageUuids; + } + + public void setFailedPrimaryStorageUuids(List failedPrimaryStorageUuids) { + this.failedPrimaryStorageUuids = failedPrimaryStorageUuids; + } + + public static APICleanupAllVmInstanceMetadataEvent __example__() { + APICleanupAllVmInstanceMetadataEvent evt = new APICleanupAllVmInstanceMetadataEvent(); + evt.failedPrimaryStorageUuids = java.util.Collections.emptyList(); + return evt; + } +} diff --git a/header/src/main/java/org/zstack/header/vm/APICleanupAllVmInstanceMetadataEventDoc_zh_cn.groovy b/header/src/main/java/org/zstack/header/vm/APICleanupAllVmInstanceMetadataEventDoc_zh_cn.groovy new file mode 100644 index 00000000000..e672bee7f5f --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/APICleanupAllVmInstanceMetadataEventDoc_zh_cn.groovy @@ -0,0 +1,29 @@ +package org.zstack.header.vm + +import org.zstack.header.errorcode.ErrorCode + +doc { + + title "清理全部云主机元数据返回" + + field { + name "failedPrimaryStorageUuids" + desc "清理失败的主存储UUID列表；具体失败原因汇总见 error 字段，逐条详情见 mn / agent 日志" + type "List" + since "5.1.0" + } + field { + name "success" + desc "操作是否成功；任一主存储清理失败则为 false" + type "boolean" + since "5.1.0" + } + ref { + name "error" + path "org.zstack.header.vm.APICleanupAllVmInstanceMetadataEvent.error" + desc "错误码；success=false 时聚合所有失败主存储的失败原因" + type "ErrorCode" + since "5.1.0" + clz ErrorCode.class + } +} diff --git a/header/src/main/java/org/zstack/header/vm/APICleanupAllVmInstanceMetadataMsg.java b/header/src/main/java/org/zstack/header/vm/APICleanupAllVmInstanceMetadataMsg.java new file mode 100644 index 00000000000..73ce53af338 --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/APICleanupAllVmInstanceMetadataMsg.java @@ -0,0 +1,33 @@ +package org.zstack.header.vm; + +import org.springframework.http.HttpMethod; +import org.zstack.header.message.APIMessage; +import org.zstack.header.message.APIParam; +import org.zstack.header.rest.RestRequest; +import org.zstack.header.storage.primary.PrimaryStorageVO; + +import java.util.List; + +@RestRequest( + path = "/vm-instances/metadata", + method = HttpMethod.DELETE, + responseClass = APICleanupAllVmInstanceMetadataEvent.class +) +public class APICleanupAllVmInstanceMetadataMsg extends APIMessage { + @APIParam(resourceType = PrimaryStorageVO.class, required = false) + private List primaryStorageUuids; + + public List getPrimaryStorageUuids() { + return primaryStorageUuids; + } + + public void setPrimaryStorageUuids(List primaryStorageUuids) { + this.primaryStorageUuids = primaryStorageUuids; + } + + public static APICleanupAllVmInstanceMetadataMsg __example__() { + APICleanupAllVmInstanceMetadataMsg msg = new APICleanupAllVmInstanceMetadataMsg(); + msg.primaryStorageUuids = java.util.Arrays.asList(uuid(), uuid()); + return msg; + } +} diff --git a/header/src/main/java/org/zstack/header/vm/APICleanupAllVmInstanceMetadataMsgDoc_zh_cn.groovy b/header/src/main/java/org/zstack/header/vm/APICleanupAllVmInstanceMetadataMsgDoc_zh_cn.groovy new file mode 100644 index 00000000000..73ab9621f11 --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/APICleanupAllVmInstanceMetadataMsgDoc_zh_cn.groovy @@ -0,0 +1,58 @@ +package org.zstack.header.vm + +import org.zstack.header.vm.APICleanupAllVmInstanceMetadataEvent + +doc { + title "清理全部云主机元数据" + + category "云主机" + + desc """清理一个或多个主存储上保存的全部云主机元数据文件，仅管理员可调用。当 primaryStorageUuids 为空（未传或传空列表）时，将清理系统中所有 Enabled+Connected 且支持云主机元数据的主存储；否则仅清理列表中指定的主存储。""" + + rest { + request { + url "DELETE /v1/vm-instances/metadata" + + header (Authorization: 'OAuth the-session-uuid') + + clz APICleanupAllVmInstanceMetadataMsg.class + + desc """""" + + params { + + column { + name "primaryStorageUuids" + enclosedIn "" + desc "需要清理云主机元数据的主存储UUID列表；为空时清理所有 Enabled+Connected 主存储上的元数据" + location "query" + type "List" + optional true + since "5.1.0" + } + column { + name "systemTags" + enclosedIn "" + desc "系统标签" + location "query" + type "List" + optional true + since "5.1.0" + } + column { + name "userTags" + enclosedIn "" + desc "用户标签" + location "query" + type "List" + optional true + since "5.1.0" + } + } + } + + response { + clz APICleanupAllVmInstanceMetadataEvent.class + } + } +} \ No newline at end of file diff --git a/header/src/main/java/org/zstack/header/vm/metadata/UpdateVmInstanceMetadataOnPrimaryStorageMsg.java b/header/src/main/java/org/zstack/header/vm/metadata/UpdateVmInstanceMetadataOnPrimaryStorageMsg.java index ba12e4b918e..6bb83741461 100644 --- a/header/src/main/java/org/zstack/header/vm/metadata/UpdateVmInstanceMetadataOnPrimaryStorageMsg.java +++ b/header/src/main/java/org/zstack/header/vm/metadata/UpdateVmInstanceMetadataOnPrimaryStorageMsg.java @@ -14,6 +14,7 @@ public class UpdateVmInstanceMetadataOnPrimaryStorageMsg extends NeedReplyMessag private String schemaVersion; private boolean storageStructureChange; private String metadataPath; + private long metadataGeneration; @Override public String getPrimaryStorageUuid() { @@ -95,4 +96,12 @@ public String getMetadataPath() { public void setMetadataPath(String metadataPath) { this.metadataPath = metadataPath; } + + public long getMetadataGeneration() { + return metadataGeneration; + } + + public void setMetadataGeneration(long metadataGeneration) { + this.metadataGeneration = metadataGeneration; + } } diff --git a/header/src/main/java/org/zstack/header/vm/metadata/VmMetadataCleanupBarrierState.java b/header/src/main/java/org/zstack/header/vm/metadata/VmMetadataCleanupBarrierState.java new file mode 100644 index 00000000000..48b08e8f4a4 --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/metadata/VmMetadataCleanupBarrierState.java @@ -0,0 +1,7 @@ +package org.zstack.header.vm.metadata; + +public enum VmMetadataCleanupBarrierState { + Idle, + Draining, + Cleaning +} diff --git a/header/src/main/java/org/zstack/header/vm/metadata/VmMetadataCleanupBarrierVO.java b/header/src/main/java/org/zstack/header/vm/metadata/VmMetadataCleanupBarrierVO.java new file mode 100644 index 00000000000..44b2e5a7db7 --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/metadata/VmMetadataCleanupBarrierVO.java @@ -0,0 +1,110 @@ +package org.zstack.header.vm.metadata; + +import org.zstack.header.managementnode.ManagementNodeVO; +import org.zstack.header.vo.ForeignKey; +import org.zstack.header.vo.ForeignKey.ReferenceOption; + +import javax.persistence.*; +import java.sql.Timestamp; + +@Entity +@Table +public class VmMetadataCleanupBarrierVO { + public static final String SINGLETON_ID = "vm-metadata-cleanup"; + + @Id + @Column + private String id; + + @Column + @Enumerated(EnumType.STRING) + private VmMetadataCleanupBarrierState state; + + @Column + private String operationUuid; + + @Column + @ForeignKey(parentEntityClass = ManagementNodeVO.class, onDeleteAction = ReferenceOption.SET_NULL) + private String managementNodeUuid; + + @Column + private Timestamp leaseExpireDate; + + @Column + private long generation; + + @Column + private Timestamp createDate; + + @Column + private Timestamp lastOpDate; + + @PreUpdate + private void preUpdate() { + lastOpDate = null; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public VmMetadataCleanupBarrierState getState() { + return state; + } + + public void setState(VmMetadataCleanupBarrierState state) { + this.state = state; + } + + public String getOperationUuid() { + return operationUuid; + } + + public void setOperationUuid(String operationUuid) { + this.operationUuid = operationUuid; + } + + public String getManagementNodeUuid() { + return managementNodeUuid; + } + + public void setManagementNodeUuid(String managementNodeUuid) { + this.managementNodeUuid = managementNodeUuid; + } + + public Timestamp getLeaseExpireDate() { + return leaseExpireDate; + } + + public void setLeaseExpireDate(Timestamp leaseExpireDate) { + this.leaseExpireDate = leaseExpireDate; + } + + public long getGeneration() { + return generation; + } + + public void setGeneration(long generation) { + this.generation = generation; + } + + public Timestamp getCreateDate() { + return createDate; + } + + public void setCreateDate(Timestamp createDate) { + this.createDate = createDate; + } + + public Timestamp getLastOpDate() { + return lastOpDate; + } + + public void setLastOpDate(Timestamp lastOpDate) { + this.lastOpDate = lastOpDate; + } +} diff --git a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageBase.java b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageBase.java index eea5f8beb92..b1fb6153d82 100755 --- a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageBase.java +++ b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageBase.java @@ -907,6 +907,8 @@ public void handleLocalMessage(Message msg) { handle((CommitVolumeSnapshotOnPrimaryStorageMsg) msg); } else if (msg instanceof PullVolumeSnapshotOnPrimaryStorageMsg) { handle((PullVolumeSnapshotOnPrimaryStorageMsg) msg); + } else if (msg instanceof CleanupAllVmMetadataOnPrimaryStorageMsg) { + handle((CleanupAllVmMetadataOnPrimaryStorageMsg) msg); } else { super.handleLocalMessage(msg); } @@ -3625,4 +3627,101 @@ public String getName() { } }); } + + @Override + protected void handle(final CleanupAllVmMetadataOnPrimaryStorageMsg msg) { + CleanupAllVmMetadataOnPrimaryStorageReply reply = new CleanupAllVmMetadataOnPrimaryStorageReply(); + + List allHostUuids = getLocalStorageHostUuids(); + if (allHostUuids.isEmpty()) { + reply.setError(operr("no host found for local primary storage[uuid:%s]", self.getUuid())); + bus.reply(msg, reply); + return; + } + + List connectedHostUuids = getConnectedLocalStorageHostUuids(); + if (connectedHostUuids.isEmpty()) { + reply.setError(operr("no connected host found for local primary storage[uuid:%s], " + + "total hosts=%d", self.getUuid(), allHostUuids.size())); + bus.reply(msg, reply); + return; + } + + List disconnectedHostUuids = new ArrayList<>(allHostUuids); + disconnectedHostUuids.removeAll(connectedHostUuids); + if (!disconnectedHostUuids.isEmpty()) { + logger.warn(String.format("local primary storage[uuid:%s] has %d disconnected hosts that cannot clean up vm metadata: %s", + self.getUuid(), disconnectedHostUuids.size(), disconnectedHostUuids)); + } + + new While<>(connectedHostUuids).all((hostUuid, com) -> { + final LocalStorageHypervisorBackend bkd; + try { + LocalStorageHypervisorFactory f = getHypervisorBackendFactoryByHostUuid(hostUuid); + bkd = f.getHypervisorBackend(self); + } catch (Exception e) { + logger.warn(String.format("[MetadataCleanupAll] failed to prepare backend for host[uuid:%s] on ps[uuid:%s]: %s", + hostUuid, self.getUuid(), e.getMessage())); + com.addError(operr("host[uuid:%s] backend prepare failed: %s", hostUuid, e.getMessage())); + com.done(); + return; + } + + bkd.handle(msg, hostUuid, new ReturnValueCompletion(com) { + @Override + public void success(CleanupAllVmMetadataOnPrimaryStorageReply returnValue) { + if (returnValue.isSkipped()) { + synchronized (reply) { + reply.setSkipped(true); + if (returnValue.getCurrentGeneration() != null + && (reply.getCurrentGeneration() == null + || returnValue.getCurrentGeneration() > reply.getCurrentGeneration())) { + reply.setCurrentGeneration(returnValue.getCurrentGeneration()); + } + } + } + com.done(); + } + + @Override + public void fail(ErrorCode errorCode) { + logger.warn(String.format("[MetadataCleanupAll] failed on host[uuid:%s] on local ps[uuid:%s]: %s", + hostUuid, self.getUuid(), errorCode)); + com.addError(errorCode); + com.done(); + } + }); + }).run(new WhileDoneCompletion(msg) { + @Override + public void done(ErrorCodeList errorCodeList) { + int failedHostCount = disconnectedHostUuids.size() + errorCodeList.getCauses().size(); + if (failedHostCount > 0) { + reply.setError(operr("local primary storage[uuid:%s] cleanup all vm metadata failed on %d/%d host(s), " + + "disconnectedHosts=%s, causes=%s", + self.getUuid(), failedHostCount, allHostUuids.size(), disconnectedHostUuids, errorCodeList)); + } + bus.reply(msg, reply); + } + }); + } + + private List getLocalStorageHostUuids() { + return SQL.New( + "select h.hostUuid from LocalStorageHostRefVO h, HostVO host" + + " where h.primaryStorageUuid = :psUuid" + + " and h.hostUuid = host.uuid", String.class) + .param("psUuid", self.getUuid()) + .list(); + } + + private List getConnectedLocalStorageHostUuids() { + return SQL.New( + "select h.hostUuid from LocalStorageHostRefVO h, HostVO host" + + " where h.primaryStorageUuid = :psUuid" + + " and h.hostUuid = host.uuid" + + " and host.status = :hstatus", String.class) + .param("psUuid", self.getUuid()) + .param("hstatus", HostStatus.Connected) + .list(); + } } diff --git a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageHypervisorBackend.java b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageHypervisorBackend.java index d8226d932b2..58d5653610a 100755 --- a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageHypervisorBackend.java +++ b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageHypervisorBackend.java @@ -132,5 +132,7 @@ public LocalStorageHypervisorBackend(PrimaryStorageVO self) { abstract void handle(CleanupVmInstanceMetadataOnPrimaryStorageMsg msg, String hostUuid, ReturnValueCompletion completion); + abstract void handle(CleanupAllVmMetadataOnPrimaryStorageMsg msg, String hostUuid, ReturnValueCompletion completion); + abstract void handle(RebaseVolumeBackingFileOnPrimaryStorageMsg msg, String hostUuid, ReturnValueCompletion completion); } diff --git a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmBackend.java b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmBackend.java index 5b4324495b2..80267106b4b 100755 --- a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmBackend.java +++ b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmBackend.java @@ -917,6 +917,7 @@ public void setHashValue(String hashValue) { public static class WriteVmMetadataCmd extends AgentCommand { public String metadata; public String metadataPath; + public long metadataGeneration; public String vmUuid; public String vmName; public String vmCategory; @@ -950,6 +951,16 @@ public static class CleanupVmMetadataCmd extends AgentCommand { public static class CleanupVmMetadataRsp extends AgentResponse { } + public static class CleanupAllVmMetadataCmd extends AgentCommand { + public String metadataDir; + public long metadataGeneration; + } + + public static class CleanupAllVmMetadataRsp extends AgentResponse { + public boolean skipped; + public Long currentGeneration; + } + public static class PrefixRebaseBackingFilesCmd extends LocalStorageKvmBackend.AgentCommand { public List filePaths; public String oldPrefix; @@ -996,6 +1007,7 @@ public static class PrefixRebaseBackingFilesRsp extends LocalStorageKvmBackend.A public static final String GET_VM_INSTANCE_METADATA_PATH = "/localstorage/vm/metadata/get"; public static final String SCAN_VM_METADATA_PATH = "/localstorage/vm/metadata/scan"; public static final String CLEANUP_VM_METADATA_PATH = "/localstorage/vm/metadata/cleanup"; + public static final String CLEANUP_ALL_VM_METADATA_PATH = "/localstorage/vm/metadata/cleanupall"; public static final String PREFIX_REBASE_BACKING_FILES_PATH = "/localstorage/snapshot/prefixrebasebackingfiles"; public LocalStorageKvmBackend() { @@ -3869,6 +3881,7 @@ void handle(UpdateVmInstanceMetadataOnPrimaryStorageMsg msg, String hostUuid, Re WriteVmMetadataCmd cmd = new WriteVmMetadataCmd(); cmd.metadata = msg.getMetadata(); cmd.metadataPath = msg.getMetadataPath(); + cmd.metadataGeneration = msg.getMetadataGeneration(); cmd.vmUuid = msg.getVmInstanceUuid(); cmd.vmName = msg.getVmInstanceName(); cmd.vmCategory = msg.getVmCategory(); @@ -3955,6 +3968,28 @@ public void fail(ErrorCode errorCode) { }); } + @Override + void handle(CleanupAllVmMetadataOnPrimaryStorageMsg msg, String hostUuid, ReturnValueCompletion completion) { + CleanupAllVmMetadataCmd cmd = new CleanupAllVmMetadataCmd(); + cmd.metadataDir = msg.getMetadataDir(); + cmd.metadataGeneration = msg.getMetadataGeneration(); + + httpCall(CLEANUP_ALL_VM_METADATA_PATH, hostUuid, cmd, CleanupAllVmMetadataRsp.class, new ReturnValueCompletion(completion) { + @Override + public void success(CleanupAllVmMetadataRsp rsp) { + CleanupAllVmMetadataOnPrimaryStorageReply reply = new CleanupAllVmMetadataOnPrimaryStorageReply(); + reply.setSkipped(rsp.skipped); + reply.setCurrentGeneration(rsp.currentGeneration); + completion.success(reply); + } + + @Override + public void fail(ErrorCode errorCode) { + completion.fail(errorCode); + } + }); + } + @Override void handle(RebaseVolumeBackingFileOnPrimaryStorageMsg msg, String hostUuid, ReturnValueCompletion completion) { PrefixRebaseBackingFilesCmd cmd = new PrefixRebaseBackingFilesCmd(); diff --git a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorage.java b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorage.java index 6b0142e5a63..d58ae83e71a 100755 --- a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorage.java +++ b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorage.java @@ -61,6 +61,7 @@ import javax.persistence.Tuple; import javax.persistence.TypedQuery; import java.io.File; +import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map; @@ -136,6 +137,8 @@ protected void handleLocalMessage(Message msg) { handle((PullVolumeSnapshotOnPrimaryStorageMsg) msg); } else if (msg instanceof RebaseVolumeBackingFileOnPrimaryStorageMsg) { handle((RebaseVolumeBackingFileOnPrimaryStorageMsg) msg); + } else if (msg instanceof CleanupAllVmMetadataOnPrimaryStorageMsg) { + handle((CleanupAllVmMetadataOnPrimaryStorageMsg) msg); } else { super.handleLocalMessage(msg); } @@ -2065,6 +2068,48 @@ public void fail(ErrorCode errorCode) { }); } + @Override + protected void handle(CleanupAllVmMetadataOnPrimaryStorageMsg msg) { + CleanupAllVmMetadataOnPrimaryStorageReply reply = new CleanupAllVmMetadataOnPrimaryStorageReply(); + List connectedHosts = factory.getConnectedHostForOperation(getSelfInventory()); + if (connectedHosts.isEmpty()) { + reply.setError(operr("no connected host found for NFS primary storage[uuid:%s]", self.getUuid())); + bus.reply(msg, reply); + return; + } + + cleanupAllOnHostWithFallback(msg, reply, connectedHosts, 0, new ArrayList<>()); + } + + private void cleanupAllOnHostWithFallback(CleanupAllVmMetadataOnPrimaryStorageMsg msg, + CleanupAllVmMetadataOnPrimaryStorageReply reply, + List connectedHosts, int idx, + List causes) { + if (idx >= connectedHosts.size()) { + reply.setError(operr("failed to cleanup all vm metadata on NFS primary storage[uuid:%s] after trying %d connected host(s): %s", + self.getUuid(), connectedHosts.size(), causes)); + bus.reply(msg, reply); + return; + } + + String hostUuid = connectedHosts.get(idx).getUuid(); + final NfsPrimaryStorageBackend backend = getBackendByHostUuid(hostUuid); + backend.handle(msg, hostUuid, new ReturnValueCompletion(msg) { + @Override + public void success(CleanupAllVmMetadataOnPrimaryStorageReply r) { + bus.reply(msg, r); + } + + @Override + public void fail(ErrorCode errorCode) { + logger.warn(String.format("[MetadataCleanupAll] NFS ps[uuid:%s] failed on host[uuid:%s]: %s; trying next connected host", + self.getUuid(), hostUuid, errorCode)); + causes.add(errorCode); + cleanupAllOnHostWithFallback(msg, reply, connectedHosts, idx + 1, causes); + } + }); + } + @Override protected void handle(RebaseVolumeBackingFileOnPrimaryStorageMsg msg) { RebaseVolumeBackingFileOnPrimaryStorageReply reply = new RebaseVolumeBackingFileOnPrimaryStorageReply(); diff --git a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageBackend.java b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageBackend.java index 35a240a6221..b0f2acced33 100755 --- a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageBackend.java +++ b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageBackend.java @@ -101,6 +101,8 @@ public interface NfsPrimaryStorageBackend { void handle(CleanupVmInstanceMetadataOnPrimaryStorageMsg msg, String hostUuid, ReturnValueCompletion completion); + void handle(CleanupAllVmMetadataOnPrimaryStorageMsg msg, String hostUuid, ReturnValueCompletion completion); + void handle(RebaseVolumeBackingFileOnPrimaryStorageMsg msg, String hostUuid, ReturnValueCompletion completion); class BitsInfo { diff --git a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackend.java b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackend.java index 445497e0871..b69e7ee09d2 100755 --- a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackend.java +++ b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackend.java @@ -138,6 +138,7 @@ public class NfsPrimaryStorageKVMBackend implements NfsPrimaryStorageBackend, public static final String GET_VM_INSTANCE_METADATA_PATH = "/nfsprimarystorage/vm/metadata/get"; public static final String SCAN_VM_METADATA_PATH = "/nfsprimarystorage/vm/metadata/scan"; public static final String CLEANUP_VM_METADATA_PATH = "/nfsprimarystorage/vm/metadata/cleanup"; + public static final String CLEANUP_ALL_VM_METADATA_PATH = "/nfsprimarystorage/vm/metadata/cleanupall"; public static final String NFS_PREFIX_REBASE_BACKING_FILES_PATH = "/nfsprimarystorage/snapshot/prefixrebasebackingfiles"; //////////////// For unit test ////////////////////////// @@ -2067,6 +2068,7 @@ public void handle(UpdateVmInstanceMetadataOnPrimaryStorageMsg msg, String hostU cmd.setUuid(msg.getPrimaryStorageUuid()); cmd.metadata = msg.getMetadata(); cmd.metadataPath = msg.getMetadataPath(); + cmd.metadataGeneration = msg.getMetadataGeneration(); cmd.vmUuid = msg.getVmInstanceUuid(); cmd.vmName = msg.getVmInstanceName(); cmd.vmCategory = msg.getVmCategory(); @@ -2196,6 +2198,40 @@ public void run(MessageReply reply) { }); } + @Override + public void handle(CleanupAllVmMetadataOnPrimaryStorageMsg msg, String hostUuid, ReturnValueCompletion completion) { + CleanupAllVmMetadataCmd cmd = new CleanupAllVmMetadataCmd(); + cmd.setUuid(msg.getPrimaryStorageUuid()); + cmd.metadataDir = msg.getMetadataDir(); + cmd.metadataGeneration = msg.getMetadataGeneration(); + + KVMHostAsyncHttpCallMsg hmsg = new KVMHostAsyncHttpCallMsg(); + hmsg.setCommand(cmd); + hmsg.setPath(CLEANUP_ALL_VM_METADATA_PATH); + hmsg.setHostUuid(hostUuid); + bus.makeTargetServiceIdByResourceUuid(hmsg, HostConstant.SERVICE_ID, hostUuid); + bus.send(hmsg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + completion.fail(reply.getError()); + return; + } + + CleanupAllVmMetadataRsp rsp = ((KVMHostAsyncHttpCallReply) reply).toResponse(CleanupAllVmMetadataRsp.class); + if (!rsp.isSuccess()) { + completion.fail(operr("failed to cleanup all vm metadata on nfs via host[uuid:%s]: %s", hostUuid, rsp.getError())); + return; + } + + CleanupAllVmMetadataOnPrimaryStorageReply r = new CleanupAllVmMetadataOnPrimaryStorageReply(); + r.setSkipped(rsp.skipped); + r.setCurrentGeneration(rsp.currentGeneration); + completion.success(r); + } + }); + } + @Override public void handle(RebaseVolumeBackingFileOnPrimaryStorageMsg msg, String hostUuid, ReturnValueCompletion completion) { NfsPrimaryStorageKVMBackendCommands.PrefixRebaseBackingFilesCmd cmd = new NfsPrimaryStorageKVMBackendCommands.PrefixRebaseBackingFilesCmd(); diff --git a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackendCommands.java b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackendCommands.java index 0901af4cadb..eaef5686ce1 100755 --- a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackendCommands.java +++ b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackendCommands.java @@ -961,6 +961,7 @@ public void setHashValue(String hashValue) { public static class WriteVmMetadataCmd extends NfsPrimaryStorageAgentCommand { public String metadata; public String metadataPath; + public long metadataGeneration; public String vmUuid; public String vmName; public String vmCategory; @@ -995,6 +996,16 @@ public static class CleanupVmMetadataCmd extends NfsPrimaryStorageAgentCommand { public static class CleanupVmMetadataRsp extends NfsPrimaryStorageAgentResponse { } + public static class CleanupAllVmMetadataCmd extends NfsPrimaryStorageAgentCommand { + public String metadataDir; + public long metadataGeneration; + } + + public static class CleanupAllVmMetadataRsp extends NfsPrimaryStorageAgentResponse { + public boolean skipped; + public Long currentGeneration; + } + public static class PrefixRebaseBackingFilesCmd extends NfsPrimaryStorageAgentCommand { public List filePaths; public String oldPrefix; diff --git a/sdk/src/main/java/org/zstack/sdk/CleanupAllVmInstanceMetadataAction.java b/sdk/src/main/java/org/zstack/sdk/CleanupAllVmInstanceMetadataAction.java new file mode 100644 index 00000000000..d5af7b802bc --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/CleanupAllVmInstanceMetadataAction.java @@ -0,0 +1,101 @@ +package org.zstack.sdk; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class CleanupAllVmInstanceMetadataAction extends AbstractAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.CleanupAllVmInstanceMetadataResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s]", error.code, error.description, error.details) + ); + } + + return this; + } + } + + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.util.List primaryStorageUuids; + + @Param(required = false) + public java.util.List systemTags; + + @Param(required = false) + public java.util.List userTags; + + @Param(required = false) + public String sessionId; + + @Param(required = false) + public String accessKeyId; + + @Param(required = false) + public String accessKeySecret; + + @Param(required = false) + public String requestIp; + + @NonAPIParam + public long timeout = -1; + + @NonAPIParam + public long pollingInterval = -1; + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.CleanupAllVmInstanceMetadataResult value = res.getResult(org.zstack.sdk.CleanupAllVmInstanceMetadataResult.class); + ret.value = value == null ? new org.zstack.sdk.CleanupAllVmInstanceMetadataResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "DELETE"; + info.path = "/vm-instances/metadata"; + info.needSession = true; + info.needPoll = true; + info.parameterName = ""; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/CleanupAllVmInstanceMetadataResult.java b/sdk/src/main/java/org/zstack/sdk/CleanupAllVmInstanceMetadataResult.java new file mode 100644 index 00000000000..48497a3ad45 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/CleanupAllVmInstanceMetadataResult.java @@ -0,0 +1,14 @@ +package org.zstack.sdk; + + + +public class CleanupAllVmInstanceMetadataResult { + public java.util.List failedPrimaryStorageUuids; + public void setFailedPrimaryStorageUuids(java.util.List failedPrimaryStorageUuids) { + this.failedPrimaryStorageUuids = failedPrimaryStorageUuids; + } + public java.util.List getFailedPrimaryStorageUuids() { + return this.failedPrimaryStorageUuids; + } + +} diff --git a/storage/src/main/java/org/zstack/storage/primary/PrimaryStorageBase.java b/storage/src/main/java/org/zstack/storage/primary/PrimaryStorageBase.java index a2cdbad18f4..3f2de150452 100755 --- a/storage/src/main/java/org/zstack/storage/primary/PrimaryStorageBase.java +++ b/storage/src/main/java/org/zstack/storage/primary/PrimaryStorageBase.java @@ -430,6 +430,8 @@ protected void handleLocalMessage(Message msg) { handle((GetVmInstanceMetadataFromPrimaryStorageMsg) msg); } else if (msg instanceof CleanupVmInstanceMetadataOnPrimaryStorageMsg) { handle((CleanupVmInstanceMetadataOnPrimaryStorageMsg) msg); + } else if (msg instanceof CleanupAllVmMetadataOnPrimaryStorageMsg) { + handle((CleanupAllVmMetadataOnPrimaryStorageMsg) msg); } else if (msg instanceof RebaseVolumeBackingFileOnPrimaryStorageMsg) { handle((RebaseVolumeBackingFileOnPrimaryStorageMsg) msg); } else { @@ -1822,6 +1824,12 @@ protected void handle(CleanupVmInstanceMetadataOnPrimaryStorageMsg msg) { bus.reply(msg, reply); } + protected void handle(CleanupAllVmMetadataOnPrimaryStorageMsg msg) { + CleanupAllVmMetadataOnPrimaryStorageReply reply = new CleanupAllVmMetadataOnPrimaryStorageReply(); + reply.setError(operr("operation not supported")); + bus.reply(msg, reply); + } + protected void handle(RebaseVolumeBackingFileOnPrimaryStorageMsg msg) { RebaseVolumeBackingFileOnPrimaryStorageReply reply = new RebaseVolumeBackingFileOnPrimaryStorageReply(); reply.setError(operr("operation not supported")); diff --git a/testlib/src/main/java/org/zstack/testlib/ApiHelper.groovy b/testlib/src/main/java/org/zstack/testlib/ApiHelper.groovy index f65e8aadc88..6fc402b96e5 100644 --- a/testlib/src/main/java/org/zstack/testlib/ApiHelper.groovy +++ b/testlib/src/main/java/org/zstack/testlib/ApiHelper.groovy @@ -14,27 +14,27 @@ abstract class ApiHelper { return res.value } } - + def createDataVolumeFromVolumeBackup(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.storage.volumebackup.CreateDataVolumeFromVolumeBackupAction.class) Closure c) { def a = new org.zstack.sdk.storage.volumebackup.CreateDataVolumeFromVolumeBackupAction() a.sessionId = Test.currentEnvSpec?.session?.uuid c.resolveStrategy = Closure.OWNER_FIRST c.delegate = a c() - + if (System.getProperty("apipath") != null) { if (a.apiId == null) { a.apiId = Platform.uuid } - + def tracker = new ApiPathTracker(a.apiId) def out = errorOut(a.call()) def path = tracker.getApiPath() if (!path.isEmpty()) { Test.apiPaths[a.class.name] = path.join(" --->\n") } - + return out } else { return errorOut(a.call()) @@ -5606,6 +5606,33 @@ abstract class ApiHelper { } + def cleanupAllVmInstanceMetadata(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.CleanupAllVmInstanceMetadataAction.class) Closure c) { + def a = new org.zstack.sdk.CleanupAllVmInstanceMetadataAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + def cleanupBillingUsage(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.CleanupBillingUsageAction.class) Closure c) { def a = new org.zstack.sdk.CleanupBillingUsageAction() a.sessionId = Test.currentEnvSpec?.session?.uuid diff --git a/testlib/src/main/java/org/zstack/testlib/VmMetadataCleanupBarrierDBRemaining.groovy b/testlib/src/main/java/org/zstack/testlib/VmMetadataCleanupBarrierDBRemaining.groovy new file mode 100644 index 00000000000..fa4c9adaa54 --- /dev/null +++ b/testlib/src/main/java/org/zstack/testlib/VmMetadataCleanupBarrierDBRemaining.groovy @@ -0,0 +1,23 @@ +package org.zstack.testlib + +import org.zstack.header.vm.metadata.VmMetadataCleanupBarrierState +import org.zstack.header.vm.metadata.VmMetadataCleanupBarrierVO + +class VmMetadataCleanupBarrierDBRemaining extends AllowedDBRemaining { + @Override + void remaining() { + table { + tableVOClass = VmMetadataCleanupBarrierVO.class + checker = { List barriers -> + return barriers.findAll { VmMetadataCleanupBarrierVO barrier -> + barrier.id != VmMetadataCleanupBarrierVO.SINGLETON_ID || + barrier.state != VmMetadataCleanupBarrierState.Idle || + barrier.operationUuid != null || + barrier.managementNodeUuid != null || + barrier.leaseExpireDate != null || + barrier.generation < 0 + } + } + } + } +}