ResourceManagerImplTest.java

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

package com.cloud.resource;

import com.cloud.agent.AgentManager;
import com.cloud.agent.api.GetVncPortAnswer;
import com.cloud.agent.api.GetVncPortCommand;
import com.cloud.capacity.dao.CapacityDao;
import com.cloud.dc.ClusterVO;
import com.cloud.dc.DataCenterVO;
import com.cloud.dc.HostPodVO;
import com.cloud.dc.dao.ClusterDao;
import com.cloud.dc.dao.DataCenterDao;
import com.cloud.dc.dao.HostPodDao;
import com.cloud.event.ActionEventUtils;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.ha.HighAvailabilityManager;
import com.cloud.host.Host;
import com.cloud.host.HostVO;
import com.cloud.host.Status;
import com.cloud.host.dao.HostDao;
import com.cloud.hypervisor.Hypervisor;
import com.cloud.storage.ScopeType;
import com.cloud.storage.StorageManager;
import com.cloud.storage.StoragePoolHostVO;
import com.cloud.storage.Volume;
import com.cloud.storage.VolumeVO;
import com.cloud.storage.dao.StoragePoolAndAccessGroupMapDao;
import com.cloud.storage.dao.StoragePoolHostDao;
import com.cloud.storage.dao.VolumeDao;
import com.cloud.utils.Ternary;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.fsm.NoTransitionException;
import com.cloud.utils.ssh.SSHCmdHelper;
import com.cloud.utils.ssh.SshException;
import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.VirtualMachine;
import com.cloud.vm.dao.VMInstanceDetailsDao;
import com.cloud.vm.dao.VMInstanceDao;
import com.trilead.ssh2.Connection;
import org.apache.cloudstack.api.command.admin.host.CancelHostAsDegradedCmd;
import org.apache.cloudstack.api.command.admin.host.DeclareHostAsDegradedCmd;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreInfo;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.BDDMockito;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.MockedConstruction;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import org.mockito.Spy;
import org.mockito.junit.MockitoJUnitRunner;

import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.UUID;

import static com.cloud.resource.ResourceState.Event.ErrorsCorrected;
import static com.cloud.resource.ResourceState.Event.InternalEnterMaintenance;
import static com.cloud.resource.ResourceState.Event.UnableToMaintain;
import static com.cloud.resource.ResourceState.Event.UnableToMigrate;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

@RunWith(MockitoJUnitRunner.class)
public class ResourceManagerImplTest {

    @Mock
    private CapacityDao capacityDao;
    @Mock
    private StorageManager storageManager;
    @Mock
    private HighAvailabilityManager haManager;
    @Mock
    private VMInstanceDetailsDao vmInstanceDetailsDao;
    @Mock
    private AgentManager agentManager;
    @Mock
    private HostDao hostDao;
    @Mock
    private ClusterDao clusterDao;
    @Mock
    private HostPodDao podDao;
    @Mock
    private DataCenterDao dcDao;
    @Mock
    private VMInstanceDao vmInstanceDao;
    @Mock
    private ConfigurationDao configurationDao;
    @Mock
    private VolumeDao volumeDao;
    @Mock
    private PrimaryDataStoreDao storagePoolDao;
    @Mock
    private StoragePoolHostDao storagePoolHostDao;

    @Spy
    @InjectMocks
    private ResourceManagerImpl resourceManager = new ResourceManagerImpl();

    @Mock
    private HostVO host;
    @Mock
    private VMInstanceVO vm1;
    @Mock
    private VMInstanceVO vm2;

    @Mock
    private GetVncPortAnswer getVncPortAnswerVm1;
    @Mock
    private GetVncPortAnswer getVncPortAnswerVm2;
    @Mock
    private VolumeVO rootDisk1;
    @Mock
    private VolumeVO rootDisk2;
    @Mock
    private VolumeVO dataDisk;

    @Mock
    private Connection sshConnection;

    @Mock
    private StoragePoolAndAccessGroupMapDao storagePoolAccessGroupMapDao;

    private static long hostId = 1L;
    private static final String hostUsername = "user";
    private static final String hostPassword = "password";
    private static final String hostPrivateKey = "privatekey";
    private static final String hostPrivateIp = "192.168.1.10";

    private static long vm1Id = 1L;
    private static String vm1InstanceName = "i-1-VM";
    private static long vm2Id = 2L;
    private static String vm2InstanceName = "i-2-VM";

    private static String vm1VncAddress = "10.2.2.2";
    private static int vm1VncPort = 5900;
    private static String vm2VncAddress = "10.2.2.2";
    private static int vm2VncPort = 5901;

    private static long poolId = 1L;
    private List<VolumeVO> rootDisks;
    private List<VolumeVO> dataDisks;
    private MockedStatic<SSHCmdHelper> sshHelperMocked;
    private MockedStatic<ActionEventUtils> actionEventUtilsMocked;
    private MockedConstruction<GetVncPortCommand> getVncPortCommandMockedConstruction;
    private AutoCloseable closeable;

    private void overrideDefaultConfigValue(final ConfigKey configKey, final String name, final Object o) throws IllegalAccessException, NoSuchFieldException {
        Field f = ConfigKey.class.getDeclaredField(name);
        f.setAccessible(true);
        f.set(configKey, o);
    }

    @Before
    public void setup() throws Exception {
        closeable = MockitoAnnotations.openMocks(this);
        when(host.getType()).thenReturn(Host.Type.Routing);
        when(host.getId()).thenReturn(hostId);
        when(host.getResourceState()).thenReturn(ResourceState.Enabled);
        when(host.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM);
        when(hostDao.findById(hostId)).thenReturn(host);
        when(host.getDetail("username")).thenReturn(hostUsername);
        when(host.getDetail("password")).thenReturn(hostPassword);
        when(configurationDao.getValue("ssh.privatekey")).thenReturn(hostPrivateKey);
        when(host.getStatus()).thenReturn(Status.Up);
        when(host.getPrivateIpAddress()).thenReturn(hostPrivateIp);
        when(vm1.getId()).thenReturn(vm1Id);
        when(vm2.getId()).thenReturn(vm2Id);
        when(vm1.getInstanceName()).thenReturn(vm1InstanceName);
        when(vm2.getInstanceName()).thenReturn(vm2InstanceName);
        when(vmInstanceDao.listByHostId(hostId)).thenReturn(new ArrayList<>());
        when(vmInstanceDao.listVmsMigratingFromHost(hostId)).thenReturn(new ArrayList<>());
        when(vmInstanceDao.listNonMigratingVmsByHostEqualsLastHost(hostId)).thenReturn(new ArrayList<>());
        actionEventUtilsMocked = Mockito.mockStatic(ActionEventUtils.class);
        BDDMockito.given(ActionEventUtils.onCompletedActionEvent(anyLong(), anyLong(), anyString(), anyString(), anyString(), anyLong(), anyString(), anyLong()))
                .willReturn(1L);
        when(getVncPortAnswerVm1.getAddress()).thenReturn(vm1VncAddress);
        when(getVncPortAnswerVm1.getPort()).thenReturn(vm1VncPort);
        when(getVncPortAnswerVm2.getAddress()).thenReturn(vm2VncAddress);
        when(getVncPortAnswerVm2.getPort()).thenReturn(vm2VncPort);
        getVncPortCommandMockedConstruction = Mockito.mockConstruction(GetVncPortCommand.class, (mock,context) -> {
            if (context.arguments().get(0).equals(vm1Id) && context.arguments().get(1) == vm1InstanceName) {
                when(agentManager.easySend(eq(hostId), eq(mock))).thenReturn(getVncPortAnswerVm1);
            } else if (context.arguments().get(0).equals(vm2Id) && context.arguments().get(1) == vm2InstanceName) {
                when(agentManager.easySend(eq(hostId), eq(mock))).thenReturn(getVncPortAnswerVm2);
            }
        });

        sshHelperMocked = Mockito.mockStatic(SSHCmdHelper.class);
        BDDMockito.given(SSHCmdHelper.acquireAuthorizedConnection(eq(hostPrivateIp), eq(22),
                eq(hostUsername), eq(hostPassword), eq(hostPrivateKey))).willReturn(sshConnection);
        BDDMockito.given(SSHCmdHelper.sshExecuteCmdOneShot(eq(sshConnection),
                eq("service cloudstack-agent restart"))).
                willReturn(new SSHCmdHelper.SSHCmdResult(0,"",""));

        overrideDefaultConfigValue(ResourceManager.KvmSshToAgentEnabled, "_defaultValue", "true");

        rootDisks = Arrays.asList(rootDisk1, rootDisk2);
        dataDisks = Collections.singletonList(dataDisk);
        when(volumeDao.findNonDestroyedVolumesByPoolId(poolId)).thenReturn(rootDisks);
        when(volumeDao.findNonDestroyedVolumesByPoolId(poolId, Volume.Type.DATADISK)).thenReturn(dataDisks);
    }

    @After
    public void tearDown() throws Exception {
        sshHelperMocked.close();
        actionEventUtilsMocked.close();
        getVncPortCommandMockedConstruction.close();
        closeable.close();
    }

    @Test
    public void testCheckAndMaintainEnterMaintenanceModeNoVms() throws NoTransitionException {
        // Test entering into maintenance with no VMs running on host.
        boolean enterMaintenanceMode = resourceManager.checkAndMaintain(hostId);
        verify(resourceManager).attemptMaintain(host);
        verify(resourceManager).setHostIntoMaintenance(host);
        verify(resourceManager, never()).setHostIntoErrorInPrepareForMaintenance(any(), any());
        verify(resourceManager, never()).setHostIntoErrorInMaintenance(any(), any());
        verify(resourceManager, never()).setHostIntoPrepareForMaintenanceAfterErrorsFixed(any());
        verify(resourceManager).resourceStateTransitTo(eq(host), eq(InternalEnterMaintenance), anyLong());

        Assert.assertTrue(enterMaintenanceMode);
    }

    @Test
    public void testCheckAndMaintainProceedsWithPrepareForMaintenanceRunningVms() throws NoTransitionException {
        // Test proceeding through with no events if pending migrating works / retries left.
        setupRunningVMs();
        setupPendingMigrationRetries();
        verifyNoChangeInMaintenance();
    }

    @Test
    public void testCheckAndMaintainErrorInMaintenanceRunningVms() throws NoTransitionException {
        // Test entering into ErrorInMaintenance when no pending migrations etc, and due to - Running VMs
        setupRunningVMs();
        setupNoPendingMigrationRetries();
        verifyErrorInMaintenanceCalls();
    }

    @Test
    public void testCheckAndMaintainErrorInMaintenanceWithErrorVms() throws NoTransitionException {
        // Test entering into ErrorInMaintenance when no pending migrations etc, and due to - no migrating but error VMs
        setupErrorVms();
        setupNoPendingMigrationRetries();
        verifyErrorInMaintenanceCalls();
    }

    @Test
    public void testCheckAndMaintainErrorInPrepareForMaintenanceFailedMigrationsPendingRetries() throws NoTransitionException {
        // Test entering into ErrorInPrepareForMaintenance when pending migrations retries and due to - Failed Migrations
        setupFailedMigrations();
        setupPendingMigrationRetries();
        when(vmInstanceDao.findByHostInStates(hostId, VirtualMachine.State.Running)).thenReturn(Arrays.asList(vm2));
        verifyErrorInPrepareForMaintenanceCalls();
    }

    @Test
    public void testCheckAndMaintainErrorInPrepareForMaintenanceWithErrorVmsPendingRetries() throws NoTransitionException {
        // Test entering into ErrorInMaintenance when pending migrations retries due to - no migrating but error VMs
        setupErrorVms();
        setupPendingMigrationRetries();
        when(vmInstanceDao.listVmsMigratingFromHost(hostId)).thenReturn(Arrays.asList(vm2));
        verifyErrorInPrepareForMaintenanceCalls();
    }

    @Test
    public void testCheckAndMaintainErrorInPrepareForMaintenanceFailedMigrationsAndMigratingVms() throws NoTransitionException {
        // Test entering into ErrorInPrepareForMaintenance when no pending migrations retries
        // but executing migration and due to - Failed Migrations
        setupFailedMigrations();
        setupNoPendingMigrationRetries();
        when(vmInstanceDao.listVmsMigratingFromHost(hostId)).thenReturn(Arrays.asList(vm2));
        verifyErrorInPrepareForMaintenanceCalls();
    }

    @Test
    public void testCheckAndMaintainErrorInPrepareForMaintenanceWithErrorVmsAndMigratingVms() throws NoTransitionException {
        // Test entering into ErrorInPrepareForMaintenance when no pending migrations retries
        // but executing migration and due to - Error Vms
        setupErrorVms();
        setupNoPendingMigrationRetries();
        when(vmInstanceDao.listVmsMigratingFromHost(hostId)).thenReturn(Arrays.asList(vm2));
        verifyErrorInPrepareForMaintenanceCalls();
    }

    @Test
    public void testCheckAndMaintainErrorInPrepareForMaintenanceFailedMigrationsAndStoppingVms() throws NoTransitionException {
        // Test entering into ErrorInPrepareForMaintenance when no pending migrations retries
        // but stopping VMs and due to - Failed Migrations
        setupFailedMigrations();
        setupNoPendingMigrationRetries();
        when(vmInstanceDao.findByHostInStates(hostId, VirtualMachine.State.Stopping)).thenReturn(Arrays.asList(vm2));
        verifyErrorInPrepareForMaintenanceCalls();
    }

    @Test
    public void testCheckAndMaintainReturnsToPrepareForMaintenanceRunningVms() throws NoTransitionException {
        // Test switching back to PrepareForMaintenance
        when(host.getResourceState()).thenReturn(ResourceState.ErrorInPrepareForMaintenance);
        setupRunningVMs();
        setupPendingMigrationRetries();
        verifyReturnToPrepareForMaintenanceCalls();
    }

    @Test
    public void testConfigureVncAccessForKVMHostFailedMigrations() {
        when(host.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM);
        List<VMInstanceVO> vms = Arrays.asList(vm1, vm2);
        resourceManager.configureVncAccessForKVMHostFailedMigrations(host, vms);
        verify(agentManager).pullAgentOutMaintenance(hostId);
        verify(resourceManager).setKVMVncAccess(hostId, vms);
        verify(agentManager, times(vms.size())).easySend(eq(hostId), any(GetVncPortCommand.class));
        verify(agentManager).pullAgentToMaintenance(hostId);
        verify(vmInstanceDetailsDao).addDetail(eq(vm1Id), eq("kvm.vnc.address"), eq(vm1VncAddress), anyBoolean());
        verify(vmInstanceDetailsDao).addDetail(eq(vm1Id), eq("kvm.vnc.port"), eq(String.valueOf(vm1VncPort)), anyBoolean());
        verify(vmInstanceDetailsDao).addDetail(eq(vm2Id), eq("kvm.vnc.address"), eq(vm2VncAddress), anyBoolean());
        verify(vmInstanceDetailsDao).addDetail(eq(vm2Id), eq("kvm.vnc.port"), eq(String.valueOf(vm2VncPort)), anyBoolean());
    }

    @Test(expected = CloudRuntimeException.class)
    public void testGetHostCredentialsMissingParameter() {
        when(host.getDetail("password")).thenReturn(null);
        when(configurationDao.getValue("ssh.privatekey")).thenReturn(null);
        resourceManager.getHostCredentials(host);
    }

    @Test
    public void testGetHostCredentials() {
        Ternary<String, String, String> credentials = resourceManager.getHostCredentials(host);
        Assert.assertNotNull(credentials);
        Assert.assertEquals(hostUsername, credentials.first());
        Assert.assertEquals(hostPassword, credentials.second());
        Assert.assertEquals(hostPrivateKey, credentials.third());
    }

    @Test(expected = CloudRuntimeException.class)
    public void testConnectAndRestartAgentOnHostCannotConnect() {
        BDDMockito.given(SSHCmdHelper.acquireAuthorizedConnection(eq(hostPrivateIp), eq(22),
                eq(hostUsername), eq(hostPassword), eq(hostPrivateKey))).willReturn(null);
        resourceManager.connectAndRestartAgentOnHost(host, hostUsername, hostPassword, hostPrivateKey);
    }

    @Test(expected = CloudRuntimeException.class)
    public void testConnectAndRestartAgentOnHostCannotRestart() throws Exception {
        BDDMockito.given(SSHCmdHelper.sshExecuteCmdOneShot(eq(sshConnection),
                eq("service cloudstack-agent restart"))).willThrow(new SshException("exception"));
        resourceManager.connectAndRestartAgentOnHost(host, hostUsername, hostPassword, hostPrivateKey);
    }

    @Test
    public void testConnectAndRestartAgentOnHost() {
        when(agentManager.getHostSshPort(any())).thenReturn(22);
        resourceManager.connectAndRestartAgentOnHost(host, hostUsername, hostPassword, hostPrivateKey);
    }

    @Test
    public void testHandleAgentSSHEnabledNotConnectedAgent() {
        when(host.getStatus()).thenReturn(Status.Disconnected);
        when(agentManager.getHostSshPort(any())).thenReturn(22);
        resourceManager.handleAgentIfNotConnected(host, false);
        verify(resourceManager).getHostCredentials(eq(host));
        verify(resourceManager).connectAndRestartAgentOnHost(eq(host), eq(hostUsername), eq(hostPassword), eq(hostPrivateKey));
    }

    @Test
    public void testHandleAgentSSHEnabledConnectedAgent() {
        when(host.getStatus()).thenReturn(Status.Up);
        resourceManager.handleAgentIfNotConnected(host, false);
        verify(resourceManager, never()).getHostCredentials(eq(host));
        verify(resourceManager, never()).connectAndRestartAgentOnHost(eq(host), eq(hostUsername), eq(hostPassword), eq(hostPrivateKey));
    }

    @Test(expected = CloudRuntimeException.class)
    public void testHandleAgentSSHDisabledNotConnectedAgent() throws NoSuchFieldException, IllegalAccessException {
        when(host.getStatus()).thenReturn(Status.Disconnected);
        overrideDefaultConfigValue(ResourceManager.KvmSshToAgentEnabled, "_defaultValue", "false");
        resourceManager.handleAgentIfNotConnected(host, false);
    }

    @Test
    public void testHandleAgentSSHDisabledConnectedAgent() {
        when(host.getStatus()).thenReturn(Status.Up);
        resourceManager.handleAgentIfNotConnected(host, false);
        verify(resourceManager, never()).getHostCredentials(eq(host));
        verify(resourceManager, never()).connectAndRestartAgentOnHost(eq(host), eq(hostUsername), eq(hostPassword), eq(hostPrivateKey));
    }

    @Test
    public void testHandleAgentVMsMigrating() {
        resourceManager.handleAgentIfNotConnected(host, true);
        verify(resourceManager, never()).getHostCredentials(eq(host));
        verify(resourceManager, never()).connectAndRestartAgentOnHost(eq(host), eq(hostUsername), eq(hostPassword), eq(hostPrivateKey));
    }

    private void setupNoPendingMigrationRetries() {
        when(haManager.hasPendingMigrationsWork(vm1.getId())).thenReturn(false);
        when(haManager.hasPendingMigrationsWork(vm2.getId())).thenReturn(false);
    }

    private void setupRunningVMs() {
        when(vmInstanceDao.listByHostId(hostId)).thenReturn(Arrays.asList(vm1, vm2));
        when(vmInstanceDao.findByHostInStates(hostId, VirtualMachine.State.Migrating, VirtualMachine.State.Running, VirtualMachine.State.Starting, VirtualMachine.State.Stopping, VirtualMachine.State.Error, VirtualMachine.State.Unknown)).thenReturn(Arrays.asList(vm1, vm2));
        when(vmInstanceDao.findByHostInStates(hostId, VirtualMachine.State.Running)).thenReturn(Arrays.asList(vm1, vm2));
    }

    private void setupPendingMigrationRetries() {
        when(haManager.hasPendingMigrationsWork(vm1.getId())).thenReturn(true);
    }

    private void setupFailedMigrations() {
        when(vmInstanceDao.listByHostId(hostId)).thenReturn(Arrays.asList(vm1, vm2));
        when(vmInstanceDao.findByHostInStates(hostId, VirtualMachine.State.Migrating, VirtualMachine.State.Running, VirtualMachine.State.Starting, VirtualMachine.State.Stopping, VirtualMachine.State.Error, VirtualMachine.State.Unknown)).thenReturn(Arrays.asList(vm1, vm2));
        when(vmInstanceDao.listNonMigratingVmsByHostEqualsLastHost(hostId)).thenReturn(Arrays.asList(vm1));
    }

    private void setupErrorVms() {
        when(vmInstanceDao.listByHostId(hostId)).thenReturn(Arrays.asList(vm1, vm2));
        when(vmInstanceDao.findByHostInStates(hostId, VirtualMachine.State.Migrating, VirtualMachine.State.Running, VirtualMachine.State.Starting, VirtualMachine.State.Stopping, VirtualMachine.State.Error, VirtualMachine.State.Unknown)).thenReturn(Arrays.asList(vm1, vm2));
        when(vmInstanceDao.findByHostInStates(hostId, VirtualMachine.State.Unknown, VirtualMachine.State.Error)).thenReturn(Arrays.asList(vm1));
    }

    private void verifyErrorInMaintenanceCalls() throws NoTransitionException {
        boolean enterMaintenanceMode = resourceManager.checkAndMaintain(hostId);
        verify(resourceManager).attemptMaintain(host);
        verify(resourceManager).setHostIntoErrorInMaintenance(eq(host), any());
        verify(resourceManager, never()).setHostIntoMaintenance(any());
        verify(resourceManager, never()).setHostIntoErrorInPrepareForMaintenance(any(), any());
        verify(resourceManager, never()).setHostIntoPrepareForMaintenanceAfterErrorsFixed(any());
        verify(resourceManager).resourceStateTransitTo(eq(host), eq(UnableToMaintain), anyLong());
        Assert.assertFalse(enterMaintenanceMode);
    }

    private void verifyErrorInPrepareForMaintenanceCalls() throws NoTransitionException {
        boolean enterMaintenanceMode = resourceManager.checkAndMaintain(hostId);
        verify(resourceManager).attemptMaintain(host);
        verify(resourceManager).setHostIntoErrorInPrepareForMaintenance(eq(host), any());
        verify(resourceManager, never()).setHostIntoMaintenance(any());
        verify(resourceManager, never()).setHostIntoErrorInMaintenance(any(), any());
        verify(resourceManager, never()).setHostIntoPrepareForMaintenanceAfterErrorsFixed(any());
        verify(resourceManager).resourceStateTransitTo(eq(host), eq(UnableToMigrate), anyLong());
        Assert.assertFalse(enterMaintenanceMode);
    }

    private void verifyReturnToPrepareForMaintenanceCalls() throws NoTransitionException {
        boolean enterMaintenanceMode = resourceManager.checkAndMaintain(hostId);
        verify(resourceManager).attemptMaintain(host);
        verify(resourceManager).setHostIntoPrepareForMaintenanceAfterErrorsFixed(eq(host));
        verify(resourceManager).resourceStateTransitTo(eq(host), eq(ErrorsCorrected), anyLong());
        verify(resourceManager, never()).setHostIntoMaintenance(any());
        verify(resourceManager, never()).setHostIntoErrorInPrepareForMaintenance(any(), any());
        verify(resourceManager, never()).setHostIntoErrorInMaintenance(any(), any());
        Assert.assertFalse(enterMaintenanceMode);
    }

    private void verifyNoChangeInMaintenance() throws NoTransitionException {
        boolean enterMaintenanceMode = resourceManager.checkAndMaintain(hostId);
        verify(resourceManager).attemptMaintain(host);
        verify(resourceManager, never()).setHostIntoMaintenance(any());
        verify(resourceManager, never()).setHostIntoErrorInPrepareForMaintenance(any(), any());
        verify(resourceManager, never()).setHostIntoErrorInMaintenance(any(), any());
        verify(resourceManager, never()).setHostIntoPrepareForMaintenanceAfterErrorsFixed(any());
        verify(resourceManager, never()).resourceStateTransitTo(any(), any(), anyLong());
        Assert.assertFalse(enterMaintenanceMode);
    }

    @Test
    public void declareHostAsDegradedTestDisconnected() throws NoTransitionException {
        prepareAndTestDeclareHostAsDegraded(Status.Disconnected, ResourceState.Enabled, ResourceState.Degraded);
    }

    @Test
    public void declareHostAsDegradedTestAlert() throws NoTransitionException {
        prepareAndTestDeclareHostAsDegraded(Status.Alert, ResourceState.Enabled, ResourceState.Degraded);
    }

    @Test(expected = InvalidParameterValueException.class)
    public void declareHostAsDegradedExpectNoTransitionException() throws NoTransitionException {
        Status[] statusArray = Status.values();
        for (int i = 0; i < statusArray.length - 1; i++) {
            if (statusArray[i] != Status.Alert && statusArray[i] != Status.Disconnected) {
                prepareAndTestDeclareHostAsDegraded(statusArray[i], ResourceState.Enabled, ResourceState.Enabled);
            }
        }
    }

    @Test(expected = NoTransitionException.class)
    public void declareHostAsDegradedTestAlreadyDegraded() throws NoTransitionException {
        prepareAndTestDeclareHostAsDegraded(Status.Alert, ResourceState.Degraded, ResourceState.Degraded);
    }

    @Test(expected = NoTransitionException.class)
    public void declareHostAsDegradedTestOnError() throws NoTransitionException {
        prepareAndTestDeclareHostAsDegraded(Status.Alert, ResourceState.Error, ResourceState.Degraded);
    }

    @Test(expected = NoTransitionException.class)
    public void declareHostAsDegradedTestOnCreating() throws NoTransitionException {
        prepareAndTestDeclareHostAsDegraded(Status.Alert, ResourceState.Creating, ResourceState.Degraded);
    }

    @Test(expected = NoTransitionException.class)
    public void declareHostAsDegradedTestOnErrorInMaintenance() throws NoTransitionException {
        prepareAndTestDeclareHostAsDegraded(Status.Alert, ResourceState.ErrorInPrepareForMaintenance, ResourceState.Degraded);
    }

    @Test
    public void declareHostAsDegradedTestSupportedStates() throws NoTransitionException {
        ResourceState[] states = ResourceState.values();
        for (int i = 0; i < states.length - 1; i++) {
            if (states[i] == ResourceState.Enabled
                    || states[i] == ResourceState.Maintenance
                    || states[i] == ResourceState.Disabled) {
                prepareAndTestDeclareHostAsDegraded(Status.Alert, states[i], ResourceState.Degraded);
            }
        }
    }

    private void prepareAndTestDeclareHostAsDegraded(Status hostStatus, ResourceState originalState, ResourceState expectedResourceState) throws NoTransitionException {
        DeclareHostAsDegradedCmd declareHostAsDegradedCmd = Mockito.spy(new DeclareHostAsDegradedCmd());
        HostVO hostVo = createDummyHost(hostStatus);
        hostVo.setResourceState(originalState);
        when(declareHostAsDegradedCmd.getId()).thenReturn(0l);
        when(hostDao.findById(0l)).thenReturn(hostVo);

        Host result = resourceManager.declareHostAsDegraded(declareHostAsDegradedCmd);

        Assert.assertEquals(expectedResourceState, hostVo.getResourceState());
    }

    @Test
    public void cancelHostAsDegradedTest() throws NoTransitionException {
        prepareAndTestCancelHostAsDegraded(Status.Alert, ResourceState.Degraded, ResourceState.Enabled);
    }

    @Test(expected = NoTransitionException.class)
    public void cancelHostAsDegradedTestHostNotDegraded() throws NoTransitionException {
        prepareAndTestCancelHostAsDegraded(Status.Alert, ResourceState.Enabled, ResourceState.Enabled);
    }

    private void prepareAndTestCancelHostAsDegraded(Status hostStatus, ResourceState originalState, ResourceState expectedResourceState) throws NoTransitionException {
        CancelHostAsDegradedCmd cancelHostAsDegradedCmd = Mockito.spy(new CancelHostAsDegradedCmd());
        HostVO hostVo = createDummyHost(hostStatus);
        hostVo.setResourceState(originalState);
        when(cancelHostAsDegradedCmd.getId()).thenReturn(0l);
        when(hostDao.findById(0l)).thenReturn(hostVo);

        Host result = resourceManager.cancelHostAsDegraded(cancelHostAsDegradedCmd);

        Assert.assertEquals(expectedResourceState, hostVo.getResourceState());
    }

    private HostVO createDummyHost(Status hostStatus) {
        return new HostVO(1L, "host01", Host.Type.Routing, "192.168.1.1", "255.255.255.0", null, null, null, null, null, null, null, null, null, null, UUID.randomUUID().toString(),
                hostStatus, "1.0", null, null, 1L, null, 0, 0, null, 0, null);
    }

    @Test
    public void testDestroyLocalStoragePoolVolumesBothRootDisksAndDataDisks() {
        resourceManager.destroyLocalStoragePoolVolumes(poolId);
        verify(volumeDao, times(rootDisks.size() + dataDisks.size()))
                .updateAndRemoveVolume(any(VolumeVO.class));
    }

    @Test
    public void testDestroyLocalStoragePoolVolumesOnlyRootDisks() {
        when(volumeDao.findNonDestroyedVolumesByPoolId(poolId, Volume.Type.DATADISK)).thenReturn(null);
        resourceManager.destroyLocalStoragePoolVolumes(poolId);
        verify(volumeDao, times(rootDisks.size())).updateAndRemoveVolume(any(VolumeVO.class));
    }

    @Test
    public void testDestroyLocalStoragePoolVolumesOnlyDataDisks() {
        when(volumeDao.findNonDestroyedVolumesByPoolId(poolId)).thenReturn(null);
        resourceManager.destroyLocalStoragePoolVolumes(poolId);
        verify(volumeDao, times(dataDisks.size())).updateAndRemoveVolume(any(VolumeVO.class));
    }

    @Test
    public void testDestroyLocalStoragePoolVolumesNoDisks() {
        when(volumeDao.findNonDestroyedVolumesByPoolId(poolId)).thenReturn(null);
        when(volumeDao.findNonDestroyedVolumesByPoolId(poolId, Volume.Type.DATADISK)).thenReturn(null);
        resourceManager.destroyLocalStoragePoolVolumes(poolId);
        verify(volumeDao, never()).updateAndRemoveVolume(any(VolumeVO.class));
    }

    @Test
    public void testEmptyHostList() {
        List<HostVO> allHosts = new ArrayList<>();
        List<String> storageAccessGroups = Arrays.asList("group1", "group2");

        List<HostVO> hostsToConnect = resourceManager.filterHostsBasedOnStorageAccessGroups(allHosts, storageAccessGroups);

        Assert.assertTrue("No hosts should be returned when the host list is empty.", hostsToConnect.isEmpty());
    }

    @Test
    public void testEmptyStorageAccessGroups() {
        List<HostVO> allHosts = Arrays.asList(Mockito.mock(HostVO.class), Mockito.mock(HostVO.class));
        List<String> storageAccessGroups = new ArrayList<>();

        for (HostVO host : allHosts) {
            Mockito.when(host.getId()).thenReturn(1L);
            Mockito.doReturn(new String[]{"group1", "group2"})
                    .when(storageManager).getStorageAccessGroups(null, null, null, 1L);
        }

        List<HostVO> hostsToConnect = resourceManager.filterHostsBasedOnStorageAccessGroups(allHosts, storageAccessGroups);

        Assert.assertTrue("All hosts should be returned when storage access groups are empty.", hostsToConnect.containsAll(allHosts));
        Assert.assertEquals("The number of returned hosts should match the total number of hosts.", allHosts.size(), hostsToConnect.size());
    }

    @Test
    public void testHostWithMatchingStorageAccessGroups() {
        HostVO host1 = Mockito.mock(HostVO.class);
        HostVO host2 = Mockito.mock(HostVO.class);
        List<HostVO> allHosts = Arrays.asList(host1, host2);
        List<String> storageAccessGroups = Arrays.asList("group1", "group2");

        Mockito.when(host1.getId()).thenReturn(1L);
        Mockito.doReturn(new String[]{"group1"})
                .when(storageManager).getStorageAccessGroups(null, null, null, 1L);

        Mockito.when(host2.getId()).thenReturn(2L);
        Mockito.doReturn(new String[]{"group3"})
                .when(storageManager).getStorageAccessGroups(null, null, null, 2L);

        List<HostVO> hostsToConnect = resourceManager.filterHostsBasedOnStorageAccessGroups(allHosts, storageAccessGroups);

        Assert.assertTrue("Only hosts with matching storage access groups should be included.", hostsToConnect.contains(host1));
        Assert.assertFalse("Hosts without matching storage access groups should not be included.", hostsToConnect.contains(host2));
        Assert.assertEquals("Only one host should match the storage access groups.", 1, hostsToConnect.size());
    }

    @Test
    public void testHostWithoutMatchingStorageAccessGroups() {
        HostVO host1 = Mockito.mock(HostVO.class);
        HostVO host2 = Mockito.mock(HostVO.class);
        List<HostVO> allHosts = Arrays.asList(host1, host2);
        List<String> storageAccessGroups = Arrays.asList("group1", "group2");

        Mockito.when(host1.getId()).thenReturn(1L);
        Mockito.doReturn(new String[]{"group3"})
                .when(storageManager).getStorageAccessGroups(null, null, null, 1L);

        Mockito.when(host2.getId()).thenReturn(2L);
        Mockito.doReturn(new String[]{"group4"})
                .when(storageManager).getStorageAccessGroups(null, null, null, 2L);

        List<HostVO> hostsToConnect = resourceManager.filterHostsBasedOnStorageAccessGroups(allHosts, storageAccessGroups);

        Assert.assertTrue("No hosts should match the storage access groups.", hostsToConnect.isEmpty());
    }

    @Test
    public void testMixedMatchingAndNonMatchingHosts() {
        HostVO host1 = Mockito.mock(HostVO.class);
        HostVO host2 = Mockito.mock(HostVO.class);
        HostVO host3 = Mockito.mock(HostVO.class);
        List<HostVO> allHosts = Arrays.asList(host1, host2, host3);
        List<String> storageAccessGroups = Arrays.asList("group1", "group2");

        Mockito.when(host1.getId()).thenReturn(1L);
        Mockito.doReturn(new String[]{"group1"})
                .when(storageManager).getStorageAccessGroups(null, null, null, 1L);

        Mockito.when(host2.getId()).thenReturn(2L);
        Mockito.doReturn(new String[]{"group3"})
                .when(storageManager).getStorageAccessGroups(null, null, null, 2L);

        Mockito.when(host3.getId()).thenReturn(3L);
        Mockito.doReturn(new String[]{"group2"})
                .when(storageManager).getStorageAccessGroups(null, null, null, 3L);

        List<HostVO> hostsToConnect = resourceManager.filterHostsBasedOnStorageAccessGroups(allHosts, storageAccessGroups);

        Assert.assertTrue("Host1 should be included as it matches 'group1'.", hostsToConnect.contains(host1));
        Assert.assertFalse("Host2 should not be included as it doesn't match any group.", hostsToConnect.contains(host2));
        Assert.assertTrue("Host3 should be included as it matches 'group2'.", hostsToConnect.contains(host3));
    }

    @Test
    public void testHostsWithEmptyStorageAccessGroups() {
        HostVO host1 = Mockito.mock(HostVO.class);
        HostVO host2 = Mockito.mock(HostVO.class);
        List<HostVO> allHosts = Arrays.asList(host1, host2);
        List<String> storageAccessGroups = Arrays.asList("group1", "group2");

        Mockito.when(host1.getId()).thenReturn(1L);
        Mockito.doReturn(new String[0])
                .when(storageManager).getStorageAccessGroups(null, null, null, 1L);

        Mockito.when(host2.getId()).thenReturn(2L);
        Mockito.doReturn(new String[0])
                .when(storageManager).getStorageAccessGroups(null, null, null, 2L);

        List<HostVO> hostsToConnect = resourceManager.filterHostsBasedOnStorageAccessGroups(allHosts, storageAccessGroups);

        Assert.assertTrue("No hosts should be included if storage access groups are empty.", hostsToConnect.isEmpty());
    }

    @Test
    public void testZoneLevelWithAllHostsUsingTags() {
        List<String> sagsToDelete = Arrays.asList("tag1", "tag2");
        Long clusterId = null;
        Long podId = null;
        Long zoneId = 3L;

        List<Long> hostIdsUsingStorageTags = Arrays.asList(1L, 2L);
        Mockito.doReturn(hostIdsUsingStorageTags).when(resourceManager).listOfHostIdsUsingTheStorageAccessGroups(sagsToDelete, clusterId, podId, zoneId);

        List<HostVO> hostsInZone = Arrays.asList(Mockito.mock(HostVO.class), Mockito.mock(HostVO.class));
        Mockito.doReturn(hostsInZone).when(hostDao).findByDataCenterId(zoneId);

        Mockito.doReturn(1L).when(hostsInZone.get(0)).getId();
        Mockito.doReturn(2L).when(hostsInZone.get(1)).getId();

        try {
            resourceManager.checkIfAllHostsInUse(sagsToDelete, clusterId, podId, zoneId);
            Assert.fail("Exception should be thrown when all hosts in the zone are using the storage access groups.");
        } catch (CloudRuntimeException e) {
            Assert.assertEquals("All hosts in the zone are using the storage access groups", e.getMessage());
        }
    }

    @Test
    public void testClusterLevelWithAllHostsUsingTags() {
        List<String> sagsToDelete = Arrays.asList("tag1", "tag2");
        Long clusterId = 1L;
        Long podId = null;
        Long zoneId = null;

        List<Long> hostIdsUsingStorageTags = Arrays.asList(1L, 2L);
        Mockito.doReturn(hostIdsUsingStorageTags).when(resourceManager).listOfHostIdsUsingTheStorageAccessGroups(sagsToDelete, clusterId, podId, zoneId);

        List<HostVO> hostsInCluster = Arrays.asList(Mockito.mock(HostVO.class), Mockito.mock(HostVO.class));
        Mockito.doReturn(hostsInCluster).when(hostDao).findByClusterId(clusterId, Host.Type.Routing);

        Mockito.doReturn(1L).when(hostsInCluster.get(0)).getId();
        Mockito.doReturn(2L).when(hostsInCluster.get(1)).getId();

        try {
            resourceManager.checkIfAllHostsInUse(sagsToDelete, clusterId, podId, zoneId);
            Assert.fail("Exception should be thrown when all hosts in the cluster are using the storage access groups.");
        } catch (CloudRuntimeException e) {
            Assert.assertEquals("All hosts in the cluster are using the storage access groups", e.getMessage());
        }
    }

    @Test
    public void testPodLevelWithAllHostsUsingTags() {
        List<String> sagsToDelete = Arrays.asList("tag1", "tag2");
        Long clusterId = null;
        Long podId = 2L;
        Long zoneId = null;

        List<Long> hostIdsUsingStorageTags = Arrays.asList(1L, 2L);
        Mockito.doReturn(hostIdsUsingStorageTags).when(resourceManager).listOfHostIdsUsingTheStorageAccessGroups(sagsToDelete, clusterId, podId, zoneId);

        List<HostVO> hostsInPod = Arrays.asList(Mockito.mock(HostVO.class), Mockito.mock(HostVO.class));
        Mockito.doReturn(hostsInPod).when(hostDao).findByPodId(podId, Host.Type.Routing);

        Mockito.doReturn(1L).when(hostsInPod.get(0)).getId();
        Mockito.doReturn(2L).when(hostsInPod.get(1)).getId();

        try {
            resourceManager.checkIfAllHostsInUse(sagsToDelete, clusterId, podId, zoneId);
            Assert.fail("Exception should be thrown when all hosts in the pod are using the storage access groups.");
        } catch (CloudRuntimeException e) {
            Assert.assertEquals("All hosts in the pod are using the storage access groups", e.getMessage());
        }
    }

    @Test
    public void testCheckIfAnyVolumesInUseWithPoolsToAdd() {
        List<String> sagsToAdd = Arrays.asList("sag1", "sag2");
        List<String> sagsToDelete = Arrays.asList("sag3", "sag4");

        HostVO host = Mockito.mock(HostVO.class);
        Mockito.when(host.getId()).thenReturn(1L);
        Mockito.when(host.getDataCenterId()).thenReturn(2L);
        Mockito.when(host.getPodId()).thenReturn(3L);
        Mockito.when(host.getClusterId()).thenReturn(4L);

        VolumeVO volume1 = Mockito.mock(VolumeVO.class);
        VolumeVO volume2 = Mockito.mock(VolumeVO.class);
        Mockito.when(volume1.getPoolId()).thenReturn(10L);
        Mockito.when(volume2.getPoolId()).thenReturn(11L);
        List<VolumeVO> volumesUsingTheStoragePoolAccessGroups = new ArrayList<>(Arrays.asList(volume1, volume2));
        Mockito.doReturn(volumesUsingTheStoragePoolAccessGroups).when(resourceManager).listOfVolumesUsingTheStorageAccessGroups(sagsToDelete, 1L, null, null, null);

        StoragePoolVO pool1 = Mockito.mock(StoragePoolVO.class);
        StoragePoolVO pool2 = Mockito.mock(StoragePoolVO.class);
        Mockito.when(pool1.getId()).thenReturn(10L);
        Mockito.when(pool2.getId()).thenReturn(12L);
        List<StoragePoolVO> poolsToAdd = Arrays.asList(pool1, pool2);

        Mockito.doReturn(poolsToAdd)
                .when(resourceManager).getStoragePoolsByAccessGroups(2L, 3L, 4L, sagsToAdd.toArray(new String[0]), true);

        try {
            resourceManager.checkIfAnyVolumesInUse(sagsToAdd, sagsToDelete, host);
            Assert.fail("Expected a CloudRuntimeException to be thrown.");
        } catch (CloudRuntimeException e) {
            Assert.assertTrue("Exception message should mention volumes in use.",
                    e.getMessage().contains("There are volumes in storage pools with the Storage Access Groups that need to be deleted"));
        }
    }

    @Test
    public void testUpdateStoragePoolConnectionsOnHostsConnect1AndDisconnect2() {
        Long poolId = 1L;
        List<String> storageAccessGroups = Arrays.asList("sag1", "sag2");

        StoragePoolVO storagePool = Mockito.mock(StoragePoolVO.class);
        Mockito.when(storagePool.getId()).thenReturn(poolId);
        Mockito.when(storagePool.getScope()).thenReturn(ScopeType.CLUSTER);
        Mockito.when(storagePool.getClusterId()).thenReturn(1L);
        Mockito.when(storagePool.getPodId()).thenReturn(1L);
        Mockito.when(storagePool.getDataCenterId()).thenReturn(1L);

        Mockito.when(storagePoolDao.findById(poolId)).thenReturn(storagePool);

        HostVO host1 = Mockito.mock(HostVO.class);
        HostVO host2 = Mockito.mock(HostVO.class);
        Mockito.when(host1.getId()).thenReturn(2L);
        Mockito.when(host2.getId()).thenReturn(3L);

        List<HostVO> clusterHosts = Arrays.asList(host1, host2);
        Mockito.doReturn(clusterHosts).when(resourceManager).listAllUpHosts(Host.Type.Routing, 1L, 1L, 1L);

        StoragePoolHostVO hostPoolRecord = Mockito.mock(StoragePoolHostVO.class);
        Mockito.when(storagePoolHostDao.findByPoolHost(poolId, 2L)).thenReturn(null);
        Mockito.when(storagePoolHostDao.findByPoolHost(poolId, 3L)).thenReturn(hostPoolRecord);

        Mockito.doReturn(new String[]{"sag1", "sag2"}).when(storageManager).getStorageAccessGroups(null, null, null, 2L);
        Mockito.doReturn(new String[]{"sag3"}).when(storageManager).getStorageAccessGroups(null, null, null, 3L);

        Mockito.doReturn(new ArrayList<Long>()).when(resourceManager).listOfHostIdsUsingTheStoragePool(poolId);

        try {
            resourceManager.updateStoragePoolConnectionsOnHosts(poolId, storageAccessGroups);

            Mockito.verify(resourceManager, Mockito.times(1)).connectHostToStoragePool(host1, storagePool);
            Mockito.verify(resourceManager, Mockito.never()).connectHostToStoragePool(host2, storagePool);
            Mockito.verify(resourceManager, Mockito.times(1)).disconnectHostFromStoragePool(host2, storagePool);
            Mockito.verify(resourceManager, Mockito.never()).disconnectHostFromStoragePool(host1, storagePool);
        } catch (CloudRuntimeException e) {
            Assert.fail("No exception should be thrown.");
        }
    }

    @Test
    public void testUpdateStoragePoolConnectionsOnHosts_ZoneScope_NoAccessGroups() {
        Long poolId = 1L;
        List<String> storageAccessGroups = new ArrayList<>();

        StoragePoolVO storagePool = Mockito.mock(StoragePoolVO.class);
        Mockito.when(storagePool.getId()).thenReturn(poolId);
        Mockito.when(storagePool.getScope()).thenReturn(ScopeType.ZONE);
        Mockito.when(storagePool.getDataCenterId()).thenReturn(1L);

        Mockito.when(storagePoolDao.findById(poolId)).thenReturn(storagePool);

        HostVO host1 = Mockito.mock(HostVO.class);
        HostVO host2 = Mockito.mock(HostVO.class);
        Mockito.when(host1.getId()).thenReturn(2L);
        Mockito.when(host2.getId()).thenReturn(3L);

        List<HostVO> zoneHosts = Arrays.asList(host1, host2);
        Mockito.doReturn(zoneHosts).when(resourceManager).listAllUpHosts(Host.Type.Routing, null, null, 1L);

        Mockito.doReturn(new String[]{"sag1", "sag2"}).when(storageManager).getStorageAccessGroups(null, null, null, 2L);
        Mockito.doReturn(new String[]{""}).when(storageManager).getStorageAccessGroups(null, null, null, 3L);

        Mockito.when(storagePoolHostDao.findByPoolHost(poolId, 2L)).thenReturn(null);
        Mockito.when(storagePoolHostDao.findByPoolHost(poolId, 3L)).thenReturn(null);

        try {
            resourceManager.updateStoragePoolConnectionsOnHosts(poolId, storageAccessGroups);

            Mockito.verify(resourceManager, Mockito.times(1)).connectHostToStoragePool(host1, storagePool);
            Mockito.verify(resourceManager, Mockito.times(1)).connectHostToStoragePool(host2, storagePool);
            Mockito.verify(resourceManager, Mockito.never()).disconnectHostFromStoragePool(Mockito.any(), Mockito.eq(storagePool));
        } catch (CloudRuntimeException e) {
            Assert.fail("No exception should be thrown.");
        }
    }

    @Test
    public void testUpdateStoragePoolConnectionsOnHosts_ConflictWithHostIdsAndVolumes() {
        Long poolId = 1L;
        List<String> storageAccessGroups = Arrays.asList("sag1", "sag2");

        StoragePoolVO storagePool = Mockito.mock(StoragePoolVO.class);
        Mockito.when(storagePool.getId()).thenReturn(poolId);
        Mockito.when(storagePool.getScope()).thenReturn(ScopeType.CLUSTER);
        Mockito.when(storagePool.getClusterId()).thenReturn(1L);
        Mockito.when(storagePool.getPodId()).thenReturn(1L);
        Mockito.when(storagePool.getDataCenterId()).thenReturn(1L);

        Mockito.when(storagePoolDao.findById(poolId)).thenReturn(storagePool);

        HostVO host1 = Mockito.mock(HostVO.class);
        HostVO host2 = Mockito.mock(HostVO.class);
        Mockito.when(host1.getId()).thenReturn(2L);
        Mockito.when(host2.getId()).thenReturn(3L);

        List<HostVO> clusterHosts = Arrays.asList(host1, host2);
        Mockito.doReturn(clusterHosts).when(resourceManager).listAllUpHosts(Host.Type.Routing, 1L, 1L, 1L);

        VolumeVO volume1 = Mockito.mock(VolumeVO.class);
        VolumeVO volume2 = Mockito.mock(VolumeVO.class);

        Mockito.when(volume1.getInstanceId()).thenReturn(100L);
        Mockito.when(volume2.getInstanceId()).thenReturn(101L);

        List<VolumeVO> volumesInPool = Arrays.asList(volume1, volume2);
        Mockito.doReturn(volumesInPool).when(volumeDao).findNonDestroyedVolumesByPoolId(poolId);

        VMInstanceVO vmInstance1 = Mockito.mock(VMInstanceVO.class);
        VMInstanceVO vmInstance2 = Mockito.mock(VMInstanceVO.class);
        Mockito.when(vmInstance1.getHostId()).thenReturn(2L);
        Mockito.when(vmInstance2.getHostId()).thenReturn(3L);

        Mockito.doReturn(vmInstance1).when(vmInstanceDao).findById(100L);
        Mockito.doReturn(vmInstance2).when(vmInstanceDao).findById(101L);

        Mockito.when(storagePoolHostDao.findByPoolHost(poolId, 2L)).thenReturn(null);
        Mockito.when(storagePoolHostDao.findByPoolHost(poolId, 3L)).thenReturn(null);

        Mockito.doReturn(new String[]{"sag1"}).when(storageManager).getStorageAccessGroups(null, null, null, 2L);
        Mockito.doReturn(new String[]{"sag3"}).when(storageManager).getStorageAccessGroups(null, null, null, 3L);

        Mockito.doReturn(Arrays.asList(2L, 3L)).when(resourceManager).listOfHostIdsUsingTheStoragePool(poolId);

        try {
            resourceManager.updateStoragePoolConnectionsOnHosts(poolId, storageAccessGroups);
            Assert.fail("Expected a CloudRuntimeException to be thrown.");
        } catch (CloudRuntimeException e) {
            Assert.assertTrue(e.getMessage().contains("Storage access groups cannot be updated as they are currently in use by some hosts."));
            Mockito.verify(resourceManager, Mockito.never()).connectHostToStoragePool(Mockito.any(), Mockito.eq(storagePool));
            Mockito.verify(resourceManager, Mockito.never()).disconnectHostFromStoragePool(Mockito.any(), Mockito.eq(storagePool));
        }
    }

    @Test(expected = CloudRuntimeException.class)
    public void testNoUpHostsThrowsException() {
        PrimaryDataStoreInfo primaryStore = Mockito.mock(PrimaryDataStoreInfo.class);
        Mockito.when(primaryStore.getClusterId()).thenReturn(1L);
        Mockito.doReturn(Collections.emptyList()).when(resourceManager).listAllUpHosts(Mockito.any(), Mockito.anyLong(), Mockito.any(), Mockito.anyLong());
        resourceManager.getEligibleUpHostsInClusterForStorageConnection(primaryStore);
    }

    @Test(expected = CloudRuntimeException.class)
    public void testNoUpAndEnabledHostsThrowsException() {
        PrimaryDataStoreInfo primaryStore = Mockito.mock(PrimaryDataStoreInfo.class);
        Mockito.when(primaryStore.getClusterId()).thenReturn(1L);
        Mockito.doReturn(Collections.emptyList()).when(resourceManager).listAllUpAndEnabledHosts(Mockito.any(), Mockito.anyLong(), Mockito.any(), Mockito.anyLong());
        resourceManager.getEligibleUpAndEnabledHostsInClusterForStorageConnection(primaryStore);
    }

    @Test
    public void testEligibleHostsMatchingStorageAccessGroups() {
        PrimaryDataStoreInfo primaryStore = Mockito.mock(PrimaryDataStoreInfo.class);
        DataStore dataStore = Mockito.mock(DataStore.class);
        Mockito.when(primaryStore.getId()).thenReturn(1L);
        Mockito.when(dataStore.getId()).thenReturn(1L);
        Mockito.when(primaryStore.getClusterId()).thenReturn(1L);

        HostVO host1 = Mockito.mock(HostVO.class);
        HostVO host2 = Mockito.mock(HostVO.class);
        List<HostVO> allHosts = Arrays.asList(host1, host2);

        Mockito.when(host1.getId()).thenReturn(1L);
        Mockito.when(host2.getId()).thenReturn(2L);

        Mockito.doReturn(allHosts).when(resourceManager).listAllUpHosts(Mockito.any(), Mockito.anyLong(), Mockito.any(), Mockito.anyLong());
        Mockito.doReturn(allHosts).when(resourceManager).listAllUpAndEnabledHosts(Mockito.any(), Mockito.anyLong(), Mockito.any(), Mockito.anyLong());
        Mockito.doReturn(allHosts).when(resourceManager).listAllUpAndEnabledHostsInOneZoneByHypervisor(Mockito.any(), Mockito.anyLong());
        Mockito.doReturn(Arrays.asList("group1", "group2")).when(storagePoolAccessGroupMapDao).getStorageAccessGroups(1L);

        Mockito.doReturn(new String[]{"group1"})
                .when(storageManager).getStorageAccessGroups(null, null, null, 1L);
        Mockito.doReturn(new String[]{"group3"})
                .when(storageManager).getStorageAccessGroups(null, null, null, 2L);

        List<HostVO> hostsToConnect = resourceManager.getEligibleUpHostsInClusterForStorageConnection(primaryStore);

        Assert.assertEquals("Only one host should match the storage access groups.", 1, hostsToConnect.size());
        Assert.assertTrue("Host1 should be included as it matches the storage access group.", hostsToConnect.contains(host1));
        Assert.assertFalse("Host2 should not be included as it does not match any storage access group.", hostsToConnect.contains(host2));

        hostsToConnect = resourceManager.getEligibleUpAndEnabledHostsInClusterForStorageConnection(primaryStore);

        Assert.assertEquals("Only one host should match the storage access groups.", 1, hostsToConnect.size());
        Assert.assertTrue("Host1 should be included as it matches the storage access group.", hostsToConnect.contains(host1));
        Assert.assertFalse("Host2 should not be included as it does not match any storage access group.", hostsToConnect.contains(host2));

        hostsToConnect = resourceManager.getEligibleUpAndEnabledHostsInZoneForStorageConnection(dataStore, 1L, Hypervisor.HypervisorType.KVM);

        Assert.assertEquals("Only one host should match the storage access groups.", 1, hostsToConnect.size());
        Assert.assertTrue("Host1 should be included as it matches the storage access group.", hostsToConnect.contains(host1));
        Assert.assertFalse("Host2 should not be included as it does not match any storage access group.", hostsToConnect.contains(host2));
    }

    @Test
    public void testUpdateZoneStorageAccessGroups() {
        long zoneId = 1L;
        long podId = 2L;
        long clusterId = 3L;
        long host1Id = 1L;
        long host2Id = 2L;
        List<String> newStorageAccessGroups = Arrays.asList("group1", "group2");

        DataCenterVO zoneVO = Mockito.mock(DataCenterVO.class);
        Mockito.when(dcDao.findById(zoneId)).thenReturn(zoneVO);
        Mockito.when(zoneVO.getId()).thenReturn(zoneId);
        Mockito.when(zoneVO.getStorageAccessGroups()).thenReturn("group1,group3");

        HostVO host1 = Mockito.mock(HostVO.class);
        HostVO host2 = Mockito.mock(HostVO.class);
        Mockito.when(host1.getId()).thenReturn(host1Id);
        Mockito.when(host2.getId()).thenReturn(host2Id);

        HostPodVO pod1 = Mockito.mock(HostPodVO.class);
        ClusterVO cluster1 = Mockito.mock(ClusterVO.class);
        Mockito.when(pod1.getId()).thenReturn(podId);
        Mockito.when(cluster1.getId()).thenReturn(clusterId);
        Mockito.when(podDao.findById(podId)).thenReturn(pod1);
        Mockito.when(clusterDao.findById(clusterId)).thenReturn(cluster1);
        Mockito.when(podDao.listByDataCenterId(zoneId)).thenReturn(Collections.singletonList(pod1));
        Mockito.when(clusterDao.listByPodId(podId)).thenReturn(Collections.singletonList(cluster1));
        Mockito.when(hostDao.findHypervisorHostInPod(podId)).thenReturn(Arrays.asList(host1, host2));
        Mockito.when(hostDao.findByDataCenterId(zoneId)).thenReturn(Arrays.asList(host1, host2));
        Mockito.when(hostDao.findByClusterId(clusterId)).thenReturn(Arrays.asList(host1, host2));

        List<Long> hostIdsUsingStorageTags = Arrays.asList(host1Id);
        Mockito.doReturn(hostIdsUsingStorageTags).when(resourceManager).listOfHostIdsUsingTheStorageAccessGroups(any(), any(), any(), any());

        Mockito.doReturn(new String[]{"group1", "group3"}).when(storageManager).getStorageAccessGroups(null, null, null, host1Id);
        Mockito.doReturn(new String[]{"group2", "group4"}).when(storageManager).getStorageAccessGroups(null, null, null, host2Id);

        resourceManager.updateZoneStorageAccessGroups(zoneId, newStorageAccessGroups);

        Mockito.verify(hostDao, Mockito.times(2)).update(host1Id, host1);
        Mockito.verify(hostDao, Mockito.times(1)).update(host2Id, host2);
    }

    @Test
    public void testUpdatePodStorageAccessGroups() {
        long podId = 2L;
        long clusterId = 3L;
        long host1Id = 1L;
        long host2Id = 2L;
        List<String> newStorageAccessGroups = Arrays.asList("group1", "group2");

        HostVO host1 = Mockito.mock(HostVO.class);
        HostVO host2 = Mockito.mock(HostVO.class);
        Mockito.when(host1.getId()).thenReturn(host1Id);
        Mockito.when(host2.getId()).thenReturn(host2Id);

        HostPodVO pod1 = Mockito.mock(HostPodVO.class);
        ClusterVO cluster1 = Mockito.mock(ClusterVO.class);
        Mockito.when(pod1.getStorageAccessGroups()).thenReturn("group1,group3");
        Mockito.when(cluster1.getId()).thenReturn(clusterId);
        Mockito.when(podDao.findById(podId)).thenReturn(pod1);
        Mockito.when(clusterDao.findById(clusterId)).thenReturn(cluster1);
        Mockito.when(clusterDao.listByPodId(podId)).thenReturn(Collections.singletonList(cluster1));
        Mockito.when(hostDao.findHypervisorHostInPod(podId)).thenReturn(Arrays.asList(host1, host2));
        Mockito.when(hostDao.findByPodId(podId, Host.Type.Routing)).thenReturn(Arrays.asList(host1, host2));
        Mockito.when(hostDao.findByClusterId(clusterId)).thenReturn(Arrays.asList(host1, host2));

        List<Long> hostIdsUsingStorageTags = Arrays.asList(host1Id);
        Mockito.doReturn(hostIdsUsingStorageTags).when(resourceManager).listOfHostIdsUsingTheStorageAccessGroups(any(), any(), any(), any());

        Mockito.doReturn(new String[]{"group1", "group3"}).when(storageManager).getStorageAccessGroups(null, null, null, host1Id);
        Mockito.doReturn(new String[]{"group2", "group4"}).when(storageManager).getStorageAccessGroups(null, null, null, host2Id);

        resourceManager.updatePodStorageAccessGroups(podId, newStorageAccessGroups);

        Mockito.verify(hostDao, Mockito.times(2)).update(host1Id, host1);
        Mockito.verify(hostDao, Mockito.times(1)).update(host2Id, host2);
    }

    @Test
    public void testUpdateClusterStorageAccessGroups() {
        long clusterId = 3L;
        long host1Id = 1L;
        long host2Id = 2L;
        List<String> newStorageAccessGroups = Arrays.asList("group1", "group2");

        HostVO host1 = Mockito.mock(HostVO.class);
        HostVO host2 = Mockito.mock(HostVO.class);
        Mockito.when(host1.getId()).thenReturn(host1Id);
        Mockito.when(host2.getId()).thenReturn(host2Id);

        ClusterVO cluster1 = Mockito.mock(ClusterVO.class);
        Mockito.when(cluster1.getStorageAccessGroups()).thenReturn("group1,group3");
        Mockito.when(cluster1.getId()).thenReturn(clusterId);
        Mockito.when(clusterDao.findById(clusterId)).thenReturn(cluster1);
        Mockito.when(hostDao.findHypervisorHostInCluster(clusterId)).thenReturn(Arrays.asList(host1, host2));
        Mockito.when(hostDao.findByClusterId(clusterId)).thenReturn(Arrays.asList(host1, host2));
        Mockito.when(hostDao.findByClusterId(clusterId, Host.Type.Routing)).thenReturn(Arrays.asList(host1, host2));

        List<Long> hostIdsUsingStorageTags = Arrays.asList(host1Id);
        Mockito.doReturn(hostIdsUsingStorageTags).when(resourceManager).listOfHostIdsUsingTheStorageAccessGroups(any(), any(), any(), any());

        Mockito.doReturn(new String[]{"group1", "group3"}).when(storageManager).getStorageAccessGroups(null, null, null, host1Id);
        Mockito.doReturn(new String[]{"group2", "group4"}).when(storageManager).getStorageAccessGroups(null, null, null, host2Id);

        resourceManager.updateClusterStorageAccessGroups(clusterId, newStorageAccessGroups);

        Mockito.verify(hostDao, Mockito.times(2)).update(host1Id, host1);
        Mockito.verify(hostDao, Mockito.times(1)).update(host2Id, host2);
    }

    @Test
    public void testUpdateHostStorageAccessGroups() {
        long hostId = 1L;
        long clusterId = 2L;
        List<String> newStorageAccessGroups = Arrays.asList("group1", "group2");

        HostVO host = Mockito.mock(HostVO.class);
        Mockito.when(host.getId()).thenReturn(hostId);
        Mockito.when(host.getClusterId()).thenReturn(clusterId);
        Mockito.when(host.getStorageAccessGroups()).thenReturn("group1,group3");

        Mockito.when(hostDao.findById(hostId)).thenReturn(host);
        Mockito.when(storageManager.getStorageAccessGroups(null, null, clusterId, null))
                .thenReturn(new String[]{"group3", "group4"});

        Mockito.doNothing().when(resourceManager).checkIfAnyVolumesInUse(any(), any(), any());
        Mockito.doNothing().when(resourceManager).updateConnectionsBetweenHostsAndStoragePools(any());

        resourceManager.updateHostStorageAccessGroups(hostId, newStorageAccessGroups);

        Mockito.verify(resourceManager).checkIfAnyVolumesInUse(eq(Arrays.asList("group1", "group2", "group3", "group4")),
                eq(Arrays.asList("group3")),
                eq(host));

        Mockito.verify(resourceManager).updateConnectionsBetweenHostsAndStoragePools(
                eq(Collections.singletonMap(host, Arrays.asList("group1", "group2", "group3", "group4")))
        );

        Mockito.verify(host).setStorageAccessGroups("group1,group2");
        Mockito.verify(hostDao).update(hostId, host);
    }

    @Test
    public void executeUserRequestDeleteHostPassesForcedFlags() throws Exception {
        Mockito.doReturn(true).when(resourceManager).doDeleteHost(anyLong(), anyBoolean(), anyBoolean());

        resourceManager.executeUserRequest(hostId, ResourceState.Event.DeleteHost, true, true);

        Mockito.verify(resourceManager).doDeleteHost(hostId, true, true);
    }

    @Test
    public void executeUserRequestDeleteHostPassesNonForcedFlags() throws Exception {
        Mockito.doReturn(true).when(resourceManager).doDeleteHost(anyLong(), anyBoolean(), anyBoolean());

        resourceManager.executeUserRequest(hostId, ResourceState.Event.DeleteHost, false, false);

        Mockito.verify(resourceManager).doDeleteHost(hostId, false, false);
    }

    @Test
    public void executeUserRequestDefaultOverloadPassesFalseForDeleteHost() throws Exception {
        Mockito.doReturn(true).when(resourceManager).doDeleteHost(anyLong(), anyBoolean(), anyBoolean());

        resourceManager.executeUserRequest(hostId, ResourceState.Event.DeleteHost);

        Mockito.verify(resourceManager).doDeleteHost(hostId, false, false);
    }
}