From 072efd85ab5a379692a608b359797ec9139409b7 Mon Sep 17 00:00:00 2001 From: Gabriel Brascher Date: Thu, 9 Jul 2020 23:02:06 -0300 Subject: [PATCH 01/11] Add strategy for stopping or migrating VMs with local storage when putting a host in maintenance --- .../com/cloud/resource/ResourceManager.java | 5 ++ .../cloud/resource/ResourceManagerImpl.java | 68 ++++++++++++++++++- .../main/java/com/cloud/vm/UserVmManager.java | 2 + .../java/com/cloud/vm/UserVmManagerImpl.java | 7 +- 4 files changed, 77 insertions(+), 5 deletions(-) diff --git a/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java b/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java index db7a27ff41c0..50f2a13abfd9 100755 --- a/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java +++ b/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java @@ -52,6 +52,11 @@ public interface ResourceManager extends ResourceService, Configurable { "Number of retries when preparing a host into Maintenance Mode is faulty before failing", false); + ConfigKey HOST_MAINTENANCE_LOCAL_STRATEGY = new ConfigKey<>("Advanced", String.class, + "host.maintenance.local.storage.strategy", "Error", + "Defines the strategy towards VMs with volumes on local storage when putting a host in maintenance. The default strategy is 'Error', preventing maintenance in such a case. To migrate away VMs running on local storage choose 'Migrating' strategy. To stop VMs, choose 'Stopping' strategy.", + true, ConfigKey.Scope.Global); + /** * Register a listener for different types of resource life cycle events. * There can only be one type of listener per type of host. diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index f2a3caa60004..61f1fd27c1c4 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -30,6 +30,18 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.deploy.DataCenterDeployment; +import com.cloud.deploy.DeployDestination; +import com.cloud.deploy.DeploymentPlanner; +import com.cloud.deploy.DeploymentPlanningManager; +import com.cloud.exception.InsufficientServerCapacityException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.storage.dao.DiskOfferingDao; +import com.cloud.vm.UserVmManager; +import com.cloud.vm.VirtualMachineProfile; +import com.cloud.vm.VirtualMachineProfileImpl; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.command.admin.cluster.AddClusterCmd; import org.apache.cloudstack.api.command.admin.cluster.DeleteClusterCmd; @@ -206,6 +218,10 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, @Inject private CapacityDao _capacityDao; @Inject + private DiskOfferingDao diskOfferingDao; + @Inject + private ServiceOfferingDao serviceOfferingDao; + @Inject private HostDao _hostDao; @Inject private HostDetailsDao _hostDetailsDao; @@ -226,6 +242,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, @Inject private IPAddressDao _publicIPAddressDao; @Inject + private DeploymentPlanningManager deploymentManager; + @Inject private VirtualMachineManager _vmMgr; @Inject private VMInstanceDao _vmDao; @@ -239,6 +257,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, private DedicatedResourceDao _dedicatedDao; @Inject private ServiceOfferingDetailsDao _serviceOfferingDetailsDao; + @Inject + private UserVmManager userVmManager; private List _discoverers; @@ -1273,6 +1293,12 @@ private boolean doMaintain(final long hostId) { } else if (HypervisorType.LXC.equals(host.getHypervisorType()) && VirtualMachine.Type.User.equals(vm.getType())){ //Migration is not supported for LXC Vms. Schedule restart instead. _haMgr.scheduleRestart(vm, false); + } else if (userVmManager.isVMUsingLocalStorage(vm.getId())) { + if (isMaintenanceLocalStrategyStopping()) { + _haMgr.scheduleStop(vm, hostId, WorkType.ForceStop); + } else if (isMaintenanceLocalStrategyMigrating()) { + migrateAwayVmWithVolume(host, vm); + } } else { s_logger.info("Maintenance: scheduling migration of VM " + vm.getUuid() + " from host " + host.getUuid()); _haMgr.scheduleMigration(vm); @@ -1282,6 +1308,26 @@ private boolean doMaintain(final long hostId) { return true; } + private void migrateAwayVmWithVolume(HostVO host, VMInstanceVO vm) { + final DataCenterDeployment plan = new DataCenterDeployment(host.getDataCenterId(), host.getPodId(), host.getClusterId(), null, null, null); + ServiceOfferingVO offeringVO = serviceOfferingDao.findById(vm.getServiceOfferingId()); + final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm, null, offeringVO, null, null); + plan.setMigrationPlan(true); + DeployDestination dest = null; + try { + dest = deploymentManager.planDeployment(profile, plan, new DeploymentPlanner.ExcludeList(), null); + } catch (InsufficientServerCapacityException e) { + throw new CloudRuntimeException(String.format("Maintenance failed, could not find deployment destination for VM [id=%s, name=%s].", vm.getId(), vm.getInstanceName()), e); + } + Host destHost = dest.getHost(); + + try { + _vmMgr.migrateWithStorage(vm.getUuid(), host.getId(), destHost.getId(), null); + } catch (ResourceUnavailableException e) { + throw new CloudRuntimeException(String.format("Maintenance failed, could not migrate VM [id=%s, name=%s] with local storage from host [id=%s, name=%s] to host [id=%s, name=%s].", vm.getId(), vm.getInstanceName(), host.getId(), host.getName(), destHost.getId(), destHost.getName()), e); + } + } + @Override public boolean maintain(final long hostId) throws AgentUnavailableException { final Boolean result = propagateResourceEvent(hostId, ResourceState.Event.AdminAskMaintenance); @@ -1322,9 +1368,13 @@ public Host maintain(final PrepareForMaintenanceCmd cmd) { } if (_storageMgr.isLocalStorageActiveOnHost(host.getId())) { - throw new CloudRuntimeException("There are active VMs using the host's local storage pool. Please stop all VMs on this host that use local storage."); + if(!isMaintenanceLocalStrategyMigrating() || !isMaintenanceLocalStrategyStopping()) { + throw new CloudRuntimeException("There are active VMs using the host's local storage pool. Please stop all VMs on this host that use local storage."); + } } + List migratingInVMs = _vmDao.findByHostInStates(hostId, State.Migrating); + if (migratingInVMs.size() > 0) { throw new CloudRuntimeException("Host contains incoming VMs migrating. Please wait for them to complete before putting to maintenance."); } @@ -1350,6 +1400,20 @@ public Host maintain(final PrepareForMaintenanceCmd cmd) { } } + protected boolean isMaintenanceLocalStrategyMigrating() { + if(org.apache.commons.lang3.StringUtils.isBlank(HOST_MAINTENANCE_LOCAL_STRATEGY.value())) { + return false; + } + return HOST_MAINTENANCE_LOCAL_STRATEGY.value().toLowerCase().equals(State.Migrating.toString().toLowerCase()); + } + + protected boolean isMaintenanceLocalStrategyStopping() { + if(org.apache.commons.lang3.StringUtils.isBlank(HOST_MAINTENANCE_LOCAL_STRATEGY.value())) { + return false; + } + return HOST_MAINTENANCE_LOCAL_STRATEGY.value().toLowerCase().equals(State.Stopping.toString().toLowerCase()); + } + /** * Add VNC details as user VM details for each VM in 'vms' (KVM hosts only) */ @@ -3091,6 +3155,6 @@ public String getConfigComponentName() { @Override public ConfigKey[] getConfigKeys() { - return new ConfigKey[0]; + return new ConfigKey[] {KvmSshToAgentEnabled, HOST_MAINTENANCE_LOCAL_STRATEGY}; } } diff --git a/server/src/main/java/com/cloud/vm/UserVmManager.java b/server/src/main/java/com/cloud/vm/UserVmManager.java index e8f709729c1e..c230a7d71167 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManager.java +++ b/server/src/main/java/com/cloud/vm/UserVmManager.java @@ -96,6 +96,8 @@ public interface UserVmManager extends UserVmService { void removeInstanceFromInstanceGroup(long vmId); + boolean isVMUsingLocalStorage(long vmId); + boolean expunge(UserVmVO vm, long callerUserId, Account caller); Pair> startVirtualMachine(long vmId, Long hostId, Map additionalParams, String deploymentPlannerToUse) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index a9c5a69abd9f..49555c1e2691 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -5775,10 +5775,10 @@ private void checkDestinationHypervisorType(StoragePool destPool, VMInstanceVO v } - private boolean isVMUsingLocalStorage(VMInstanceVO vm) { + public boolean isVMUsingLocalStorage(long vmId) { boolean usesLocalStorage = false; - List volumes = _volsDao.findByInstance(vm.getId()); + List volumes = _volsDao.findByInstance(vmId); for (VolumeVO vol : volumes) { DiskOfferingVO diskOffering = _diskOfferingDao.findById(vol.getDiskOfferingId()); if (diskOffering.isUseLocalStorage()) { @@ -5838,7 +5838,7 @@ public VirtualMachine migrateVirtualMachine(Long vmId, Host destinationHost) thr throw new InvalidParameterValueException("Unsupported Hypervisor Type for User VM migration, we support XenServer/VMware/KVM/Ovm/Hyperv/Ovm3 only"); } - if (isVMUsingLocalStorage(vm)) { + if (isVMUsingLocalStorage(vm.getId())) { if (s_logger.isDebugEnabled()) { s_logger.debug(vm + " is using Local Storage, cannot migrate this VM."); } @@ -6262,6 +6262,7 @@ public VirtualMachine migrateVirtualMachineWithVolume(Long vmId, Host destinatio List vmVolumes = _volsDao.findUsableVolumesForInstance(vm.getId()); Map volToPoolObjectMap = new HashMap(); + if (!isVMUsingLocalStorage(vm) && MapUtils.isEmpty(volumeToPool) && (destinationHost.getClusterId().equals(srcHost.getClusterId()) || isVmVolumesOnZoneWideStore(vm))){ // If volumes do not have to be migrated From f6cafb44286a844f1a7861a9f2022806787a12cf Mon Sep 17 00:00:00 2001 From: Gabriel Brascher Date: Wed, 5 Aug 2020 16:27:36 -0300 Subject: [PATCH 02/11] Change log level from debug to error --- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 49555c1e2691..ed94ead70db5 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -5828,9 +5828,7 @@ public VirtualMachine migrateVirtualMachine(Long vmId, Host destinationHost) thr } if (!isOnSupportedHypevisorForMigration(vm)) { - if (s_logger.isDebugEnabled()) { - s_logger.debug(vm + " is not XenServer/VMware/KVM/Ovm/Hyperv, cannot migrate this VM form hypervisor type " + vm.getHypervisorType()); - } + s_logger.error(vm + " is not XenServer/VMware/KVM/Ovm/Hyperv, cannot migrate this VM form hypervisor type " + vm.getHypervisorType()); throw new InvalidParameterValueException("Unsupported Hypervisor Type for VM migration, we support XenServer/VMware/KVM/Ovm/Hyperv/Ovm3 only"); } @@ -5839,9 +5837,7 @@ public VirtualMachine migrateVirtualMachine(Long vmId, Host destinationHost) thr } if (isVMUsingLocalStorage(vm.getId())) { - if (s_logger.isDebugEnabled()) { - s_logger.debug(vm + " is using Local Storage, cannot migrate this VM."); - } + s_logger.error(vm + " is using Local Storage, cannot migrate this VM."); throw new InvalidParameterValueException("Unsupported operation, VM uses Local storage, cannot migrate"); } From f98ec6df4ec74062e015bff117fe77e6b920c39e Mon Sep 17 00:00:00 2001 From: Gabriel Brascher Date: Wed, 5 Aug 2020 18:10:34 -0300 Subject: [PATCH 03/11] Code enhancements --- .../cloud/resource/ResourceManagerImpl.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index 61f1fd27c1c4..c56cdb5d84c6 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -1308,23 +1308,29 @@ private boolean doMaintain(final long hostId) { return true; } + /** + * Looks for Hosts able to allocate the VM and migrates the VM with its volume. + */ private void migrateAwayVmWithVolume(HostVO host, VMInstanceVO vm) { - final DataCenterDeployment plan = new DataCenterDeployment(host.getDataCenterId(), host.getPodId(), host.getClusterId(), null, null, null); - ServiceOfferingVO offeringVO = serviceOfferingDao.findById(vm.getServiceOfferingId()); - final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm, null, offeringVO, null, null); - plan.setMigrationPlan(true); + final DataCenterDeployment plan = new DataCenterDeployment(host.getDataCenterId(), host.getPodId(), host.getClusterId(), null, null, null); + ServiceOfferingVO offeringVO = serviceOfferingDao.findById(vm.getServiceOfferingId()); + final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm, null, offeringVO, null, null); + plan.setMigrationPlan(true); DeployDestination dest = null; try { dest = deploymentManager.planDeployment(profile, plan, new DeploymentPlanner.ExcludeList(), null); } catch (InsufficientServerCapacityException e) { - throw new CloudRuntimeException(String.format("Maintenance failed, could not find deployment destination for VM [id=%s, name=%s].", vm.getId(), vm.getInstanceName()), e); + throw new CloudRuntimeException(String.format("Maintenance failed, could not find deployment destination for VM [id=%s, name=%s].", vm.getId(), vm.getInstanceName()), + e); } Host destHost = dest.getHost(); try { _vmMgr.migrateWithStorage(vm.getUuid(), host.getId(), destHost.getId(), null); } catch (ResourceUnavailableException e) { - throw new CloudRuntimeException(String.format("Maintenance failed, could not migrate VM [id=%s, name=%s] with local storage from host [id=%s, name=%s] to host [id=%s, name=%s].", vm.getId(), vm.getInstanceName(), host.getId(), host.getName(), destHost.getId(), destHost.getName()), e); + throw new CloudRuntimeException( + String.format("Maintenance failed, could not migrate VM [id=%s, name=%s] with local storage from host [id=%s, name=%s] to host [id=%s, name=%s].", vm.getId(), + vm.getInstanceName(), host.getId(), host.getName(), destHost.getId(), destHost.getName()), e); } } From d91473fc251764cfe0d7092d8e97648fa019931a Mon Sep 17 00:00:00 2001 From: Gabriel Brascher Date: Mon, 22 Feb 2021 11:55:22 -0300 Subject: [PATCH 04/11] Merge master branch and fix conflicts --- .../java/com/cloud/resource/ResourceManagerImpl.java | 6 +++--- server/src/main/java/com/cloud/vm/UserVmManager.java | 2 +- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 9 ++++----- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index c56cdb5d84c6..dc447012eaea 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -1293,11 +1293,11 @@ private boolean doMaintain(final long hostId) { } else if (HypervisorType.LXC.equals(host.getHypervisorType()) && VirtualMachine.Type.User.equals(vm.getType())){ //Migration is not supported for LXC Vms. Schedule restart instead. _haMgr.scheduleRestart(vm, false); - } else if (userVmManager.isVMUsingLocalStorage(vm.getId())) { + } else if (userVmManager.isVMUsingLocalStorage(vm)) { if (isMaintenanceLocalStrategyStopping()) { _haMgr.scheduleStop(vm, hostId, WorkType.ForceStop); } else if (isMaintenanceLocalStrategyMigrating()) { - migrateAwayVmWithVolume(host, vm); + migrateAwayVmWithVolumes(host, vm); } } else { s_logger.info("Maintenance: scheduling migration of VM " + vm.getUuid() + " from host " + host.getUuid()); @@ -1311,7 +1311,7 @@ private boolean doMaintain(final long hostId) { /** * Looks for Hosts able to allocate the VM and migrates the VM with its volume. */ - private void migrateAwayVmWithVolume(HostVO host, VMInstanceVO vm) { + private void migrateAwayVmWithVolumes(HostVO host, VMInstanceVO vm) { final DataCenterDeployment plan = new DataCenterDeployment(host.getDataCenterId(), host.getPodId(), host.getClusterId(), null, null, null); ServiceOfferingVO offeringVO = serviceOfferingDao.findById(vm.getServiceOfferingId()); final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm, null, offeringVO, null, null); diff --git a/server/src/main/java/com/cloud/vm/UserVmManager.java b/server/src/main/java/com/cloud/vm/UserVmManager.java index c230a7d71167..e4206efe5d84 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManager.java +++ b/server/src/main/java/com/cloud/vm/UserVmManager.java @@ -96,7 +96,7 @@ public interface UserVmManager extends UserVmService { void removeInstanceFromInstanceGroup(long vmId); - boolean isVMUsingLocalStorage(long vmId); + boolean isVMUsingLocalStorage(VMInstanceVO vm); boolean expunge(UserVmVO vm, long callerUserId, Account caller); diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index ed94ead70db5..7de8f76f10c8 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -5775,10 +5775,10 @@ private void checkDestinationHypervisorType(StoragePool destPool, VMInstanceVO v } - public boolean isVMUsingLocalStorage(long vmId) { + public boolean isVMUsingLocalStorage(VMInstanceVO vm) { boolean usesLocalStorage = false; - List volumes = _volsDao.findByInstance(vmId); + List volumes = _volsDao.findByInstance(vm.getId()); for (VolumeVO vol : volumes) { DiskOfferingVO diskOffering = _diskOfferingDao.findById(vol.getDiskOfferingId()); if (diskOffering.isUseLocalStorage()) { @@ -5828,7 +5828,7 @@ public VirtualMachine migrateVirtualMachine(Long vmId, Host destinationHost) thr } if (!isOnSupportedHypevisorForMigration(vm)) { - s_logger.error(vm + " is not XenServer/VMware/KVM/Ovm/Hyperv, cannot migrate this VM form hypervisor type " + vm.getHypervisorType()); + s_logger.error(vm + " is not XenServer/VMware/KVM/Ovm/Hyperv, cannot migrate this VM from hypervisor type " + vm.getHypervisorType()); throw new InvalidParameterValueException("Unsupported Hypervisor Type for VM migration, we support XenServer/VMware/KVM/Ovm/Hyperv/Ovm3 only"); } @@ -5836,7 +5836,7 @@ public VirtualMachine migrateVirtualMachine(Long vmId, Host destinationHost) thr throw new InvalidParameterValueException("Unsupported Hypervisor Type for User VM migration, we support XenServer/VMware/KVM/Ovm/Hyperv/Ovm3 only"); } - if (isVMUsingLocalStorage(vm.getId())) { + if (isVMUsingLocalStorage(vm)) { s_logger.error(vm + " is using Local Storage, cannot migrate this VM."); throw new InvalidParameterValueException("Unsupported operation, VM uses Local storage, cannot migrate"); } @@ -6258,7 +6258,6 @@ public VirtualMachine migrateVirtualMachineWithVolume(Long vmId, Host destinatio List vmVolumes = _volsDao.findUsableVolumesForInstance(vm.getId()); Map volToPoolObjectMap = new HashMap(); - if (!isVMUsingLocalStorage(vm) && MapUtils.isEmpty(volumeToPool) && (destinationHost.getClusterId().equals(srcHost.getClusterId()) || isVmVolumesOnZoneWideStore(vm))){ // If volumes do not have to be migrated From d12df67aaf7521d05dc8fa5e22366fbd0299fb01 Mon Sep 17 00:00:00 2001 From: Gabriel Brascher Date: Mon, 22 Feb 2021 12:37:09 -0300 Subject: [PATCH 05/11] Change from ForceStop to Stop --- .../src/main/java/com/cloud/resource/ResourceManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index dc447012eaea..5939c75c8bbd 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -1295,7 +1295,7 @@ private boolean doMaintain(final long hostId) { _haMgr.scheduleRestart(vm, false); } else if (userVmManager.isVMUsingLocalStorage(vm)) { if (isMaintenanceLocalStrategyStopping()) { - _haMgr.scheduleStop(vm, hostId, WorkType.ForceStop); + _haMgr.scheduleStop(vm, hostId, WorkType.Stop); } else if (isMaintenanceLocalStrategyMigrating()) { migrateAwayVmWithVolumes(host, vm); } From 7a5d894489c71dca6da3fab08806c92c90783156 Mon Sep 17 00:00:00 2001 From: Gabriel Brascher Date: Mon, 1 Mar 2021 18:20:32 -0300 Subject: [PATCH 06/11] Add stop vs force-sto options for host.maintenance.local.storage.strategy --- .../com/cloud/resource/ResourceManager.java | 6 ++++- .../cloud/resource/ResourceManagerImpl.java | 23 +++++++++++++------ 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java b/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java index 50f2a13abfd9..de30475eb6f1 100755 --- a/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java +++ b/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java @@ -54,7 +54,11 @@ public interface ResourceManager extends ResourceService, Configurable { ConfigKey HOST_MAINTENANCE_LOCAL_STRATEGY = new ConfigKey<>("Advanced", String.class, "host.maintenance.local.storage.strategy", "Error", - "Defines the strategy towards VMs with volumes on local storage when putting a host in maintenance. The default strategy is 'Error', preventing maintenance in such a case. To migrate away VMs running on local storage choose 'Migrating' strategy. To stop VMs, choose 'Stopping' strategy.", + "Defines the strategy towards VMs with volumes on local storage when putting a host in maintenance. " + + "The default strategy is 'Error', preventing maintenance in such a case. " + + "Choose 'Migration' strategy to migrate away VMs running on local storage. " + + "To stop VMs, choose 'Stop' strategy. " + + "To force-stop VMs, choose 'ForceStop' strategy", true, ConfigKey.Scope.Global); /** diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index 5939c75c8bbd..8c64c6ff550e 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -1294,9 +1294,11 @@ private boolean doMaintain(final long hostId) { //Migration is not supported for LXC Vms. Schedule restart instead. _haMgr.scheduleRestart(vm, false); } else if (userVmManager.isVMUsingLocalStorage(vm)) { - if (isMaintenanceLocalStrategyStopping()) { + if (isMaintenanceLocalStrategyStop()) { _haMgr.scheduleStop(vm, hostId, WorkType.Stop); - } else if (isMaintenanceLocalStrategyMigrating()) { + } else if (isMaintenanceLocalStrategyForceStop()) { + _haMgr.scheduleStop(vm, hostId, WorkType.ForceStop); + } else if (isMaintenanceLocalStrategyMigrate()) { migrateAwayVmWithVolumes(host, vm); } } else { @@ -1374,7 +1376,7 @@ public Host maintain(final PrepareForMaintenanceCmd cmd) { } if (_storageMgr.isLocalStorageActiveOnHost(host.getId())) { - if(!isMaintenanceLocalStrategyMigrating() || !isMaintenanceLocalStrategyStopping()) { + if(!isMaintenanceLocalStrategyMigrate() || !isMaintenanceLocalStrategyStop()) { throw new CloudRuntimeException("There are active VMs using the host's local storage pool. Please stop all VMs on this host that use local storage."); } } @@ -1406,18 +1408,25 @@ public Host maintain(final PrepareForMaintenanceCmd cmd) { } } - protected boolean isMaintenanceLocalStrategyMigrating() { + protected boolean isMaintenanceLocalStrategyMigrate() { if(org.apache.commons.lang3.StringUtils.isBlank(HOST_MAINTENANCE_LOCAL_STRATEGY.value())) { return false; } - return HOST_MAINTENANCE_LOCAL_STRATEGY.value().toLowerCase().equals(State.Migrating.toString().toLowerCase()); + return HOST_MAINTENANCE_LOCAL_STRATEGY.value().toLowerCase().equals(WorkType.Migration.toString().toLowerCase()); } - protected boolean isMaintenanceLocalStrategyStopping() { + protected boolean isMaintenanceLocalStrategyStop() { if(org.apache.commons.lang3.StringUtils.isBlank(HOST_MAINTENANCE_LOCAL_STRATEGY.value())) { return false; } - return HOST_MAINTENANCE_LOCAL_STRATEGY.value().toLowerCase().equals(State.Stopping.toString().toLowerCase()); + return HOST_MAINTENANCE_LOCAL_STRATEGY.value().toLowerCase().equals(WorkType.Stop.toString().toLowerCase()); + } + + protected boolean isMaintenanceLocalStrategyForceStop() { + if(org.apache.commons.lang3.StringUtils.isBlank(HOST_MAINTENANCE_LOCAL_STRATEGY.value())) { + return false; + } + return HOST_MAINTENANCE_LOCAL_STRATEGY.value().toLowerCase().equals(WorkType.ForceStop.toString().toLowerCase()); } /** From 7d4c802bef677d99860093bd04252f8d27a7b985 Mon Sep 17 00:00:00 2001 From: Gabriel Brascher Date: Wed, 3 Mar 2021 07:01:48 -0300 Subject: [PATCH 07/11] Stop VM on Host must be of WokType ForceStop, due to host already on PrepareForMaintenance --- .../com/cloud/resource/ResourceManager.java | 1 - .../cloud/resource/ResourceManagerImpl.java | 23 ++++--------------- 2 files changed, 4 insertions(+), 20 deletions(-) diff --git a/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java b/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java index de30475eb6f1..92349ed80525 100755 --- a/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java +++ b/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java @@ -57,7 +57,6 @@ public interface ResourceManager extends ResourceService, Configurable { "Defines the strategy towards VMs with volumes on local storage when putting a host in maintenance. " + "The default strategy is 'Error', preventing maintenance in such a case. " + "Choose 'Migration' strategy to migrate away VMs running on local storage. " - + "To stop VMs, choose 'Stop' strategy. " + "To force-stop VMs, choose 'ForceStop' strategy", true, ConfigKey.Scope.Global); diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index 8c64c6ff550e..0ec7d855ddb1 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -1294,9 +1294,7 @@ private boolean doMaintain(final long hostId) { //Migration is not supported for LXC Vms. Schedule restart instead. _haMgr.scheduleRestart(vm, false); } else if (userVmManager.isVMUsingLocalStorage(vm)) { - if (isMaintenanceLocalStrategyStop()) { - _haMgr.scheduleStop(vm, hostId, WorkType.Stop); - } else if (isMaintenanceLocalStrategyForceStop()) { + if (isMaintenanceLocalStrategyForceStop()) { _haMgr.scheduleStop(vm, hostId, WorkType.ForceStop); } else if (isMaintenanceLocalStrategyMigrate()) { migrateAwayVmWithVolumes(host, vm); @@ -1376,7 +1374,7 @@ public Host maintain(final PrepareForMaintenanceCmd cmd) { } if (_storageMgr.isLocalStorageActiveOnHost(host.getId())) { - if(!isMaintenanceLocalStrategyMigrate() || !isMaintenanceLocalStrategyStop()) { + if(!isMaintenanceLocalStrategyMigrate() || !isMaintenanceLocalStrategyForceStop()) { throw new CloudRuntimeException("There are active VMs using the host's local storage pool. Please stop all VMs on this host that use local storage."); } } @@ -1409,24 +1407,11 @@ public Host maintain(final PrepareForMaintenanceCmd cmd) { } protected boolean isMaintenanceLocalStrategyMigrate() { - if(org.apache.commons.lang3.StringUtils.isBlank(HOST_MAINTENANCE_LOCAL_STRATEGY.value())) { - return false; - } - return HOST_MAINTENANCE_LOCAL_STRATEGY.value().toLowerCase().equals(WorkType.Migration.toString().toLowerCase()); - } - - protected boolean isMaintenanceLocalStrategyStop() { - if(org.apache.commons.lang3.StringUtils.isBlank(HOST_MAINTENANCE_LOCAL_STRATEGY.value())) { - return false; - } - return HOST_MAINTENANCE_LOCAL_STRATEGY.value().toLowerCase().equals(WorkType.Stop.toString().toLowerCase()); + return WorkType.Migration.toString().toLowerCase().equals(HOST_MAINTENANCE_LOCAL_STRATEGY.value().toLowerCase()); } protected boolean isMaintenanceLocalStrategyForceStop() { - if(org.apache.commons.lang3.StringUtils.isBlank(HOST_MAINTENANCE_LOCAL_STRATEGY.value())) { - return false; - } - return HOST_MAINTENANCE_LOCAL_STRATEGY.value().toLowerCase().equals(WorkType.ForceStop.toString().toLowerCase()); + return WorkType.ForceStop.toString().toLowerCase().equals(HOST_MAINTENANCE_LOCAL_STRATEGY.value().toLowerCase()); } /** From d644d0f91be4175dcc4b20bca9b64db932062499 Mon Sep 17 00:00:00 2001 From: Gabriel Brascher Date: Wed, 3 Mar 2021 11:47:35 -0300 Subject: [PATCH 08/11] Enhance log in case of User chooses a non-existing strategy. --- .../cloud/resource/ResourceManagerImpl.java | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index 0ec7d855ddb1..bb7a4d123ec5 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -1298,6 +1298,11 @@ private boolean doMaintain(final long hostId) { _haMgr.scheduleStop(vm, hostId, WorkType.ForceStop); } else if (isMaintenanceLocalStrategyMigrate()) { migrateAwayVmWithVolumes(host, vm); + } else if (!isMaintenanceLocalStrategyDefault()){ + String.format( + "Unsupported host.maintenance.local.storage.strategy: %s. Please set a strategy according to the global settings description: " + + "'Error', 'Migration', or 'ForceStop'.", + HOST_MAINTENANCE_LOCAL_STRATEGY.value().toString()); } } else { s_logger.info("Maintenance: scheduling migration of VM " + vm.getUuid() + " from host " + host.getUuid()); @@ -1407,11 +1412,28 @@ public Host maintain(final PrepareForMaintenanceCmd cmd) { } protected boolean isMaintenanceLocalStrategyMigrate() { - return WorkType.Migration.toString().toLowerCase().equals(HOST_MAINTENANCE_LOCAL_STRATEGY.value().toLowerCase()); + if(org.apache.commons.lang3.StringUtils.isBlank(HOST_MAINTENANCE_LOCAL_STRATEGY.value())) { + return false; + } + return HOST_MAINTENANCE_LOCAL_STRATEGY.value().toLowerCase().equals(WorkType.Migration.toString().toLowerCase()); } protected boolean isMaintenanceLocalStrategyForceStop() { - return WorkType.ForceStop.toString().toLowerCase().equals(HOST_MAINTENANCE_LOCAL_STRATEGY.value().toLowerCase()); + if(org.apache.commons.lang3.StringUtils.isBlank(HOST_MAINTENANCE_LOCAL_STRATEGY.value())) { + return false; + } + return HOST_MAINTENANCE_LOCAL_STRATEGY.value().toLowerCase().equals(WorkType.ForceStop.toString().toLowerCase()); + } + + /** + * Returns true if the host.maintenance.local.storage.strategy is the Default: "Error", blank, empty, or null. + */ + protected boolean isMaintenanceLocalStrategyDefault() { + if (org.apache.commons.lang3.StringUtils.isBlank(HOST_MAINTENANCE_LOCAL_STRATEGY.value()) + || HOST_MAINTENANCE_LOCAL_STRATEGY.value().toLowerCase().equals(State.Error.toString().toLowerCase())) { + return true; + } + return false; } /** From 45c6d4237ce72c037ebb5f04d10f470e4bbb446c Mon Sep 17 00:00:00 2001 From: Gabriel Brascher Date: Wed, 3 Mar 2021 14:41:37 -0300 Subject: [PATCH 09/11] Set log level to warn if strategy does not exist --- .../src/main/java/com/cloud/resource/ResourceManagerImpl.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index bb7a4d123ec5..0a25187e78fa 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -1299,10 +1299,12 @@ private boolean doMaintain(final long hostId) { } else if (isMaintenanceLocalStrategyMigrate()) { migrateAwayVmWithVolumes(host, vm); } else if (!isMaintenanceLocalStrategyDefault()){ - String.format( + String logMessage = String.format( "Unsupported host.maintenance.local.storage.strategy: %s. Please set a strategy according to the global settings description: " + "'Error', 'Migration', or 'ForceStop'.", HOST_MAINTENANCE_LOCAL_STRATEGY.value().toString()); + + s_logger.warn(logMessage); } } else { s_logger.info("Maintenance: scheduling migration of VM " + vm.getUuid() + " from host " + host.getUuid()); From f909a54872751f572b47d86ec489a3d3b4fdc337 Mon Sep 17 00:00:00 2001 From: Gabriel Brascher Date: Thu, 4 Mar 2021 14:42:31 -0300 Subject: [PATCH 10/11] Set log level to warn if strategy does not exist --- .../main/java/com/cloud/resource/ResourceManagerImpl.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index 0a25187e78fa..d93381a8767b 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -1303,8 +1303,8 @@ private boolean doMaintain(final long hostId) { "Unsupported host.maintenance.local.storage.strategy: %s. Please set a strategy according to the global settings description: " + "'Error', 'Migration', or 'ForceStop'.", HOST_MAINTENANCE_LOCAL_STRATEGY.value().toString()); - - s_logger.warn(logMessage); + s_logger.error(logMessage); + throw new CloudRuntimeException("There are active VMs using the host's local storage pool. Please stop all VMs on this host that use local storage."); } } else { s_logger.info("Maintenance: scheduling migration of VM " + vm.getUuid() + " from host " + host.getUuid()); @@ -1381,7 +1381,7 @@ public Host maintain(final PrepareForMaintenanceCmd cmd) { } if (_storageMgr.isLocalStorageActiveOnHost(host.getId())) { - if(!isMaintenanceLocalStrategyMigrate() || !isMaintenanceLocalStrategyForceStop()) { + if(!isMaintenanceLocalStrategyMigrate() && !isMaintenanceLocalStrategyForceStop()) { throw new CloudRuntimeException("There are active VMs using the host's local storage pool. Please stop all VMs on this host that use local storage."); } } From c4cd6441be9aa9ac2fccc4e03f44496a4861672b Mon Sep 17 00:00:00 2001 From: Gabriel Brascher Date: Thu, 4 Mar 2021 14:44:36 -0300 Subject: [PATCH 11/11] Set log level to error and throw exception if strategy does not exist --- .../src/main/java/com/cloud/resource/ResourceManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index d93381a8767b..75a8aae83036 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -1431,7 +1431,7 @@ protected boolean isMaintenanceLocalStrategyForceStop() { * Returns true if the host.maintenance.local.storage.strategy is the Default: "Error", blank, empty, or null. */ protected boolean isMaintenanceLocalStrategyDefault() { - if (org.apache.commons.lang3.StringUtils.isBlank(HOST_MAINTENANCE_LOCAL_STRATEGY.value()) + if (org.apache.commons.lang3.StringUtils.isBlank(HOST_MAINTENANCE_LOCAL_STRATEGY.value().toString()) || HOST_MAINTENANCE_LOCAL_STRATEGY.value().toLowerCase().equals(State.Error.toString().toLowerCase())) { return true; }