Examples with VirtualMachine com.cloud.vm.VirtualMachine used on opensource projects

Example 41 with VirtualMachine

use of com.cloud.vm.VirtualMachine in project cloudstack by apache.

the class DeploymentPlanningManagerImpl method planDeployment.

@Override
public DeployDestination planDeployment(VirtualMachineProfile vmProfile, DeploymentPlan plan, ExcludeList avoids, DeploymentPlanner planner) throws InsufficientServerCapacityException, AffinityConflictException {
    ServiceOffering offering = vmProfile.getServiceOffering();
    int cpu_requested = offering.getCpu() * offering.getSpeed();
    long ram_requested = offering.getRamSize() * 1024L * 1024L;
    VirtualMachine vm = vmProfile.getVirtualMachine();
    DataCenter dc = _dcDao.findById(vm.getDataCenterId());
    if (vm.getType() == VirtualMachine.Type.User || vm.getType() == VirtualMachine.Type.DomainRouter) {
        checkForNonDedicatedResources(vmProfile, dc, avoids);
    }
    if (s_logger.isDebugEnabled()) {
        s_logger.debug("DeploymentPlanner allocation algorithm: " + planner);
        s_logger.debug("Trying to allocate a host and storage pools from dc:" + plan.getDataCenterId() + ", pod:" + plan.getPodId() + ",cluster:" + plan.getClusterId() + ", requested cpu: " + cpu_requested + ", requested ram: " + ram_requested);
        s_logger.debug("Is ROOT volume READY (pool already allocated)?: " + (plan.getPoolId() != null ? "Yes" : "No"));
    }
    String haVmTag = (String) vmProfile.getParameter(VirtualMachineProfile.Param.HaTag);
    if (plan.getHostId() != null && haVmTag == null) {
        Long hostIdSpecified = plan.getHostId();
        if (s_logger.isDebugEnabled()) {
            s_logger.debug("DeploymentPlan has host_id specified, choosing this host and making no checks on this host: " + hostIdSpecified);
        }
        HostVO host = _hostDao.findById(hostIdSpecified);
        if (host == null) {
            s_logger.debug("The specified host cannot be found");
        } else if (avoids.shouldAvoid(host)) {
            s_logger.debug("The specified host is in avoid set");
        } else {
            if (s_logger.isDebugEnabled()) {
                s_logger.debug("Looking for suitable pools for this host under zone: " + host.getDataCenterId() + ", pod: " + host.getPodId() + ", cluster: " + host.getClusterId());
            }
            Pod pod = _podDao.findById(host.getPodId());
            Cluster cluster = _clusterDao.findById(host.getClusterId());
            if (vm.getHypervisorType() == HypervisorType.BareMetal) {
                DeployDestination dest = new DeployDestination(dc, pod, cluster, host, new HashMap<Volume, StoragePool>());
                s_logger.debug("Returning Deployment Destination: " + dest);
                return dest;
            }
            // search for storage under the zone, pod, cluster of the host.
            DataCenterDeployment lastPlan = new DataCenterDeployment(host.getDataCenterId(), host.getPodId(), host.getClusterId(), hostIdSpecified, plan.getPoolId(), null, plan.getReservationContext());
            Pair<Map<Volume, List<StoragePool>>, List<Volume>> result = findSuitablePoolsForVolumes(vmProfile, lastPlan, avoids, HostAllocator.RETURN_UPTO_ALL);
            Map<Volume, List<StoragePool>> suitableVolumeStoragePools = result.first();
            List<Volume> readyAndReusedVolumes = result.second();
            // choose the potential pool for this VM for this host
            if (!suitableVolumeStoragePools.isEmpty()) {
                List<Host> suitableHosts = new ArrayList<Host>();
                suitableHosts.add(host);
                Pair<Host, Map<Volume, StoragePool>> potentialResources = findPotentialDeploymentResources(suitableHosts, suitableVolumeStoragePools, avoids, getPlannerUsage(planner, vmProfile, plan, avoids), readyAndReusedVolumes);
                if (potentialResources != null) {
                    pod = _podDao.findById(host.getPodId());
                    cluster = _clusterDao.findById(host.getClusterId());
                    Map<Volume, StoragePool> storageVolMap = potentialResources.second();
                    // we don't have to prepare this volume.
                    for (Volume vol : readyAndReusedVolumes) {
                        storageVolMap.remove(vol);
                    }
                    DeployDestination dest = new DeployDestination(dc, pod, cluster, host, storageVolMap);
                    s_logger.debug("Returning Deployment Destination: " + dest);
                    return dest;
                }
            }
        }
        s_logger.debug("Cannot deploy to specified host, returning.");
        return null;
    }
    // call affinitygroup chain
    long vmGroupCount = _affinityGroupVMMapDao.countAffinityGroupsForVm(vm.getId());
    if (vmGroupCount > 0) {
        for (AffinityGroupProcessor processor : _affinityProcessors) {
            processor.process(vmProfile, plan, avoids);
        }
    }
    if (vm.getType() == VirtualMachine.Type.User) {
        checkForNonDedicatedResources(vmProfile, dc, avoids);
    }
    if (s_logger.isDebugEnabled()) {
        s_logger.debug("Deploy avoids pods: " + avoids.getPodsToAvoid() + ", clusters: " + avoids.getClustersToAvoid() + ", hosts: " + avoids.getHostsToAvoid());
    }
    // check if datacenter is in avoid set
    if (avoids.shouldAvoid(dc)) {
        if (s_logger.isDebugEnabled()) {
            s_logger.debug("DataCenter id = '" + dc.getId() + "' provided is in avoid set, DeploymentPlanner cannot allocate the VM, returning.");
        }
        return null;
    }
    if (planner == null) {
        String plannerName = offering.getDeploymentPlanner();
        if (plannerName == null) {
            if (vm.getHypervisorType() == HypervisorType.BareMetal) {
                plannerName = "BareMetalPlanner";
            } else {
                plannerName = _configDao.getValue(Config.VmDeploymentPlanner.key());
            }
        }
        planner = getDeploymentPlannerByName(plannerName);
    }
    if (vm.getLastHostId() != null && haVmTag == null) {
        s_logger.debug("This VM has last host_id specified, trying to choose the same host: " + vm.getLastHostId());
        HostVO host = _hostDao.findById(vm.getLastHostId());
        ServiceOfferingDetailsVO offeringDetails = null;
        if (host == null) {
            s_logger.debug("The last host of this VM cannot be found");
        } else if (avoids.shouldAvoid(host)) {
            s_logger.debug("The last host of this VM is in avoid set");
        } else if (plan.getClusterId() != null && host.getClusterId() != null && !plan.getClusterId().equals(host.getClusterId())) {
            s_logger.debug("The last host of this VM cannot be picked as the plan specifies different clusterId: " + plan.getClusterId());
        } else if (_capacityMgr.checkIfHostReachMaxGuestLimit(host)) {
            s_logger.debug("The last Host, hostId: " + host.getId() + " already has max Running VMs(count includes system VMs), skipping this and trying other available hosts");
        } else if ((offeringDetails = _serviceOfferingDetailsDao.findDetail(offering.getId(), GPU.Keys.vgpuType.toString())) != null) {
            ServiceOfferingDetailsVO groupName = _serviceOfferingDetailsDao.findDetail(offering.getId(), GPU.Keys.pciDevice.toString());
            if (!_resourceMgr.isGPUDeviceAvailable(host.getId(), groupName.getValue(), offeringDetails.getValue())) {
                s_logger.debug("The last host of this VM does not have required GPU devices available");
            }
        } else {
            if (host.getStatus() == Status.Up && host.getResourceState() == ResourceState.Enabled) {
                boolean hostTagsMatch = true;
                if (offering.getHostTag() != null) {
                    _hostDao.loadHostTags(host);
                    if (!(host.getHostTags() != null && host.getHostTags().contains(offering.getHostTag()))) {
                        hostTagsMatch = false;
                    }
                }
                if (hostTagsMatch) {
                    long cluster_id = host.getClusterId();
                    ClusterDetailsVO cluster_detail_cpu = _clusterDetailsDao.findDetail(cluster_id, "cpuOvercommitRatio");
                    ClusterDetailsVO cluster_detail_ram = _clusterDetailsDao.findDetail(cluster_id, "memoryOvercommitRatio");
                    Float cpuOvercommitRatio = Float.parseFloat(cluster_detail_cpu.getValue());
                    Float memoryOvercommitRatio = Float.parseFloat(cluster_detail_ram.getValue());
                    boolean hostHasCpuCapability, hostHasCapacity = false;
                    hostHasCpuCapability = _capacityMgr.checkIfHostHasCpuCapability(host.getId(), offering.getCpu(), offering.getSpeed());
                    if (hostHasCpuCapability) {
                        // first check from reserved capacity
                        hostHasCapacity = _capacityMgr.checkIfHostHasCapacity(host.getId(), cpu_requested, ram_requested, true, cpuOvercommitRatio, memoryOvercommitRatio, true);
                        // if not reserved, check the free capacity
                        if (!hostHasCapacity)
                            hostHasCapacity = _capacityMgr.checkIfHostHasCapacity(host.getId(), cpu_requested, ram_requested, false, cpuOvercommitRatio, memoryOvercommitRatio, true);
                    }
                    if (hostHasCapacity && hostHasCpuCapability) {
                        s_logger.debug("The last host of this VM is UP and has enough capacity");
                        s_logger.debug("Now checking for suitable pools under zone: " + host.getDataCenterId() + ", pod: " + host.getPodId() + ", cluster: " + host.getClusterId());
                        Pod pod = _podDao.findById(host.getPodId());
                        Cluster cluster = _clusterDao.findById(host.getClusterId());
                        if (vm.getHypervisorType() == HypervisorType.BareMetal) {
                            DeployDestination dest = new DeployDestination(dc, pod, cluster, host, new HashMap<Volume, StoragePool>());
                            s_logger.debug("Returning Deployment Destination: " + dest);
                            return dest;
                        }
                        // search for storage under the zone, pod, cluster
                        // of
                        // the last host.
                        DataCenterDeployment lastPlan = new DataCenterDeployment(host.getDataCenterId(), host.getPodId(), host.getClusterId(), host.getId(), plan.getPoolId(), null);
                        Pair<Map<Volume, List<StoragePool>>, List<Volume>> result = findSuitablePoolsForVolumes(vmProfile, lastPlan, avoids, HostAllocator.RETURN_UPTO_ALL);
                        Map<Volume, List<StoragePool>> suitableVolumeStoragePools = result.first();
                        List<Volume> readyAndReusedVolumes = result.second();
                        // host
                        if (!suitableVolumeStoragePools.isEmpty()) {
                            List<Host> suitableHosts = new ArrayList<Host>();
                            suitableHosts.add(host);
                            Pair<Host, Map<Volume, StoragePool>> potentialResources = findPotentialDeploymentResources(suitableHosts, suitableVolumeStoragePools, avoids, getPlannerUsage(planner, vmProfile, plan, avoids), readyAndReusedVolumes);
                            if (potentialResources != null) {
                                Map<Volume, StoragePool> storageVolMap = potentialResources.second();
                                // this volume.
                                for (Volume vol : readyAndReusedVolumes) {
                                    storageVolMap.remove(vol);
                                }
                                DeployDestination dest = new DeployDestination(dc, pod, cluster, host, storageVolMap);
                                s_logger.debug("Returning Deployment Destination: " + dest);
                                return dest;
                            }
                        }
                    } else {
                        s_logger.debug("The last host of this VM does not have enough capacity");
                    }
                } else {
                    s_logger.debug("Service Offering host tag does not match the last host of this VM");
                }
            } else {
                s_logger.debug("The last host of this VM is not UP or is not enabled, host status is: " + host.getStatus().name() + ", host resource state is: " + host.getResourceState());
            }
        }
        s_logger.debug("Cannot choose the last host to deploy this VM ");
    }
    DeployDestination dest = null;
    List<Long> clusterList = null;
    if (planner != null && planner.canHandle(vmProfile, plan, avoids)) {
        while (true) {
            if (planner instanceof DeploymentClusterPlanner) {
                ExcludeList plannerAvoidInput = new ExcludeList(avoids.getDataCentersToAvoid(), avoids.getPodsToAvoid(), avoids.getClustersToAvoid(), avoids.getHostsToAvoid(), avoids.getPoolsToAvoid());
                clusterList = ((DeploymentClusterPlanner) planner).orderClusters(vmProfile, plan, avoids);
                if (clusterList != null && !clusterList.isEmpty()) {
                    // planner refactoring. call allocators to list hosts
                    ExcludeList plannerAvoidOutput = new ExcludeList(avoids.getDataCentersToAvoid(), avoids.getPodsToAvoid(), avoids.getClustersToAvoid(), avoids.getHostsToAvoid(), avoids.getPoolsToAvoid());
                    resetAvoidSet(plannerAvoidOutput, plannerAvoidInput);
                    dest = checkClustersforDestination(clusterList, vmProfile, plan, avoids, dc, getPlannerUsage(planner, vmProfile, plan, avoids), plannerAvoidOutput);
                    if (dest != null) {
                        return dest;
                    }
                    // reset the avoid input to the planners
                    resetAvoidSet(avoids, plannerAvoidOutput);
                } else {
                    return null;
                }
            } else {
                dest = planner.plan(vmProfile, plan, avoids);
                if (dest != null) {
                    long hostId = dest.getHost().getId();
                    avoids.addHost(dest.getHost().getId());
                    if (checkIfHostFitsPlannerUsage(hostId, DeploymentPlanner.PlannerResourceUsage.Shared)) {
                        // found destination
                        return dest;
                    } else {
                        // deployment picked it up for dedicated access
                        continue;
                    }
                } else {
                    return null;
                }
            }
        }
    }
    return dest;
}

Example 42 with VirtualMachine

use of com.cloud.vm.VirtualMachine in project cloudstack by apache.

the class FirstFitPlanner method orderClusters.

@Override
public List<Long> orderClusters(VirtualMachineProfile vmProfile, DeploymentPlan plan, ExcludeList avoid) throws InsufficientServerCapacityException {
    VirtualMachine vm = vmProfile.getVirtualMachine();
    DataCenter dc = dcDao.findById(vm.getDataCenterId());
    //check if datacenter is in avoid set
    if (avoid.shouldAvoid(dc)) {
        if (s_logger.isDebugEnabled()) {
            s_logger.debug("DataCenter id = '" + dc.getId() + "' provided is in avoid set, DeploymentPlanner cannot allocate the VM, returning.");
        }
        return null;
    }
    List<Long> clusterList = new ArrayList<Long>();
    if (plan.getClusterId() != null) {
        Long clusterIdSpecified = plan.getClusterId();
        s_logger.debug("Searching resources only under specified Cluster: " + clusterIdSpecified);
        ClusterVO cluster = clusterDao.findById(plan.getClusterId());
        if (cluster != null) {
            if (avoid.shouldAvoid(cluster)) {
                s_logger.debug("The specified cluster is in avoid set, returning.");
            } else {
                clusterList.add(clusterIdSpecified);
                removeClustersCrossingThreshold(clusterList, avoid, vmProfile, plan);
            }
        } else {
            s_logger.debug("The specified cluster cannot be found, returning.");
            avoid.addCluster(plan.getClusterId());
            return null;
        }
    } else if (plan.getPodId() != null) {
        //consider clusters under this pod only
        Long podIdSpecified = plan.getPodId();
        s_logger.debug("Searching resources only under specified Pod: " + podIdSpecified);
        HostPodVO pod = podDao.findById(podIdSpecified);
        if (pod != null) {
            if (avoid.shouldAvoid(pod)) {
                s_logger.debug("The specified pod is in avoid set, returning.");
            } else {
                clusterList = scanClustersForDestinationInZoneOrPod(podIdSpecified, false, vmProfile, plan, avoid);
                if (clusterList == null) {
                    avoid.addPod(plan.getPodId());
                }
            }
        } else {
            s_logger.debug("The specified Pod cannot be found, returning.");
            avoid.addPod(plan.getPodId());
            return null;
        }
    } else {
        s_logger.debug("Searching all possible resources under this Zone: " + plan.getDataCenterId());
        boolean applyAllocationAtPods = Boolean.parseBoolean(configDao.getValue(Config.ApplyAllocationAlgorithmToPods.key()));
        if (applyAllocationAtPods) {
            //start scan at all pods under this zone.
            clusterList = scanPodsForDestination(vmProfile, plan, avoid);
        } else {
            //start scan at clusters under this zone.
            clusterList = scanClustersForDestinationInZoneOrPod(plan.getDataCenterId(), true, vmProfile, plan, avoid);
        }
    }
    if (clusterList != null && !clusterList.isEmpty()) {
        ServiceOffering offering = vmProfile.getServiceOffering();
        // In case of non-GPU VMs, protect GPU enabled Hosts and prefer VM deployment on non-GPU Hosts.
        if ((serviceOfferingDetailsDao.findDetail(offering.getId(), GPU.Keys.vgpuType.toString()) == null) && !(hostGpuGroupsDao.listHostIds().isEmpty())) {
            int requiredCpu = offering.getCpu() * offering.getSpeed();
            long requiredRam = offering.getRamSize() * 1024L * 1024L;
            reorderClustersBasedOnImplicitTags(clusterList, requiredCpu, requiredRam);
        }
    }
    return clusterList;
}

Example 43 with VirtualMachine

use of com.cloud.vm.VirtualMachine in project cloudstack by apache.

the class HypervisorGuruBase method toVirtualMachineTO.

protected VirtualMachineTO toVirtualMachineTO(VirtualMachineProfile vmProfile) {
    ServiceOffering offering = _serviceOfferingDao.findById(vmProfile.getId(), vmProfile.getServiceOfferingId());
    VirtualMachine vm = vmProfile.getVirtualMachine();
    Long minMemory = (long) (offering.getRamSize() / vmProfile.getMemoryOvercommitRatio());
    int minspeed = (int) (offering.getSpeed() / vmProfile.getCpuOvercommitRatio());
    int maxspeed = (offering.getSpeed());
    VirtualMachineTO to = new VirtualMachineTO(vm.getId(), vm.getInstanceName(), vm.getType(), offering.getCpu(), minspeed, maxspeed, minMemory * 1024l * 1024l, offering.getRamSize() * 1024l * 1024l, null, null, vm.isHaEnabled(), vm.limitCpuUse(), vm.getVncPassword());
    to.setBootArgs(vmProfile.getBootArgs());
    List<NicProfile> nicProfiles = vmProfile.getNics();
    NicTO[] nics = new NicTO[nicProfiles.size()];
    int i = 0;
    for (NicProfile nicProfile : nicProfiles) {
        nics[i++] = toNicTO(nicProfile);
    }
    to.setNics(nics);
    to.setDisks(vmProfile.getDisks().toArray(new DiskTO[vmProfile.getDisks().size()]));
    if (vmProfile.getTemplate().getBits() == 32) {
        to.setArch("i686");
    } else {
        to.setArch("x86_64");
    }
    Map<String, String> detailsInVm = _userVmDetailsDao.listDetailsKeyPairs(vm.getId());
    if (detailsInVm != null) {
        to.setDetails(detailsInVm);
    }
    // Set GPU details
    ServiceOfferingDetailsVO offeringDetail = null;
    if ((offeringDetail = _serviceOfferingDetailsDao.findDetail(offering.getId(), GPU.Keys.vgpuType.toString())) != null) {
        ServiceOfferingDetailsVO groupName = _serviceOfferingDetailsDao.findDetail(offering.getId(), GPU.Keys.pciDevice.toString());
        to.setGpuDevice(_resourceMgr.getGPUDevice(vm.getHostId(), groupName.getValue(), offeringDetail.getValue()));
    }
    // Workaround to make sure the TO has the UUID we need for Niciri integration
    VMInstanceVO vmInstance = _virtualMachineDao.findById(to.getId());
    // check if XStools/VMWare tools are present in the VM and dynamic scaling feature is enabled (per zone/global)
    Boolean isDynamicallyScalable = vmInstance.isDynamicallyScalable() && UserVmManager.EnableDynamicallyScaleVm.valueIn(vm.getDataCenterId());
    to.setEnableDynamicallyScaleVm(isDynamicallyScalable);
    to.setUuid(vmInstance.getUuid());
    to.setVmData(vmProfile.getVmData());
    to.setConfigDriveLabel(vmProfile.getConfigDriveLabel());
    to.setConfigDriveIsoRootFolder(vmProfile.getConfigDriveIsoRootFolder());
    to.setConfigDriveIsoFile(vmProfile.getConfigDriveIsoFile());
    return to;
}

Example 44 with VirtualMachine

use of com.cloud.vm.VirtualMachine in project cloudstack by apache.

the class HighAvailabilityManagerImpl method destroyVM.

protected Long destroyVM(final HaWorkVO work) {
    final VirtualMachine vm = _itMgr.findById(work.getInstanceId());
    s_logger.info("Destroying " + vm.toString());
    try {
        if (vm.getState() != State.Destroyed) {
            s_logger.info("VM is no longer in Destroyed state " + vm.toString());
            return null;
        }
        if (vm.getHostId() != null) {
            _itMgr.destroy(vm.getUuid(), false);
            s_logger.info("Successfully destroy " + vm);
            return null;
        } else {
            if (s_logger.isDebugEnabled()) {
                s_logger.debug(vm + " has already been stopped");
            }
            return null;
        }
    } catch (final AgentUnavailableException e) {
        s_logger.debug("Agnet is not available" + e.getMessage());
    } catch (OperationTimedoutException e) {
        s_logger.debug("operation timed out: " + e.getMessage());
    } catch (ConcurrentOperationException e) {
        s_logger.debug("concurrent operation: " + e.getMessage());
    }
    return (System.currentTimeMillis() >> 10) + _stopRetryInterval;
}

Example 45 with VirtualMachine

use of com.cloud.vm.VirtualMachine in project cloudstack by apache.

the class HighAvailabilityManagerImpl method restart.

protected Long restart(final HaWorkVO work) {
    List<HaWorkVO> items = _haDao.listFutureHaWorkForVm(work.getInstanceId(), work.getId());
    if (items.size() > 0) {
        StringBuilder str = new StringBuilder("Cancelling this work item because newer ones have been scheduled.  Work Ids = [");
        for (HaWorkVO item : items) {
            str.append(item.getId()).append(", ");
        }
        str.delete(str.length() - 2, str.length()).append("]");
        s_logger.info(str.toString());
        return null;
    }
    items = _haDao.listRunningHaWorkForVm(work.getInstanceId());
    if (items.size() > 0) {
        StringBuilder str = new StringBuilder("Waiting because there's HA work being executed on an item currently.  Work Ids =[");
        for (HaWorkVO item : items) {
            str.append(item.getId()).append(", ");
        }
        str.delete(str.length() - 2, str.length()).append("]");
        s_logger.info(str.toString());
        return (System.currentTimeMillis() >> 10) + _investigateRetryInterval;
    }
    long vmId = work.getInstanceId();
    VirtualMachine vm = _itMgr.findById(work.getInstanceId());
    if (vm == null) {
        s_logger.info("Unable to find vm: " + vmId);
        return null;
    }
    s_logger.info("HA on " + vm);
    if (vm.getState() != work.getPreviousState() || vm.getUpdated() != work.getUpdateTime()) {
        s_logger.info("VM " + vm + " has been changed.  Current State = " + vm.getState() + " Previous State = " + work.getPreviousState() + " last updated = " + vm.getUpdated() + " previous updated = " + work.getUpdateTime());
        return null;
    }
    AlertManager.AlertType alertType = AlertManager.AlertType.ALERT_TYPE_USERVM;
    if (VirtualMachine.Type.DomainRouter.equals(vm.getType())) {
        alertType = AlertManager.AlertType.ALERT_TYPE_DOMAIN_ROUTER;
    } else if (VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) {
        alertType = AlertManager.AlertType.ALERT_TYPE_CONSOLE_PROXY;
    } else if (VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType())) {
        alertType = AlertManager.AlertType.ALERT_TYPE_SSVM;
    }
    HostVO host = _hostDao.findById(work.getHostId());
    boolean isHostRemoved = false;
    if (host == null) {
        host = _hostDao.findByIdIncludingRemoved(work.getHostId());
        if (host != null) {
            s_logger.debug("VM " + vm.toString() + " is now no longer on host " + work.getHostId() + " as the host is removed");
            isHostRemoved = true;
        }
    }
    DataCenterVO dcVO = _dcDao.findById(host.getDataCenterId());
    HostPodVO podVO = _podDao.findById(host.getPodId());
    String hostDesc = "name: " + host.getName() + "(id:" + host.getId() + "), availability zone: " + dcVO.getName() + ", pod: " + podVO.getName();
    Boolean alive = null;
    if (work.getStep() == Step.Investigating) {
        if (!isHostRemoved) {
            if (vm.getHostId() == null || vm.getHostId() != work.getHostId()) {
                s_logger.info("VM " + vm.toString() + " is now no longer on host " + work.getHostId());
                return null;
            }
            Investigator investigator = null;
            for (Investigator it : investigators) {
                investigator = it;
                try {
                    alive = investigator.isVmAlive(vm, host);
                    s_logger.info(investigator.getName() + " found " + vm + " to be alive? " + alive);
                    break;
                } catch (UnknownVM e) {
                    s_logger.info(investigator.getName() + " could not find " + vm);
                }
            }
            boolean fenced = false;
            if (alive == null) {
                s_logger.debug("Fencing off VM that we don't know the state of");
                for (FenceBuilder fb : fenceBuilders) {
                    Boolean result = fb.fenceOff(vm, host);
                    s_logger.info("Fencer " + fb.getName() + " returned " + result);
                    if (result != null && result) {
                        fenced = true;
                        break;
                    }
                }
            } else if (!alive) {
                fenced = true;
            } else {
                s_logger.debug("VM " + vm.getInstanceName() + " is found to be alive by " + investigator.getName());
                if (host.getStatus() == Status.Up) {
                    s_logger.info(vm + " is alive and host is up. No need to restart it.");
                    return null;
                } else {
                    s_logger.debug("Rescheduling because the host is not up but the vm is alive");
                    return (System.currentTimeMillis() >> 10) + _investigateRetryInterval;
                }
            }
            if (!fenced) {
                s_logger.debug("We were unable to fence off the VM " + vm);
                _alertMgr.sendAlert(alertType, vm.getDataCenterId(), vm.getPodIdToDeployIn(), "Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc, "Insufficient capacity to restart VM, name: " + vm.getHostName() + ", id: " + vmId + " which was running on host " + hostDesc);
                return (System.currentTimeMillis() >> 10) + _restartRetryInterval;
            }
            try {
                _itMgr.advanceStop(vm.getUuid(), true);
            } catch (ResourceUnavailableException e) {
                assert false : "How do we hit this when force is true?";
                throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
            } catch (OperationTimedoutException e) {
                assert false : "How do we hit this when force is true?";
                throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
            } catch (ConcurrentOperationException e) {
                assert false : "How do we hit this when force is true?";
                throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
            }
            work.setStep(Step.Scheduled);
            _haDao.update(work.getId(), work);
        } else {
            s_logger.debug("How come that HA step is Investigating and the host is removed? Calling forced Stop on Vm anyways");
            try {
                _itMgr.advanceStop(vm.getUuid(), true);
            } catch (ResourceUnavailableException e) {
                assert false : "How do we hit this when force is true?";
                throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
            } catch (OperationTimedoutException e) {
                assert false : "How do we hit this when force is true?";
                throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
            } catch (ConcurrentOperationException e) {
                assert false : "How do we hit this when force is true?";
                throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
            }
        }
    }
    vm = _itMgr.findById(vm.getId());
    if (!_forceHA && !vm.isHaEnabled()) {
        if (s_logger.isDebugEnabled()) {
            s_logger.debug("VM is not HA enabled so we're done.");
        }
        // VM doesn't require HA
        return null;
    }
    if ((host == null || host.getRemoved() != null || host.getState() != Status.Up) && !volumeMgr.canVmRestartOnAnotherServer(vm.getId())) {
        if (s_logger.isDebugEnabled()) {
            s_logger.debug("VM can not restart on another server.");
        }
        return null;
    }
    try {
        HashMap<VirtualMachineProfile.Param, Object> params = new HashMap<VirtualMachineProfile.Param, Object>();
        if (_haTag != null) {
            params.put(VirtualMachineProfile.Param.HaTag, _haTag);
        }
        WorkType wt = work.getWorkType();
        if (wt.equals(WorkType.HA)) {
            params.put(VirtualMachineProfile.Param.HaOperation, true);
        }
        try {
            // First try starting the vm with its original planner, if it doesn't succeed send HAPlanner as its an emergency.
            _itMgr.advanceStart(vm.getUuid(), params, null);
        } catch (InsufficientCapacityException e) {
            s_logger.warn("Failed to deploy vm " + vmId + " with original planner, sending HAPlanner");
            _itMgr.advanceStart(vm.getUuid(), params, _haPlanners.get(0));
        }
        VMInstanceVO started = _instanceDao.findById(vm.getId());
        if (started != null && started.getState() == VirtualMachine.State.Running) {
            s_logger.info("VM is now restarted: " + vmId + " on " + started.getHostId());
            return null;
        }
        if (s_logger.isDebugEnabled()) {
            s_logger.debug("Rescheduling VM " + vm.toString() + " to try again in " + _restartRetryInterval);
        }
    } catch (final InsufficientCapacityException e) {
        s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
        _alertMgr.sendAlert(alertType, vm.getDataCenterId(), vm.getPodIdToDeployIn(), "Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc, "Insufficient capacity to restart VM, name: " + vm.getHostName() + ", id: " + vmId + " which was running on host " + hostDesc);
    } catch (final ResourceUnavailableException e) {
        s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
        _alertMgr.sendAlert(alertType, vm.getDataCenterId(), vm.getPodIdToDeployIn(), "Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc, "The Storage is unavailable for trying to restart VM, name: " + vm.getHostName() + ", id: " + vmId + " which was running on host " + hostDesc);
    } catch (ConcurrentOperationException e) {
        s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
        _alertMgr.sendAlert(alertType, vm.getDataCenterId(), vm.getPodIdToDeployIn(), "Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc, "The Storage is unavailable for trying to restart VM, name: " + vm.getHostName() + ", id: " + vmId + " which was running on host " + hostDesc);
    } catch (OperationTimedoutException e) {
        s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
        _alertMgr.sendAlert(alertType, vm.getDataCenterId(), vm.getPodIdToDeployIn(), "Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc, "The Storage is unavailable for trying to restart VM, name: " + vm.getHostName() + ", id: " + vmId + " which was running on host " + hostDesc);
    }
    vm = _itMgr.findById(vm.getId());
    work.setUpdateTime(vm.getUpdated());
    work.setPreviousState(vm.getState());
    return (System.currentTimeMillis() >> 10) + _restartRetryInterval;
}