Examples with WorkType com.cloud.ha.HaWork.WorkType used on opensource projects

Example 1 with WorkType

use of com.cloud.ha.HaWork.WorkType in project cosmic by MissionCriticalCloud.

the class HighAvailabilityManagerImpl method restart.

protected Long restart(final HaWorkVO work) {
    List<HaWorkVO> items = _haDao.listFutureHaWorkForVm(work.getInstanceId(), work.getId());
    if (items.size() > 0) {
        final StringBuilder str = new StringBuilder("Cancelling this work item because newer ones have been scheduled.  Work Ids = [");
        for (final HaWorkVO item : items) {
            str.append(item.getId()).append(", ");
        }
        str.delete(str.length() - 2, str.length()).append("]");
        s_logger.info(str.toString());
        return null;
    }
    items = _haDao.listRunningHaWorkForVm(work.getInstanceId());
    if (items.size() > 0) {
        final StringBuilder str = new StringBuilder("Waiting because there's HA work being executed on an item currently.  Work Ids =[");
        for (final HaWorkVO item : items) {
            str.append(item.getId()).append(", ");
        }
        str.delete(str.length() - 2, str.length()).append("]");
        s_logger.info(str.toString());
        return (System.currentTimeMillis() >> 10) + _investigateRetryInterval;
    }
    final long vmId = work.getInstanceId();
    VirtualMachine vm = _itMgr.findById(work.getInstanceId());
    if (vm == null) {
        s_logger.info("Unable to find vm: " + vmId);
        return null;
    }
    s_logger.info("HA on " + vm);
    if (vm.getState() != work.getPreviousState() || vm.getUpdated() != work.getUpdateTime()) {
        s_logger.info("VM " + vm + " has been changed.  Current State = " + vm.getState() + " Previous State = " + work.getPreviousState() + " last updated = " + vm.getUpdated() + " previous updated = " + work.getUpdateTime());
        return null;
    }
    AlertManager.AlertType alertType = AlertManager.AlertType.ALERT_TYPE_USERVM;
    if (VirtualMachine.Type.DomainRouter.equals(vm.getType())) {
        alertType = AlertManager.AlertType.ALERT_TYPE_DOMAIN_ROUTER;
    } else if (VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) {
        alertType = AlertManager.AlertType.ALERT_TYPE_CONSOLE_PROXY;
    } else if (VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType())) {
        alertType = AlertManager.AlertType.ALERT_TYPE_SSVM;
    }
    HostVO host = _hostDao.findById(work.getHostId());
    boolean isHostRemoved = false;
    if (host == null) {
        host = _hostDao.findByIdIncludingRemoved(work.getHostId());
        if (host != null) {
            s_logger.debug("VM " + vm.toString() + " is now no longer on host " + work.getHostId() + " as the host is removed");
            isHostRemoved = true;
        }
    }
    final Zone zone = zoneRepository.findOne(host.getDataCenterId());
    final HostPodVO podVO = _podDao.findById(host.getPodId());
    final String hostDesc = "name: " + host.getName() + "(id:" + host.getId() + "), availability zone: " + zone.getName() + ", pod: " + podVO.getName();
    Boolean alive = null;
    if (work.getStep() == Step.Investigating) {
        if (!isHostRemoved) {
            if (vm.getHostId() == null || vm.getHostId() != work.getHostId()) {
                s_logger.info("VM " + vm.toString() + " is now no longer on host " + work.getHostId());
                return null;
            }
            Investigator investigator = null;
            for (final Investigator it : investigators) {
                investigator = it;
                try {
                    alive = investigator.isVmAlive(vm, host);
                    s_logger.info(investigator.getName() + " found " + vm + " to be alive? " + alive);
                    break;
                } catch (final UnknownVM e) {
                    s_logger.info(investigator.getName() + " could not find " + vm);
                }
            }
            boolean fenced = false;
            if (alive == null) {
                s_logger.debug("Fencing off VM that we don't know the state of");
                for (final FenceBuilder fb : fenceBuilders) {
                    final Boolean result = fb.fenceOff(vm, host);
                    s_logger.info("Fencer " + fb.getName() + " returned " + result);
                    if (result != null && result) {
                        fenced = true;
                        break;
                    }
                }
            } else if (!alive) {
                fenced = true;
            } else {
                s_logger.debug("VM " + vm.getInstanceName() + " is found to be alive by " + investigator.getName());
                if (host.getStatus() == Status.Up) {
                    s_logger.info(vm + " is alive and host is up. No need to restart it.");
                    return null;
                } else {
                    s_logger.debug("Rescheduling because the host is not up but the vm is alive");
                    return (System.currentTimeMillis() >> 10) + _investigateRetryInterval;
                }
            }
            if (!fenced) {
                s_logger.debug("We were unable to fence off the VM " + vm);
                _alertMgr.sendAlert(alertType, vm.getDataCenterId(), vm.getPodIdToDeployIn(), "Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc, "Insufficient capacity to restart VM, name: " + vm.getHostName() + ", id: " + vmId + " which was running on host " + hostDesc);
                return (System.currentTimeMillis() >> 10) + _restartRetryInterval;
            }
            try {
                _itMgr.advanceStop(vm.getUuid(), true);
            } catch (final ResourceUnavailableException e) {
                assert false : "How do we hit this when force is true?";
                throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
            } catch (final OperationTimedoutException e) {
                assert false : "How do we hit this when force is true?";
                throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
            } catch (final ConcurrentOperationException e) {
                assert false : "How do we hit this when force is true?";
                throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
            }
            work.setStep(Step.Scheduled);
            _haDao.update(work.getId(), work);
        } else {
            s_logger.debug("How come that HA step is Investigating and the host is removed? Calling forced Stop on Vm anyways");
            try {
                _itMgr.advanceStop(vm.getUuid(), true);
            } catch (final ResourceUnavailableException e) {
                assert false : "How do we hit this when force is true?";
                throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
            } catch (final OperationTimedoutException e) {
                assert false : "How do we hit this when force is true?";
                throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
            } catch (final ConcurrentOperationException e) {
                assert false : "How do we hit this when force is true?";
                throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
            }
        }
    }
    vm = _itMgr.findById(vm.getId());
    if (!_forceHA && !vm.isHaEnabled()) {
        if (s_logger.isDebugEnabled()) {
            s_logger.debug("VM is not HA enabled so we're done.");
        }
        // VM doesn't require HA
        return null;
    }
    if ((host == null || host.getRemoved() != null || host.getState() != Status.Up) && !volumeMgr.canVmRestartOnAnotherServer(vm.getId())) {
        if (s_logger.isDebugEnabled()) {
            s_logger.debug("VM can not restart on another server.");
        }
        return null;
    }
    try {
        final HashMap<VirtualMachineProfile.Param, Object> params = new HashMap<>();
        if (_haTag != null) {
            params.put(VirtualMachineProfile.Param.HaTag, _haTag);
        }
        final WorkType wt = work.getWorkType();
        if (wt.equals(WorkType.HA)) {
            params.put(VirtualMachineProfile.Param.HaOperation, true);
        }
        try {
            // First try starting the vm with its original planner, if it doesn't succeed send HAPlanner as its an emergency.
            _itMgr.advanceStart(vm.getUuid(), params, null);
        } catch (final InsufficientCapacityException e) {
            s_logger.warn("Failed to deploy vm " + vmId + " with original planner, sending HAPlanner");
            _itMgr.advanceStart(vm.getUuid(), params, _haPlanners.get(0));
        }
        final VMInstanceVO started = _instanceDao.findById(vm.getId());
        if (started != null && started.getState() == VirtualMachine.State.Running) {
            s_logger.info("VM is now restarted: " + vmId + " on " + started.getHostId());
            return null;
        }
        if (s_logger.isDebugEnabled()) {
            s_logger.debug("Rescheduling VM " + vm.toString() + " to try again in " + _restartRetryInterval);
        }
    } catch (final InsufficientCapacityException e) {
        s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
        _alertMgr.sendAlert(alertType, vm.getDataCenterId(), vm.getPodIdToDeployIn(), "Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc, "Insufficient capacity to restart VM, name: " + vm.getHostName() + ", id: " + vmId + " which was running on host " + hostDesc);
    } catch (final ResourceUnavailableException e) {
        s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
        _alertMgr.sendAlert(alertType, vm.getDataCenterId(), vm.getPodIdToDeployIn(), "Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc, "The Storage is unavailable for trying to restart VM, name: " + vm.getHostName() + ", id: " + vmId + " which was running on host " + hostDesc);
    } catch (final ConcurrentOperationException e) {
        s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
        _alertMgr.sendAlert(alertType, vm.getDataCenterId(), vm.getPodIdToDeployIn(), "Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc, "The Storage is unavailable for trying to restart VM, name: " + vm.getHostName() + ", id: " + vmId + " which was running on host " + hostDesc);
    } catch (final OperationTimedoutException e) {
        s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
        _alertMgr.sendAlert(alertType, vm.getDataCenterId(), vm.getPodIdToDeployIn(), "Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc, "The Storage is unavailable for trying to restart VM, name: " + vm.getHostName() + ", id: " + vmId + " which was running on host " + hostDesc);
    }
    vm = _itMgr.findById(vm.getId());
    work.setUpdateTime(vm.getUpdated());
    work.setPreviousState(vm.getState());
    return (System.currentTimeMillis() >> 10) + _restartRetryInterval;
}

Example 2 with WorkType

use of com.cloud.ha.HaWork.WorkType in project cosmic by MissionCriticalCloud.

the class HighAvailabilityManagerImpl method cancelScheduledMigrations.

@Override
public void cancelScheduledMigrations(final HostVO host) {
    final WorkType type = host.getType() == HostVO.Type.Storage ? WorkType.Stop : WorkType.Migration;
    _haDao.deleteMigrationWorkItems(host.getId(), type, _serverId);
}

Example 3 with WorkType

use of com.cloud.ha.HaWork.WorkType in project cosmic by MissionCriticalCloud.

the class HighAvailabilityManagerImpl method processWork.

private void processWork(final HaWorkVO work) {
    final WorkType wt = work.getWorkType();
    try {
        final Long nextTime;
        if (wt == WorkType.Migration) {
            nextTime = migrate(work);
        } else if (wt == WorkType.HA) {
            nextTime = restart(work);
        } else if (wt == WorkType.Stop || wt == WorkType.CheckStop || wt == WorkType.ForceStop) {
            nextTime = stopVM(work);
        } else if (wt == WorkType.Destroy) {
            nextTime = destroyVM(work);
        } else {
            assert false : "How did we get here with " + wt.toString();
            return;
        }
        if (nextTime == null) {
            s_logger.info("Completed work " + work);
            work.setStep(Step.Done);
        } else {
            rescheduleWork(work, nextTime.longValue());
        }
    } catch (final Exception e) {
        s_logger.warn("Encountered unhandled exception during HA process, reschedule work", e);
        final long nextTime = getRescheduleTime(wt);
        rescheduleWork(work, nextTime);
        // if restart failed in the middle due to exception, VM state may has been changed
        // recapture into the HA worker so that it can really continue in it next turn
        final VMInstanceVO vm = _instanceDao.findById(work.getInstanceId());
        work.setUpdateTime(vm.getUpdated());
        work.setPreviousState(vm.getState());
    }
    if (!Step.Done.equals(work.getStep()) && work.getTimesTried() >= _maxRetries) {
        s_logger.warn("Giving up, retried max. times for work: " + work);
        work.setStep(Step.Done);
    }
    _haDao.update(work.getId(), work);
}