Examples with OperationTimedoutException com.cloud.exception.OperationTimedoutException used on opensource projects

Example 76 with OperationTimedoutException

use of com.cloud.exception.OperationTimedoutException in project cosmic by MissionCriticalCloud.

the class ConsoleProxyManagerImpl method startProxy.

@Override
public ConsoleProxyVO startProxy(final long proxyVmId, final boolean ignoreRestartSetting) {
    try {
        ConsoleProxyVO proxy = _consoleProxyDao.findById(proxyVmId);
        if (proxy.getState() == VirtualMachine.State.Running) {
            return proxy;
        }
        final String restart = _configDao.getValue(Config.ConsoleProxyRestart.key());
        if (!ignoreRestartSetting && restart != null && restart.equalsIgnoreCase("false")) {
            return null;
        }
        if (proxy.getState() == VirtualMachine.State.Stopped) {
            _itMgr.advanceStart(proxy.getUuid(), null, null);
            proxy = _consoleProxyDao.findById(proxy.getId());
            return proxy;
        }
        // For VMs that are in Stopping, Starting, Migrating state, let client to wait by returning null
        // as sooner or later, Starting/Migrating state will be transited to Running and Stopping will be transited
        // to Stopped to allow Starting of it
        logger.warn("Console proxy is not in correct state to be started: " + proxy.getState());
        return null;
    } catch (final InsufficientCapacityException | ResourceUnavailableException | CloudRuntimeException | OperationTimedoutException e) {
        logger.warn("Exception while trying to start console proxy", e);
        return null;
    }
}

Example 77 with OperationTimedoutException

use of com.cloud.exception.OperationTimedoutException in project cosmic by MissionCriticalCloud.

the class VirtualMachineManagerImpl method migrate.

protected void migrate(final VMInstanceVO vm, final long srcHostId, final DeployDestination dest) throws ResourceUnavailableException, ConcurrentOperationException {
    s_logger.info("Migrating " + vm + " to " + dest);
    final long dstHostId = dest.getHost().getId();
    final Host fromHost = _hostDao.findById(srcHostId);
    if (fromHost == null) {
        s_logger.info("Unable to find the host to migrate from: " + srcHostId);
        throw new CloudRuntimeException("Unable to find the host to migrate from: " + srcHostId);
    }
    if (fromHost.getClusterId().longValue() != dest.getCluster().getId()) {
        final List<VolumeVO> volumes = _volsDao.findCreatedByInstance(vm.getId());
        for (final VolumeVO volume : volumes) {
            if (!_storagePoolDao.findById(volume.getPoolId()).getScope().equals(ScopeType.ZONE)) {
                s_logger.info("Source and destination host are not in same cluster and all volumes are not on zone wide primary store, unable to migrate to host: " + dest.getHost().getId());
                throw new CloudRuntimeException("Source and destination host are not in same cluster and all volumes are not on zone wide primary store, unable to migrate to host: " + dest.getHost().getId());
            }
        }
    }
    final VirtualMachineGuru vmGuru = getVmGuru(vm);
    if (vm.getState() != State.Running) {
        s_logger.debug("VM is not Running, unable to migrate the vm " + vm);
        throw new CloudRuntimeException("VM is not Running, unable to migrate the vm currently " + vm + " , current state: " + vm.getState().toString());
    }
    AlertManager.AlertType alertType = AlertManager.AlertType.ALERT_TYPE_USERVM_MIGRATE;
    if (VirtualMachine.Type.DomainRouter.equals(vm.getType())) {
        alertType = AlertManager.AlertType.ALERT_TYPE_DOMAIN_ROUTER_MIGRATE;
    } else if (VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) {
        alertType = AlertManager.AlertType.ALERT_TYPE_CONSOLE_PROXY_MIGRATE;
    }
    final VirtualMachineProfile vmSrc = new VirtualMachineProfileImpl(vm);
    for (final NicProfile nic : _networkMgr.getNicProfiles(vm)) {
        vmSrc.addNic(nic);
    }
    final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm, null, _offeringDao.findById(vm.getId(), vm.getServiceOfferingId()), null, null);
    _networkMgr.prepareNicForMigration(profile, dest);
    volumeMgr.prepareForMigration(profile, dest);
    profile.setConfigDriveLabel(VmConfigDriveLabel.value());
    final VirtualMachineTO to = toVmTO(profile);
    final PrepareForMigrationCommand pfmc = new PrepareForMigrationCommand(to);
    ItWorkVO work = new ItWorkVO(UUID.randomUUID().toString(), _nodeId, State.Migrating, vm.getType(), vm.getId());
    work.setStep(Step.Prepare);
    work.setResourceType(ItWorkVO.ResourceType.Host);
    work.setResourceId(dstHostId);
    work = _workDao.persist(work);
    Answer pfma = null;
    try {
        pfma = _agentMgr.send(dstHostId, pfmc);
        if (pfma == null || !pfma.getResult()) {
            final String details = pfma != null ? pfma.getDetails() : "null answer returned";
            final String msg = "Unable to prepare for migration due to " + details;
            pfma = null;
            throw new AgentUnavailableException(msg, dstHostId);
        }
    } catch (final OperationTimedoutException e1) {
        throw new AgentUnavailableException("Operation timed out", dstHostId);
    } finally {
        if (pfma == null) {
            _networkMgr.rollbackNicForMigration(vmSrc, profile);
            work.setStep(Step.Done);
            _workDao.update(work.getId(), work);
        }
    }
    vm.setLastHostId(srcHostId);
    try {
        if (vm == null || vm.getHostId() == null || vm.getHostId() != srcHostId || !changeState(vm, Event.MigrationRequested, dstHostId, work, Step.Migrating)) {
            _networkMgr.rollbackNicForMigration(vmSrc, profile);
            s_logger.info("Migration cancelled because state has changed: " + vm);
            throw new ConcurrentOperationException("Migration cancelled because state has changed: " + vm);
        }
    } catch (final NoTransitionException e1) {
        _networkMgr.rollbackNicForMigration(vmSrc, profile);
        s_logger.info("Migration cancelled because " + e1.getMessage());
        throw new ConcurrentOperationException("Migration cancelled because " + e1.getMessage());
    }
    boolean migrated = false;
    try {
        final boolean isWindows = _guestOsCategoryDao.findById(_guestOsDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows");
        final MigrateCommand mc = new MigrateCommand(vm.getInstanceName(), dest.getHost().getPrivateIpAddress(), isWindows, to, getExecuteInSequence(vm.getHypervisorType()));
        mc.setHostGuid(dest.getHost().getGuid());
        try {
            final Answer ma = _agentMgr.send(vm.getLastHostId(), mc);
            if (ma == null || !ma.getResult()) {
                final String details = ma != null ? ma.getDetails() : "null answer returned";
                throw new CloudRuntimeException(details);
            }
        } catch (final OperationTimedoutException e) {
            if (e.isActive()) {
                s_logger.warn("Active migration command so scheduling a restart for " + vm);
                _haMgr.scheduleRestart(vm, true);
            }
            throw new AgentUnavailableException("Operation timed out on migrating " + vm, dstHostId);
        }
        try {
            if (!changeState(vm, VirtualMachine.Event.OperationSucceeded, dstHostId, work, Step.Started)) {
                throw new ConcurrentOperationException("Unable to change the state for " + vm);
            }
        } catch (final NoTransitionException e1) {
            throw new ConcurrentOperationException("Unable to change state due to " + e1.getMessage());
        }
        try {
            if (!checkVmOnHost(vm, dstHostId)) {
                s_logger.error("Unable to complete migration for " + vm);
                try {
                    _agentMgr.send(srcHostId, new Commands(cleanup(vm)), null);
                } catch (final AgentUnavailableException e) {
                    s_logger.error("AgentUnavailableException while cleanup on source host: " + srcHostId);
                }
                cleanup(vmGuru, new VirtualMachineProfileImpl(vm), work, Event.AgentReportStopped, true);
                throw new CloudRuntimeException("Unable to complete migration for " + vm);
            }
        } catch (final OperationTimedoutException e) {
            s_logger.debug("Error while checking the vm " + vm + " on host " + dstHostId, e);
        }
        migrated = true;
    } finally {
        if (!migrated) {
            s_logger.info("Migration was unsuccessful.  Cleaning up: " + vm);
            _networkMgr.rollbackNicForMigration(vmSrc, profile);
            _alertMgr.sendAlert(alertType, fromHost.getDataCenterId(), fromHost.getPodId(), "Unable to migrate vm " + vm.getInstanceName() + " from host " + fromHost.getName() + " in zone " + dest.getZone().getName() + " and pod " + dest.getPod().getName(), "Migrate Command failed.  Please check logs.");
            try {
                _agentMgr.send(dstHostId, new Commands(cleanup(vm)), null);
            } catch (final AgentUnavailableException ae) {
                s_logger.info("Looks like the destination Host is unavailable for cleanup");
            }
            try {
                stateTransitTo(vm, Event.OperationFailed, srcHostId);
            } catch (final NoTransitionException e) {
                s_logger.warn(e.getMessage());
            }
        } else {
            _networkMgr.commitNicForMigration(vmSrc, profile);
        }
        work.setStep(Step.Done);
        _workDao.update(work.getId(), work);
    }
}

Example 78 with OperationTimedoutException

use of com.cloud.exception.OperationTimedoutException in project cosmic by MissionCriticalCloud.

the class VirtualMachineManagerImpl method advanceStop.

private void advanceStop(final VMInstanceVO vm, final boolean cleanUpEvenIfUnableToStop) throws AgentUnavailableException, OperationTimedoutException, ConcurrentOperationException {
    final State state = vm.getState();
    if (state == State.Stopped) {
        s_logger.debug("VM is already stopped: " + vm);
        return;
    }
    if (state == State.Destroyed || state == State.Expunging || state == State.Error) {
        s_logger.debug("Stopped called on " + vm + " but the state is " + state);
        return;
    }
    // grab outstanding work item if any
    final ItWorkVO work = _workDao.findByOutstandingWork(vm.getId(), vm.getState());
    if (work != null) {
        s_logger.debug("Found an outstanding work item for this vm " + vm + " with state:" + vm.getState() + ", work id:" + work.getId());
    }
    final Long hostId = vm.getHostId();
    if (hostId == null) {
        if (!cleanUpEvenIfUnableToStop) {
            s_logger.debug("HostId is null but this is not a forced stop, cannot stop vm " + vm + " with state:" + vm.getState());
            throw new CloudRuntimeException("Unable to stop " + vm);
        }
        try {
            stateTransitTo(vm, Event.AgentReportStopped, null, null);
        } catch (final NoTransitionException e) {
            s_logger.warn(e.getMessage());
        }
        // mark outstanding work item if any as done
        if (work != null) {
            s_logger.debug("Updating work item to Done, id:" + work.getId());
            work.setStep(Step.Done);
            _workDao.update(work.getId(), work);
        }
        return;
    }
    final VirtualMachineGuru vmGuru = getVmGuru(vm);
    final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm);
    try {
        if (!stateTransitTo(vm, Event.StopRequested, vm.getHostId())) {
            throw new ConcurrentOperationException("VM is being operated on.");
        }
    } catch (final NoTransitionException e1) {
        if (!cleanUpEvenIfUnableToStop) {
            throw new CloudRuntimeException("We cannot stop " + vm + " when it is in state " + vm.getState());
        }
        final boolean doCleanup = true;
        s_logger.debug("Unable to transition the state but we're moving on because it's forced stop");
        if (doCleanup) {
            if (cleanup(vmGuru, new VirtualMachineProfileImpl(vm), work, Event.StopRequested, cleanUpEvenIfUnableToStop)) {
                try {
                    if (work != null) {
                        s_logger.debug("Updating work item to Done, id:" + work.getId());
                    }
                    if (!changeState(vm, Event.AgentReportStopped, null, work, Step.Done)) {
                        throw new CloudRuntimeException("Unable to stop " + vm);
                    }
                } catch (final NoTransitionException e) {
                    s_logger.warn("Unable to cleanup " + vm);
                    throw new CloudRuntimeException("Unable to stop " + vm, e);
                }
            } else {
                s_logger.debug("Failed to cleanup VM: " + vm);
                throw new CloudRuntimeException("Failed to cleanup " + vm + " , current state " + vm.getState());
            }
        }
    }
    if (vm.getState() != State.Stopping) {
        throw new CloudRuntimeException("We cannot proceed with stop VM " + vm + " since it is not in 'Stopping' state, current state: " + vm.getState());
    }
    vmGuru.prepareStop(profile);
    final StopCommand stop = new StopCommand(vm, getExecuteInSequence(vm.getHypervisorType()), false, cleanUpEvenIfUnableToStop);
    boolean stopped = false;
    Answer answer = null;
    try {
        answer = _agentMgr.send(vm.getHostId(), stop);
        if (answer != null) {
            if (answer instanceof StopAnswer) {
                final StopAnswer stopAns = (StopAnswer) answer;
                if (vm.getType() == VirtualMachine.Type.User) {
                    final String platform = stopAns.getPlatform();
                    if (platform != null) {
                        final UserVmVO userVm = _userVmDao.findById(vm.getId());
                        _userVmDao.loadDetails(userVm);
                        userVm.setDetail("platform", platform);
                        _userVmDao.saveDetails(userVm);
                    }
                }
            }
            stopped = answer.getResult();
            if (!stopped) {
                throw new CloudRuntimeException("Unable to stop the virtual machine due to " + answer.getDetails());
            }
            vmGuru.finalizeStop(profile, answer);
            final GPUDeviceTO gpuDevice = stop.getGpuDevice();
            if (gpuDevice != null) {
                _resourceMgr.updateGPUDetails(vm.getHostId(), gpuDevice.getGroupDetails());
            }
        } else {
            throw new CloudRuntimeException("Invalid answer received in response to a StopCommand on " + vm.instanceName);
        }
    } catch (final AgentUnavailableException e) {
        s_logger.warn("Unable to stop vm, agent unavailable: " + e.toString());
    } catch (final OperationTimedoutException e) {
        s_logger.warn("Unable to stop vm, operation timed out: " + e.toString());
    } finally {
        if (!stopped) {
            if (!cleanUpEvenIfUnableToStop) {
                s_logger.warn("Unable to stop vm " + vm);
                try {
                    stateTransitTo(vm, Event.OperationFailed, vm.getHostId());
                } catch (final NoTransitionException e) {
                    s_logger.warn("Unable to transition the state " + vm);
                }
                throw new CloudRuntimeException("Unable to stop " + vm);
            } else {
                s_logger.warn("Unable to actually stop " + vm + " but continue with release because it's a force stop");
                vmGuru.finalizeStop(profile, answer);
            }
        }
    }
    s_logger.debug(vm + " is stopped on the host.  Proceeding to release resource held.");
    try {
        _networkMgr.release(profile, cleanUpEvenIfUnableToStop);
        s_logger.debug("Successfully released network resources for the vm " + vm);
    } catch (final Exception e) {
        s_logger.warn("Unable to release some network resources.", e);
    }
    volumeMgr.release(profile);
    s_logger.debug("Successfully released storage resources for the vm " + vm);
    try {
        if (work != null) {
            s_logger.debug("Updating the outstanding work item to Done, id:" + work.getId());
            work.setStep(Step.Done);
            _workDao.update(work.getId(), work);
        }
        if (!stateTransitTo(vm, Event.OperationSucceeded, null)) {
            throw new CloudRuntimeException("unable to stop " + vm);
        }
    } catch (final NoTransitionException e) {
        s_logger.warn(e.getMessage());
        throw new CloudRuntimeException("Unable to stop " + vm);
    }
}

Example 79 with OperationTimedoutException

use of com.cloud.exception.OperationTimedoutException in project cosmic by MissionCriticalCloud.

the class VirtualMachineManagerImpl method plugNic.

public boolean plugNic(final Network network, final NicTO nic, final VirtualMachineTO vm, final ReservationContext context, final DeployDestination dest) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException {
    boolean result = true;
    final VMInstanceVO router = _vmDao.findById(vm.getId());
    if (router.getState() == State.Running) {
        try {
            final PlugNicCommand plugNicCmd = new PlugNicCommand(nic, vm.getName(), vm.getType(), vm.getDetails());
            final Commands cmds = new Commands(Command.OnError.Stop);
            cmds.addCommand("plugnic", plugNicCmd);
            _agentMgr.send(dest.getHost().getId(), cmds);
            final PlugNicAnswer plugNicAnswer = cmds.getAnswer(PlugNicAnswer.class);
            if (!(plugNicAnswer != null && plugNicAnswer.getResult())) {
                s_logger.warn("Unable to plug nic for vm " + vm.getName());
                result = false;
            }
        } catch (final OperationTimedoutException e) {
            throw new AgentUnavailableException("Unable to plug nic for router " + vm.getName() + " in network " + network, dest.getHost().getId(), e);
        }
    } else {
        s_logger.warn("Unable to apply PlugNic, vm " + router + " is not in the right state " + router.getState());
        throw new ResourceUnavailableException("Unable to apply PlugNic on the backend," + " vm " + vm + " is not in the right state", DataCenter.class, router.getDataCenterId());
    }
    return result;
}

Example 80 with OperationTimedoutException

use of com.cloud.exception.OperationTimedoutException in project cosmic by MissionCriticalCloud.

the class VirtualMachineManagerImpl method orchestrateReConfigureVm.

private VMInstanceVO orchestrateReConfigureVm(final String vmUuid, final ServiceOffering oldServiceOffering, final boolean reconfiguringOnExistingHost) throws ResourceUnavailableException, ConcurrentOperationException {
    final VMInstanceVO vm = _vmDao.findByUuid(vmUuid);
    final long newServiceofferingId = vm.getServiceOfferingId();
    final ServiceOffering newServiceOffering = _offeringDao.findById(vm.getId(), newServiceofferingId);
    final HostVO hostVo = _hostDao.findById(vm.getHostId());
    final Float memoryOvercommitRatio = CapacityManager.MemOverprovisioningFactor.valueIn(hostVo.getClusterId());
    final Float cpuOvercommitRatio = CapacityManager.CpuOverprovisioningFactor.valueIn(hostVo.getClusterId());
    final long minMemory = (long) (newServiceOffering.getRamSize() / memoryOvercommitRatio);
    final ScaleVmCommand reconfigureCmd = new ScaleVmCommand(vm.getInstanceName(), newServiceOffering.getCpu(), minMemory * 1024L * 1024L, newServiceOffering.getRamSize() * 1024L * 1024L, newServiceOffering.getLimitCpuUse());
    final Long dstHostId = vm.getHostId();
    final ItWorkVO work = new ItWorkVO(UUID.randomUUID().toString(), _nodeId, State.Running, vm.getType(), vm.getId());
    work.setStep(Step.Prepare);
    work.setResourceType(ItWorkVO.ResourceType.Host);
    work.setResourceId(vm.getHostId());
    _workDao.persist(work);
    boolean success = false;
    try {
        if (reconfiguringOnExistingHost) {
            vm.setServiceOfferingId(oldServiceOffering.getId());
            // release the old capacity
            _capacityMgr.releaseVmCapacity(vm, false, false, vm.getHostId());
            vm.setServiceOfferingId(newServiceofferingId);
            // lock the new capacity
            _capacityMgr.allocateVmCapacity(vm, false);
        }
        final Answer reconfigureAnswer = _agentMgr.send(vm.getHostId(), reconfigureCmd);
        if (reconfigureAnswer == null || !reconfigureAnswer.getResult()) {
            s_logger.error("Unable to scale vm due to " + (reconfigureAnswer == null ? "" : reconfigureAnswer.getDetails()));
            throw new CloudRuntimeException("Unable to scale vm due to " + (reconfigureAnswer == null ? "" : reconfigureAnswer.getDetails()));
        }
        success = true;
    } catch (final OperationTimedoutException e) {
        throw new AgentUnavailableException("Operation timed out on reconfiguring " + vm, dstHostId);
    } catch (final AgentUnavailableException e) {
        throw e;
    } finally {
        if (!success) {
            // release the new capacity
            _capacityMgr.releaseVmCapacity(vm, false, false, vm.getHostId());
            vm.setServiceOfferingId(oldServiceOffering.getId());
            // allocate the old capacity
            _capacityMgr.allocateVmCapacity(vm, false);
        }
    }
    return vm;
}