Examples with ConcurrentOperationException com.cloud.exception.ConcurrentOperationException used on opensource projects

Example 81 with ConcurrentOperationException

use of com.cloud.exception.ConcurrentOperationException in project cloudstack by apache.

the class HighAvailabilityManagerImpl method destroyVM.

protected Long destroyVM(final HaWorkVO work) {
    final VirtualMachine vm = _itMgr.findById(work.getInstanceId());
    s_logger.info("Destroying " + vm.toString());
    try {
        if (vm.getState() != State.Destroyed) {
            s_logger.info("VM is no longer in Destroyed state " + vm.toString());
            return null;
        }
        if (vm.getHostId() != null) {
            _itMgr.destroy(vm.getUuid(), false);
            s_logger.info("Successfully destroy " + vm);
            return null;
        } else {
            if (s_logger.isDebugEnabled()) {
                s_logger.debug(vm + " has already been stopped");
            }
            return null;
        }
    } catch (final AgentUnavailableException e) {
        s_logger.debug("Agnet is not available" + e.getMessage());
    } catch (OperationTimedoutException e) {
        s_logger.debug("operation timed out: " + e.getMessage());
    } catch (ConcurrentOperationException e) {
        s_logger.debug("concurrent operation: " + e.getMessage());
    }
    return (System.currentTimeMillis() >> 10) + _stopRetryInterval;
}

Example 82 with ConcurrentOperationException

use of com.cloud.exception.ConcurrentOperationException in project cloudstack by apache.

the class HighAvailabilityManagerImpl method restart.

protected Long restart(final HaWorkVO work) {
    List<HaWorkVO> items = _haDao.listFutureHaWorkForVm(work.getInstanceId(), work.getId());
    if (items.size() > 0) {
        StringBuilder str = new StringBuilder("Cancelling this work item because newer ones have been scheduled.  Work Ids = [");
        for (HaWorkVO item : items) {
            str.append(item.getId()).append(", ");
        }
        str.delete(str.length() - 2, str.length()).append("]");
        s_logger.info(str.toString());
        return null;
    }
    items = _haDao.listRunningHaWorkForVm(work.getInstanceId());
    if (items.size() > 0) {
        StringBuilder str = new StringBuilder("Waiting because there's HA work being executed on an item currently.  Work Ids =[");
        for (HaWorkVO item : items) {
            str.append(item.getId()).append(", ");
        }
        str.delete(str.length() - 2, str.length()).append("]");
        s_logger.info(str.toString());
        return (System.currentTimeMillis() >> 10) + _investigateRetryInterval;
    }
    long vmId = work.getInstanceId();
    VirtualMachine vm = _itMgr.findById(work.getInstanceId());
    if (vm == null) {
        s_logger.info("Unable to find vm: " + vmId);
        return null;
    }
    s_logger.info("HA on " + vm);
    if (vm.getState() != work.getPreviousState() || vm.getUpdated() != work.getUpdateTime()) {
        s_logger.info("VM " + vm + " has been changed.  Current State = " + vm.getState() + " Previous State = " + work.getPreviousState() + " last updated = " + vm.getUpdated() + " previous updated = " + work.getUpdateTime());
        return null;
    }
    AlertManager.AlertType alertType = AlertManager.AlertType.ALERT_TYPE_USERVM;
    if (VirtualMachine.Type.DomainRouter.equals(vm.getType())) {
        alertType = AlertManager.AlertType.ALERT_TYPE_DOMAIN_ROUTER;
    } else if (VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) {
        alertType = AlertManager.AlertType.ALERT_TYPE_CONSOLE_PROXY;
    } else if (VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType())) {
        alertType = AlertManager.AlertType.ALERT_TYPE_SSVM;
    }
    HostVO host = _hostDao.findById(work.getHostId());
    boolean isHostRemoved = false;
    if (host == null) {
        host = _hostDao.findByIdIncludingRemoved(work.getHostId());
        if (host != null) {
            s_logger.debug("VM " + vm.toString() + " is now no longer on host " + work.getHostId() + " as the host is removed");
            isHostRemoved = true;
        }
    }
    DataCenterVO dcVO = _dcDao.findById(host.getDataCenterId());
    HostPodVO podVO = _podDao.findById(host.getPodId());
    String hostDesc = "name: " + host.getName() + "(id:" + host.getId() + "), availability zone: " + dcVO.getName() + ", pod: " + podVO.getName();
    Boolean alive = null;
    if (work.getStep() == Step.Investigating) {
        if (!isHostRemoved) {
            if (vm.getHostId() == null || vm.getHostId() != work.getHostId()) {
                s_logger.info("VM " + vm.toString() + " is now no longer on host " + work.getHostId());
                return null;
            }
            Investigator investigator = null;
            for (Investigator it : investigators) {
                investigator = it;
                try {
                    alive = investigator.isVmAlive(vm, host);
                    s_logger.info(investigator.getName() + " found " + vm + " to be alive? " + alive);
                    break;
                } catch (UnknownVM e) {
                    s_logger.info(investigator.getName() + " could not find " + vm);
                }
            }
            boolean fenced = false;
            if (alive == null) {
                s_logger.debug("Fencing off VM that we don't know the state of");
                for (FenceBuilder fb : fenceBuilders) {
                    Boolean result = fb.fenceOff(vm, host);
                    s_logger.info("Fencer " + fb.getName() + " returned " + result);
                    if (result != null && result) {
                        fenced = true;
                        break;
                    }
                }
            } else if (!alive) {
                fenced = true;
            } else {
                s_logger.debug("VM " + vm.getInstanceName() + " is found to be alive by " + investigator.getName());
                if (host.getStatus() == Status.Up) {
                    s_logger.info(vm + " is alive and host is up. No need to restart it.");
                    return null;
                } else {
                    s_logger.debug("Rescheduling because the host is not up but the vm is alive");
                    return (System.currentTimeMillis() >> 10) + _investigateRetryInterval;
                }
            }
            if (!fenced) {
                s_logger.debug("We were unable to fence off the VM " + vm);
                _alertMgr.sendAlert(alertType, vm.getDataCenterId(), vm.getPodIdToDeployIn(), "Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc, "Insufficient capacity to restart VM, name: " + vm.getHostName() + ", id: " + vmId + " which was running on host " + hostDesc);
                return (System.currentTimeMillis() >> 10) + _restartRetryInterval;
            }
            try {
                _itMgr.advanceStop(vm.getUuid(), true);
            } catch (ResourceUnavailableException e) {
                assert false : "How do we hit this when force is true?";
                throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
            } catch (OperationTimedoutException e) {
                assert false : "How do we hit this when force is true?";
                throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
            } catch (ConcurrentOperationException e) {
                assert false : "How do we hit this when force is true?";
                throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
            }
            work.setStep(Step.Scheduled);
            _haDao.update(work.getId(), work);
        } else {
            s_logger.debug("How come that HA step is Investigating and the host is removed? Calling forced Stop on Vm anyways");
            try {
                _itMgr.advanceStop(vm.getUuid(), true);
            } catch (ResourceUnavailableException e) {
                assert false : "How do we hit this when force is true?";
                throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
            } catch (OperationTimedoutException e) {
                assert false : "How do we hit this when force is true?";
                throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
            } catch (ConcurrentOperationException e) {
                assert false : "How do we hit this when force is true?";
                throw new CloudRuntimeException("Caught exception even though it should be handled.", e);
            }
        }
    }
    vm = _itMgr.findById(vm.getId());
    if (!_forceHA && !vm.isHaEnabled()) {
        if (s_logger.isDebugEnabled()) {
            s_logger.debug("VM is not HA enabled so we're done.");
        }
        // VM doesn't require HA
        return null;
    }
    if ((host == null || host.getRemoved() != null || host.getState() != Status.Up) && !volumeMgr.canVmRestartOnAnotherServer(vm.getId())) {
        if (s_logger.isDebugEnabled()) {
            s_logger.debug("VM can not restart on another server.");
        }
        return null;
    }
    try {
        HashMap<VirtualMachineProfile.Param, Object> params = new HashMap<VirtualMachineProfile.Param, Object>();
        if (_haTag != null) {
            params.put(VirtualMachineProfile.Param.HaTag, _haTag);
        }
        WorkType wt = work.getWorkType();
        if (wt.equals(WorkType.HA)) {
            params.put(VirtualMachineProfile.Param.HaOperation, true);
        }
        try {
            // First try starting the vm with its original planner, if it doesn't succeed send HAPlanner as its an emergency.
            _itMgr.advanceStart(vm.getUuid(), params, null);
        } catch (InsufficientCapacityException e) {
            s_logger.warn("Failed to deploy vm " + vmId + " with original planner, sending HAPlanner");
            _itMgr.advanceStart(vm.getUuid(), params, _haPlanners.get(0));
        }
        VMInstanceVO started = _instanceDao.findById(vm.getId());
        if (started != null && started.getState() == VirtualMachine.State.Running) {
            s_logger.info("VM is now restarted: " + vmId + " on " + started.getHostId());
            return null;
        }
        if (s_logger.isDebugEnabled()) {
            s_logger.debug("Rescheduling VM " + vm.toString() + " to try again in " + _restartRetryInterval);
        }
    } catch (final InsufficientCapacityException e) {
        s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
        _alertMgr.sendAlert(alertType, vm.getDataCenterId(), vm.getPodIdToDeployIn(), "Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc, "Insufficient capacity to restart VM, name: " + vm.getHostName() + ", id: " + vmId + " which was running on host " + hostDesc);
    } catch (final ResourceUnavailableException e) {
        s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
        _alertMgr.sendAlert(alertType, vm.getDataCenterId(), vm.getPodIdToDeployIn(), "Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc, "The Storage is unavailable for trying to restart VM, name: " + vm.getHostName() + ", id: " + vmId + " which was running on host " + hostDesc);
    } catch (ConcurrentOperationException e) {
        s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
        _alertMgr.sendAlert(alertType, vm.getDataCenterId(), vm.getPodIdToDeployIn(), "Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc, "The Storage is unavailable for trying to restart VM, name: " + vm.getHostName() + ", id: " + vmId + " which was running on host " + hostDesc);
    } catch (OperationTimedoutException e) {
        s_logger.warn("Unable to restart " + vm.toString() + " due to " + e.getMessage());
        _alertMgr.sendAlert(alertType, vm.getDataCenterId(), vm.getPodIdToDeployIn(), "Unable to restart " + vm.getHostName() + " which was running on host " + hostDesc, "The Storage is unavailable for trying to restart VM, name: " + vm.getHostName() + ", id: " + vmId + " which was running on host " + hostDesc);
    }
    vm = _itMgr.findById(vm.getId());
    work.setUpdateTime(vm.getUpdated());
    work.setPreviousState(vm.getState());
    return (System.currentTimeMillis() >> 10) + _restartRetryInterval;
}

Example 83 with ConcurrentOperationException

use of com.cloud.exception.ConcurrentOperationException in project cloudstack by apache.

the class VpcManagerImpl method createVpcPrivateGateway.

@Override
@DB
@ActionEvent(eventType = EventTypes.EVENT_PRIVATE_GATEWAY_CREATE, eventDescription = "creating VPC private gateway", create = true)
public PrivateGateway createVpcPrivateGateway(final long vpcId, Long physicalNetworkId, final String broadcastUri, final String ipAddress, final String gateway, final String netmask, final long gatewayOwnerId, final Long networkOfferingId, final Boolean isSourceNat, final Long aclId) throws ResourceAllocationException, ConcurrentOperationException, InsufficientCapacityException {
    // Validate parameters
    final Vpc vpc = getActiveVpc(vpcId);
    if (vpc == null) {
        final InvalidParameterValueException ex = new InvalidParameterValueException("Unable to find Enabled VPC by id specified");
        ex.addProxyObject(String.valueOf(vpcId), "VPC");
        throw ex;
    }
    PhysicalNetwork physNet = null;
    // Validate physical network
    if (physicalNetworkId == null) {
        final List<? extends PhysicalNetwork> pNtwks = _ntwkModel.getPhysicalNtwksSupportingTrafficType(vpc.getZoneId(), TrafficType.Guest);
        if (pNtwks.isEmpty() || pNtwks.size() != 1) {
            throw new InvalidParameterValueException("Physical network can't be determined; pass physical network id");
        }
        physNet = pNtwks.get(0);
        physicalNetworkId = physNet.getId();
    }
    if (physNet == null) {
        physNet = _entityMgr.findById(PhysicalNetwork.class, physicalNetworkId);
    }
    final Long dcId = physNet.getDataCenterId();
    final Long physicalNetworkIdFinal = physicalNetworkId;
    final PhysicalNetwork physNetFinal = physNet;
    VpcGatewayVO gatewayVO = null;
    try {
        gatewayVO = Transaction.execute(new TransactionCallbackWithException<VpcGatewayVO, Exception>() {

            @Override
            public VpcGatewayVO doInTransaction(final TransactionStatus status) throws ResourceAllocationException, ConcurrentOperationException, InsufficientCapacityException {
                s_logger.debug("Creating Private gateway for VPC " + vpc);
                // 1) create private network unless it is existing and
                // lswitch'd
                Network privateNtwk = null;
                if (BroadcastDomainType.getSchemeValue(BroadcastDomainType.fromString(broadcastUri)) == BroadcastDomainType.Lswitch) {
                    final String cidr = NetUtils.ipAndNetMaskToCidr(gateway, netmask);
                    privateNtwk = _ntwkDao.getPrivateNetwork(broadcastUri, cidr, gatewayOwnerId, dcId, networkOfferingId);
                // if the dcid is different we get no network so next we
                // try to create it
                }
                if (privateNtwk == null) {
                    s_logger.info("creating new network for vpc " + vpc + " using broadcast uri: " + broadcastUri);
                    final String networkName = "vpc-" + vpc.getName() + "-privateNetwork";
                    privateNtwk = _ntwkSvc.createPrivateNetwork(networkName, networkName, physicalNetworkIdFinal, broadcastUri, ipAddress, null, gateway, netmask, gatewayOwnerId, vpcId, isSourceNat, networkOfferingId);
                } else {
                    // create the nic/ip as createPrivateNetwork
                    // doesn''t do that work for us now
                    s_logger.info("found and using existing network for vpc " + vpc + ": " + broadcastUri);
                    final DataCenterVO dc = _dcDao.lockRow(physNetFinal.getDataCenterId(), true);
                    // add entry to private_ip_address table
                    PrivateIpVO privateIp = _privateIpDao.findByIpAndSourceNetworkId(privateNtwk.getId(), ipAddress);
                    if (privateIp != null) {
                        throw new InvalidParameterValueException("Private ip address " + ipAddress + " already used for private gateway" + " in zone " + _entityMgr.findById(DataCenter.class, dcId).getName());
                    }
                    final Long mac = dc.getMacAddress();
                    final Long nextMac = mac + 1;
                    dc.setMacAddress(nextMac);
                    s_logger.info("creating private ip adress for vpc (" + ipAddress + ", " + privateNtwk.getId() + ", " + nextMac + ", " + vpcId + ", " + isSourceNat + ")");
                    privateIp = new PrivateIpVO(ipAddress, privateNtwk.getId(), nextMac, vpcId, isSourceNat);
                    _privateIpDao.persist(privateIp);
                    _dcDao.update(dc.getId(), dc);
                }
                long networkAclId = NetworkACL.DEFAULT_DENY;
                if (aclId != null) {
                    final NetworkACLVO aclVO = _networkAclDao.findById(aclId);
                    if (aclVO == null) {
                        throw new InvalidParameterValueException("Invalid network acl id passed ");
                    }
                    if (aclVO.getVpcId() != vpcId && !(aclId == NetworkACL.DEFAULT_DENY || aclId == NetworkACL.DEFAULT_ALLOW)) {
                        throw new InvalidParameterValueException("Private gateway and network acl are not in the same vpc");
                    }
                    networkAclId = aclId;
                }
                {
                    // experimental block, this is a hack
                    // set vpc id in network to null
                    // might be needed for all types of broadcast domains
                    // the ugly hack is that vpc gateway nets are created as
                    // guest network
                    // while they are not.
                    // A more permanent solution would be to define a type of
                    // 'gatewaynetwork'
                    // so that handling code is not mixed between the two
                    final NetworkVO gatewaynet = _ntwkDao.findById(privateNtwk.getId());
                    gatewaynet.setVpcId(null);
                    _ntwkDao.persist(gatewaynet);
                }
                // 2) create gateway entry
                final VpcGatewayVO gatewayVO = new VpcGatewayVO(ipAddress, VpcGateway.Type.Private, vpcId, privateNtwk.getDataCenterId(), privateNtwk.getId(), broadcastUri, gateway, netmask, vpc.getAccountId(), vpc.getDomainId(), isSourceNat, networkAclId);
                _vpcGatewayDao.persist(gatewayVO);
                s_logger.debug("Created vpc gateway entry " + gatewayVO);
                return gatewayVO;
            }
        });
    } catch (final Exception e) {
        ExceptionUtil.rethrowRuntime(e);
        ExceptionUtil.rethrow(e, InsufficientCapacityException.class);
        ExceptionUtil.rethrow(e, ResourceAllocationException.class);
        throw new IllegalStateException(e);
    }
    CallContext.current().setEventDetails("Private Gateway Id: " + gatewayVO.getId());
    return getVpcPrivateGateway(gatewayVO.getId());
}

Example 84 with ConcurrentOperationException

use of com.cloud.exception.ConcurrentOperationException in project cloudstack by apache.

the class VpcManagerImpl method deleteVpcPrivateGateway.

@Override
@ActionEvent(eventType = EventTypes.EVENT_PRIVATE_GATEWAY_DELETE, eventDescription = "deleting private gateway")
@DB
public boolean deleteVpcPrivateGateway(final long gatewayId) throws ConcurrentOperationException, ResourceUnavailableException {
    final VpcGatewayVO gatewayVO = _vpcGatewayDao.acquireInLockTable(gatewayId);
    if (gatewayVO == null || gatewayVO.getType() != VpcGateway.Type.Private) {
        throw new ConcurrentOperationException("Unable to lock gateway " + gatewayId);
    }
    try {
        Transaction.execute(new TransactionCallbackNoReturn() {

            @Override
            public void doInTransactionWithoutResult(final TransactionStatus status) {
                // don't allow to remove gateway when there are static
                // routes associated with it
                final long routeCount = _staticRouteDao.countRoutesByGateway(gatewayVO.getId());
                if (routeCount > 0) {
                    throw new CloudRuntimeException("Can't delete private gateway " + gatewayVO + " as it has " + routeCount + " static routes applied. Remove the routes first");
                }
                gatewayVO.setState(VpcGateway.State.Deleting);
                _vpcGatewayDao.update(gatewayVO.getId(), gatewayVO);
                s_logger.debug("Marked gateway " + gatewayVO + " with state " + VpcGateway.State.Deleting);
            }
        });
        // 1) delete the gateway on the backend
        final List<Provider> providersToImplement = getVpcProviders(gatewayVO.getVpcId());
        final PrivateGateway gateway = getVpcPrivateGateway(gatewayId);
        for (final VpcProvider provider : getVpcElements()) {
            if (providersToImplement.contains(provider.getProvider())) {
                if (provider.deletePrivateGateway(gateway)) {
                    s_logger.debug("Private gateway " + gateway + " was applied succesfully on the backend");
                } else {
                    s_logger.warn("Private gateway " + gateway + " failed to apply on the backend");
                    gatewayVO.setState(VpcGateway.State.Ready);
                    _vpcGatewayDao.update(gatewayVO.getId(), gatewayVO);
                    s_logger.debug("Marked gateway " + gatewayVO + " with state " + VpcGateway.State.Ready);
                    return false;
                }
            }
        }
        // 2) Delete private gateway from the DB
        return deletePrivateGatewayFromTheDB(gateway);
    } finally {
        if (gatewayVO != null) {
            _vpcGatewayDao.releaseFromLockTable(gatewayId);
        }
    }
}

Example 85 with ConcurrentOperationException

use of com.cloud.exception.ConcurrentOperationException in project cloudstack by apache.

the class InternalLoadBalancerElement method implementInternalLbVms.

protected boolean implementInternalLbVms(Network network, DeployDestination dest) throws ResourceUnavailableException {
    //1) Get all the Ips from the network having LB rules assigned
    List<String> ips = _appLbDao.listLbIpsBySourceIpNetworkIdAndScheme(network.getId(), Scheme.Internal);
    //2) Start internal lb vms for the ips having active rules
    for (String ip : ips) {
        Ip sourceIp = new Ip(ip);
        long active = _appLbDao.countActiveBySourceIp(sourceIp, network.getId());
        if (active > 0) {
            s_logger.debug("Have to implement internal lb vm for source ip " + sourceIp + " as a part of network " + network + " implement as there are " + active + " internal lb rules exist for this ip");
            List<? extends VirtualRouter> internalLbVms;
            try {
                internalLbVms = _internalLbMgr.deployInternalLbVm(network, sourceIp, dest, _accountMgr.getAccount(network.getAccountId()), null);
            } catch (InsufficientCapacityException e) {
                s_logger.warn("Failed to deploy element " + getName() + " for ip " + sourceIp + " due to:", e);
                return false;
            } catch (ConcurrentOperationException e) {
                s_logger.warn("Failed to deploy element " + getName() + " for ip " + sourceIp + " due to:", e);
                return false;
            }
            if (internalLbVms == null || internalLbVms.isEmpty()) {
                throw new ResourceUnavailableException("Can't deploy " + getName() + " to handle LB rules", DataCenter.class, network.getDataCenterId());
            }
        }
    }
    return true;
}