Examples with DpdkTO com.cloud.agent.api.to.DpdkTO used on opensource projects

Example 1 with DpdkTO

use of com.cloud.agent.api.to.DpdkTO in project cloudstack by apache.

the class LibvirtStopCommandWrapper method execute.

@Override
public Answer execute(final StopCommand command, final LibvirtComputingResource libvirtComputingResource) {
    final String vmName = command.getVmName();
    final Map<String, Boolean> vlanToPersistenceMap = command.getVlanToPersistenceMap();
    final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper();
    if (command.checkBeforeCleanup()) {
        try {
            final Connect conn = libvirtUtilitiesHelper.getConnectionByVmName(vmName);
            final Domain vm = conn.domainLookupByName(command.getVmName());
            if (vm != null && vm.getInfo().state == DomainState.VIR_DOMAIN_RUNNING) {
                return new StopAnswer(command, "vm is still running on host", false);
            }
        } catch (final Exception e) {
            s_logger.debug("Failed to get vm status in case of checkboforecleanup is true", e);
        }
    }
    File pemFile = new File(LibvirtComputingResource.SSHPRVKEYPATH);
    try {
        if (vmName.startsWith("s-") || vmName.startsWith("v-")) {
            // move the command line file to backup.
            s_logger.debug("backing up the cmdline");
            try {
                Pair<Boolean, String> ret = SshHelper.sshExecute(command.getControlIp(), 3922, "root", pemFile, null, "cp -f " + CMDLINE_PATH + " " + CMDLINE_BACKUP_PATH);
                if (!ret.first()) {
                    s_logger.debug("Failed to backup cmdline file due to " + ret.second());
                }
            } catch (Exception e) {
                s_logger.debug("Failed to backup cmdline file due to " + e.getMessage());
            }
        }
        final Connect conn = libvirtUtilitiesHelper.getConnectionByVmName(vmName);
        final List<DiskDef> disks = libvirtComputingResource.getDisks(conn, vmName);
        final List<InterfaceDef> ifaces = libvirtComputingResource.getInterfaces(conn, vmName);
        libvirtComputingResource.destroyNetworkRulesForVM(conn, vmName);
        final String result = libvirtComputingResource.stopVM(conn, vmName, command.isForceStop());
        performAgentStopHook(vmName, libvirtComputingResource);
        if (result == null) {
            if (disks != null && disks.size() > 0) {
                for (final DiskDef disk : disks) {
                    libvirtComputingResource.cleanupDisk(disk);
                }
            } else {
                // When using iSCSI-based managed storage, if the user shuts a VM down from the guest OS (as opposed to doing so from CloudStack),
                // info needs to be passed to the KVM agent to have it disconnect KVM from the applicable iSCSI volumes.
                List<Map<String, String>> volumesToDisconnect = command.getVolumesToDisconnect();
                if (volumesToDisconnect != null) {
                    for (Map<String, String> volumeToDisconnect : volumesToDisconnect) {
                        libvirtComputingResource.cleanupDisk(volumeToDisconnect);
                    }
                }
            }
            if (CollectionUtils.isEmpty(ifaces)) {
                Map<String, DpdkTO> dpdkInterfaceMapping = command.getDpdkInterfaceMapping();
                if (MapUtils.isNotEmpty(dpdkInterfaceMapping)) {
                    for (DpdkTO to : dpdkInterfaceMapping.values()) {
                        String portToRemove = to.getPort();
                        String cmd = String.format("ovs-vsctl del-port %s", portToRemove);
                        s_logger.debug("Removing DPDK port: " + portToRemove);
                        Script.runSimpleBashScript(cmd);
                    }
                }
            } else {
                for (final InterfaceDef iface : ifaces) {
                    String vlanId = libvirtComputingResource.getVlanIdFromBridgeName(iface.getBrName());
                    // each interface at this point, so inform all vif drivers
                    for (final VifDriver vifDriver : libvirtComputingResource.getAllVifDrivers()) {
                        vifDriver.unplug(iface, libvirtComputingResource.shouldDeleteBridge(vlanToPersistenceMap, vlanId));
                    }
                }
            }
        }
        return new StopAnswer(command, result, true);
    } catch (final LibvirtException e) {
        s_logger.debug("unable to stop VM:" + vmName + " due to" + e.getMessage());
        try {
            if (vmName.startsWith("s-") || vmName.startsWith("v-"))
                s_logger.debug("restoring cmdline file from backup");
            Pair<Boolean, String> ret = SshHelper.sshExecute(command.getControlIp(), 3922, "root", pemFile, null, "mv " + CMDLINE_BACKUP_PATH + " " + CMDLINE_PATH);
            if (!ret.first()) {
                s_logger.debug("unable to restore cmdline due to " + ret.second());
            }
        } catch (final Exception ex) {
            s_logger.debug("unable to restore cmdline due to:" + ex.getMessage());
        }
        return new StopAnswer(command, e.getMessage(), false);
    }
}

Example 2 with DpdkTO

use of com.cloud.agent.api.to.DpdkTO in project cloudstack by apache.

the class VirtualMachineManagerImpl method migrate.

protected void migrate(final VMInstanceVO vm, final long srcHostId, final DeployDestination dest) throws ResourceUnavailableException, ConcurrentOperationException {
    s_logger.info("Migrating " + vm + " to " + dest);
    final long dstHostId = dest.getHost().getId();
    final Host fromHost = _hostDao.findById(srcHostId);
    if (fromHost == null) {
        s_logger.info("Unable to find the host to migrate from: " + srcHostId);
        throw new CloudRuntimeException("Unable to find the host to migrate from: " + srcHostId);
    }
    if (fromHost.getClusterId() != dest.getCluster().getId() && vm.getHypervisorType() != HypervisorType.VMware) {
        final List<VolumeVO> volumes = _volsDao.findCreatedByInstance(vm.getId());
        for (final VolumeVO volume : volumes) {
            if (!_storagePoolDao.findById(volume.getPoolId()).getScope().equals(ScopeType.ZONE)) {
                s_logger.info("Source and destination host are not in same cluster and all volumes are not on zone wide primary store, unable to migrate to host: " + dest.getHost().getId());
                throw new CloudRuntimeException("Source and destination host are not in same cluster and all volumes are not on zone wide primary store, unable to migrate to host: " + dest.getHost().getId());
            }
        }
    }
    final VirtualMachineGuru vmGuru = getVmGuru(vm);
    if (vm.getState() != State.Running) {
        if (s_logger.isDebugEnabled()) {
            s_logger.debug("VM is not Running, unable to migrate the vm " + vm);
        }
        throw new CloudRuntimeException("VM is not Running, unable to migrate the vm currently " + vm + " , current state: " + vm.getState().toString());
    }
    AlertManager.AlertType alertType = AlertManager.AlertType.ALERT_TYPE_USERVM_MIGRATE;
    if (VirtualMachine.Type.DomainRouter.equals(vm.getType())) {
        alertType = AlertManager.AlertType.ALERT_TYPE_DOMAIN_ROUTER_MIGRATE;
    } else if (VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) {
        alertType = AlertManager.AlertType.ALERT_TYPE_CONSOLE_PROXY_MIGRATE;
    }
    final VirtualMachineProfile vmSrc = new VirtualMachineProfileImpl(vm);
    vmSrc.setHost(fromHost);
    for (final NicProfile nic : _networkMgr.getNicProfiles(vm)) {
        vmSrc.addNic(nic);
    }
    final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm, null, _offeringDao.findById(vm.getId(), vm.getServiceOfferingId()), null, null);
    profile.setHost(dest.getHost());
    _networkMgr.prepareNicForMigration(profile, dest);
    volumeMgr.prepareForMigration(profile, dest);
    profile.setConfigDriveLabel(VmConfigDriveLabel.value());
    final VirtualMachineTO to = toVmTO(profile);
    final PrepareForMigrationCommand pfmc = new PrepareForMigrationCommand(to);
    ItWorkVO work = new ItWorkVO(UUID.randomUUID().toString(), _nodeId, State.Migrating, vm.getType(), vm.getId());
    work.setStep(Step.Prepare);
    work.setResourceType(ItWorkVO.ResourceType.Host);
    work.setResourceId(dstHostId);
    work = _workDao.persist(work);
    Answer pfma = null;
    try {
        pfma = _agentMgr.send(dstHostId, pfmc);
        if (pfma == null || !pfma.getResult()) {
            final String details = pfma != null ? pfma.getDetails() : "null answer returned";
            final String msg = "Unable to prepare for migration due to " + details;
            pfma = null;
            throw new AgentUnavailableException(msg, dstHostId);
        }
    } catch (final OperationTimedoutException e1) {
        throw new AgentUnavailableException("Operation timed out", dstHostId);
    } finally {
        if (pfma == null) {
            _networkMgr.rollbackNicForMigration(vmSrc, profile);
            volumeMgr.release(vm.getId(), dstHostId);
            work.setStep(Step.Done);
            _workDao.update(work.getId(), work);
        }
    }
    vm.setLastHostId(srcHostId);
    try {
        if (vm.getHostId() == null || vm.getHostId() != srcHostId || !changeState(vm, Event.MigrationRequested, dstHostId, work, Step.Migrating)) {
            _networkMgr.rollbackNicForMigration(vmSrc, profile);
            if (vm != null) {
                volumeMgr.release(vm.getId(), dstHostId);
            }
            s_logger.info("Migration cancelled because state has changed: " + vm);
            throw new ConcurrentOperationException("Migration cancelled because state has changed: " + vm);
        }
    } catch (final NoTransitionException e1) {
        _networkMgr.rollbackNicForMigration(vmSrc, profile);
        volumeMgr.release(vm.getId(), dstHostId);
        s_logger.info("Migration cancelled because " + e1.getMessage());
        throw new ConcurrentOperationException("Migration cancelled because " + e1.getMessage());
    } catch (final CloudRuntimeException e2) {
        _networkMgr.rollbackNicForMigration(vmSrc, profile);
        volumeMgr.release(vm.getId(), dstHostId);
        s_logger.info("Migration cancelled because " + e2.getMessage());
        work.setStep(Step.Done);
        _workDao.update(work.getId(), work);
        try {
            stateTransitTo(vm, Event.OperationFailed, srcHostId);
        } catch (final NoTransitionException e3) {
            s_logger.warn(e3.getMessage());
        }
        throw new CloudRuntimeException("Migration cancelled because " + e2.getMessage());
    }
    boolean migrated = false;
    Map<String, DpdkTO> dpdkInterfaceMapping = null;
    try {
        final boolean isWindows = _guestOsCategoryDao.findById(_guestOsDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows");
        Map<String, Boolean> vlanToPersistenceMap = getVlanToPersistenceMapForVM(vm.getId());
        final MigrateCommand mc = new MigrateCommand(vm.getInstanceName(), dest.getHost().getPrivateIpAddress(), isWindows, to, getExecuteInSequence(vm.getHypervisorType()));
        if (MapUtils.isNotEmpty(vlanToPersistenceMap)) {
            mc.setVlanToPersistenceMap(vlanToPersistenceMap);
        }
        boolean kvmAutoConvergence = StorageManager.KvmAutoConvergence.value();
        mc.setAutoConvergence(kvmAutoConvergence);
        mc.setHostGuid(dest.getHost().getGuid());
        dpdkInterfaceMapping = ((PrepareForMigrationAnswer) pfma).getDpdkInterfaceMapping();
        if (MapUtils.isNotEmpty(dpdkInterfaceMapping)) {
            mc.setDpdkInterfaceMapping(dpdkInterfaceMapping);
        }
        try {
            final Answer ma = _agentMgr.send(vm.getLastHostId(), mc);
            if (ma == null || !ma.getResult()) {
                final String details = ma != null ? ma.getDetails() : "null answer returned";
                throw new CloudRuntimeException(details);
            }
        } catch (final OperationTimedoutException e) {
            if (e.isActive()) {
                s_logger.warn("Active migration command so scheduling a restart for " + vm, e);
                _haMgr.scheduleRestart(vm, true);
            }
            throw new AgentUnavailableException("Operation timed out on migrating " + vm, dstHostId);
        }
        try {
            if (!changeState(vm, VirtualMachine.Event.OperationSucceeded, dstHostId, work, Step.Started)) {
                throw new ConcurrentOperationException("Unable to change the state for " + vm);
            }
        } catch (final NoTransitionException e1) {
            throw new ConcurrentOperationException("Unable to change state due to " + e1.getMessage());
        }
        try {
            if (!checkVmOnHost(vm, dstHostId)) {
                s_logger.error("Unable to complete migration for " + vm);
                try {
                    _agentMgr.send(srcHostId, new Commands(cleanup(vm, dpdkInterfaceMapping)), null);
                } catch (final AgentUnavailableException e) {
                    s_logger.error("AgentUnavailableException while cleanup on source host: " + srcHostId, e);
                }
                cleanup(vmGuru, new VirtualMachineProfileImpl(vm), work, Event.AgentReportStopped, true);
                throw new CloudRuntimeException("Unable to complete migration for " + vm);
            }
        } catch (final OperationTimedoutException e) {
            s_logger.warn("Error while checking the vm " + vm + " on host " + dstHostId, e);
        }
        migrated = true;
    } finally {
        if (!migrated) {
            s_logger.info("Migration was unsuccessful.  Cleaning up: " + vm);
            _networkMgr.rollbackNicForMigration(vmSrc, profile);
            volumeMgr.release(vm.getId(), dstHostId);
            _alertMgr.sendAlert(alertType, fromHost.getDataCenterId(), fromHost.getPodId(), "Unable to migrate vm " + vm.getInstanceName() + " from host " + fromHost.getName() + " in zone " + dest.getDataCenter().getName() + " and pod " + dest.getPod().getName(), "Migrate Command failed.  Please check logs.");
            try {
                _agentMgr.send(dstHostId, new Commands(cleanup(vm, dpdkInterfaceMapping)), null);
            } catch (final AgentUnavailableException ae) {
                s_logger.warn("Looks like the destination Host is unavailable for cleanup", ae);
            }
            _networkMgr.setHypervisorHostname(profile, dest, false);
            try {
                stateTransitTo(vm, Event.OperationFailed, srcHostId);
            } catch (final NoTransitionException e) {
                s_logger.warn(e.getMessage());
            }
        } else {
            _networkMgr.commitNicForMigration(vmSrc, profile);
            volumeMgr.release(vm.getId(), srcHostId);
            _networkMgr.setHypervisorHostname(profile, dest, true);
            updateVmPod(vm, dstHostId);
        }
        work.setStep(Step.Done);
        _workDao.update(work.getId(), work);
    }
}

Example 3 with DpdkTO

use of com.cloud.agent.api.to.DpdkTO in project cloudstack by apache.

the class LibvirtMigrateCommandWrapperTest method testReplaceDPDKPorts.

@Test
public void testReplaceDPDKPorts() throws ParserConfigurationException, IOException, SAXException, TransformerException {
    final LibvirtMigrateCommandWrapper lw = new LibvirtMigrateCommandWrapper();
    Map<String, DpdkTO> dpdkPortMapping = new HashMap<>();
    DpdkTO to = new DpdkTO("/var/run/libvirt-vhost-user", "csdpdk-7", "client");
    dpdkPortMapping.put("02:00:18:91:00:10", to);
    String replaced = lw.replaceDpdkInterfaces(sourceDPDKVMToMigrate, dpdkPortMapping);
    Assert.assertTrue(replaced.contains("csdpdk-7"));
    Assert.assertFalse(replaced.contains("csdpdk-1"));
}

Example 4 with DpdkTO

use of com.cloud.agent.api.to.DpdkTO in project cloudstack by apache.

the class LibvirtPrepareForMigrationCommandWrapper method execute.

@Override
public Answer execute(final PrepareForMigrationCommand command, final LibvirtComputingResource libvirtComputingResource) {
    final VirtualMachineTO vm = command.getVirtualMachine();
    if (command.isRollback()) {
        return handleRollback(command, libvirtComputingResource);
    }
    if (s_logger.isDebugEnabled()) {
        s_logger.debug("Preparing host for migrating " + vm);
    }
    final NicTO[] nics = vm.getNics();
    Map<String, DpdkTO> dpdkInterfaceMapping = new HashMap<>();
    boolean skipDisconnect = false;
    final KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr();
    try {
        final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper();
        final Connect conn = libvirtUtilitiesHelper.getConnectionByVmName(vm.getName());
        for (final NicTO nic : nics) {
            LibvirtVMDef.InterfaceDef interfaceDef = libvirtComputingResource.getVifDriver(nic.getType(), nic.getName()).plug(nic, null, "", vm.getExtraConfig());
            if (interfaceDef != null && interfaceDef.getNetType() == GuestNetType.VHOSTUSER) {
                DpdkTO to = new DpdkTO(interfaceDef.getDpdkOvsPath(), interfaceDef.getDpdkSourcePort(), interfaceDef.getInterfaceMode());
                dpdkInterfaceMapping.put(nic.getMac(), to);
            }
        }
        /* setup disks, e.g for iso */
        final DiskTO[] volumes = vm.getDisks();
        for (final DiskTO volume : volumes) {
            if (volume.getType() == Volume.Type.ISO) {
                final DataTO data = volume.getData();
                if (data != null && data.getPath() != null && data.getPath().startsWith(ConfigDrive.CONFIGDRIVEDIR)) {
                    libvirtComputingResource.getVolumePath(conn, volume, vm.isConfigDriveOnHostCache());
                } else {
                    libvirtComputingResource.getVolumePath(conn, volume);
                }
            }
        }
        skipDisconnect = true;
        if (!storagePoolMgr.connectPhysicalDisksViaVmSpec(vm)) {
            return new PrepareForMigrationAnswer(command, "failed to connect physical disks to host");
        }
        PrepareForMigrationAnswer answer = new PrepareForMigrationAnswer(command);
        if (MapUtils.isNotEmpty(dpdkInterfaceMapping)) {
            answer.setDpdkInterfaceMapping(dpdkInterfaceMapping);
        }
        return answer;
    } catch (final LibvirtException | CloudRuntimeException | InternalErrorException | URISyntaxException e) {
        if (MapUtils.isNotEmpty(dpdkInterfaceMapping)) {
            for (DpdkTO to : dpdkInterfaceMapping.values()) {
                String cmd = String.format("ovs-vsctl del-port %s", to.getPort());
                s_logger.debug("Removing DPDK port: " + to.getPort());
                Script.runSimpleBashScript(cmd);
            }
        }
        return new PrepareForMigrationAnswer(command, e.toString());
    } finally {
        if (!skipDisconnect) {
            storagePoolMgr.disconnectPhysicalDisksViaVmSpec(vm);
        }
    }
}

Example 5 with DpdkTO

use of com.cloud.agent.api.to.DpdkTO in project cloudstack by apache.

the class LibvirtMigrateCommandWrapper method execute.

@Override
public Answer execute(final MigrateCommand command, final LibvirtComputingResource libvirtComputingResource) {
    final String vmName = command.getVmName();
    final Map<String, Boolean> vlanToPersistenceMap = command.getVlanToPersistenceMap();
    final String destinationUri = createMigrationURI(command.getDestinationIp(), libvirtComputingResource);
    final List<MigrateDiskInfo> migrateDiskInfoList = command.getMigrateDiskInfoList();
    String result = null;
    List<InterfaceDef> ifaces = null;
    List<DiskDef> disks;
    Domain dm = null;
    Connect dconn = null;
    Domain destDomain = null;
    Connect conn = null;
    String xmlDesc = null;
    List<Ternary<String, Boolean, String>> vmsnapshots = null;
    try {
        final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper();
        conn = libvirtUtilitiesHelper.getConnectionByVmName(vmName);
        ifaces = libvirtComputingResource.getInterfaces(conn, vmName);
        disks = libvirtComputingResource.getDisks(conn, vmName);
        VirtualMachineTO to = command.getVirtualMachine();
        dm = conn.domainLookupByName(vmName);
        /*
                We replace the private IP address with the address of the destination host.
                This is because the VNC listens on the private IP address of the hypervisor,
                but that address is of course different on the target host.

                MigrateCommand.getDestinationIp() returns the private IP address of the target
                hypervisor. So it's safe to use.

                The Domain.migrate method from libvirt supports passing a different XML
                description for the instance to be used on the target host.

                This is supported by libvirt-java from version 0.50.0

                CVE-2015-3252: Get XML with sensitive information suitable for migration by using
                               VIR_DOMAIN_XML_MIGRATABLE flag (value = 8)
                               https://libvirt.org/html/libvirt-libvirt-domain.html#virDomainXMLFlags

                               Use VIR_DOMAIN_XML_SECURE (value = 1) prior to v1.0.0.
             */
        // 1000000 equals v1.0.0
        final int xmlFlag = conn.getLibVirVersion() >= 1000000 ? 8 : 1;
        final String target = command.getDestinationIp();
        xmlDesc = dm.getXMLDesc(xmlFlag);
        xmlDesc = replaceIpForVNCInDescFile(xmlDesc, target);
        String oldIsoVolumePath = getOldVolumePath(disks, vmName);
        String newIsoVolumePath = getNewVolumePathIfDatastoreHasChanged(libvirtComputingResource, conn, to);
        if (newIsoVolumePath != null && !newIsoVolumePath.equals(oldIsoVolumePath)) {
            s_logger.debug("Editing mount path");
            xmlDesc = replaceDiskSourceFile(xmlDesc, newIsoVolumePath, vmName);
        }
        // delete the metadata of vm snapshots before migration
        vmsnapshots = libvirtComputingResource.cleanVMSnapshotMetadata(dm);
        // Verify Format of backing file
        for (DiskDef disk : disks) {
            if (disk.getDeviceType() == DiskDef.DeviceType.DISK && disk.getDiskFormatType() == DiskDef.DiskFmtType.QCOW2) {
                libvirtComputingResource.setBackingFileFormat(disk.getDiskPath());
            }
        }
        Map<String, MigrateCommand.MigrateDiskInfo> mapMigrateStorage = command.getMigrateStorage();
        // migrateStorage is declared as final because the replaceStorage method may mutate mapMigrateStorage, but
        // migrateStorage's value should always only be associated with the initial state of mapMigrateStorage.
        final boolean migrateStorage = MapUtils.isNotEmpty(mapMigrateStorage);
        final boolean migrateStorageManaged = command.isMigrateStorageManaged();
        if (migrateStorage) {
            xmlDesc = replaceStorage(xmlDesc, mapMigrateStorage, migrateStorageManaged);
        }
        Map<String, DpdkTO> dpdkPortsMapping = command.getDpdkInterfaceMapping();
        if (MapUtils.isNotEmpty(dpdkPortsMapping)) {
            xmlDesc = replaceDpdkInterfaces(xmlDesc, dpdkPortsMapping);
        }
        dconn = libvirtUtilitiesHelper.retrieveQemuConnection(destinationUri);
        if (to.getType() == VirtualMachine.Type.User) {
            libvirtComputingResource.detachAndAttachConfigDriveISO(conn, vmName);
        }
        // run migration in thread so we can monitor it
        s_logger.info("Live migration of instance " + vmName + " initiated to destination host: " + dconn.getURI());
        final ExecutorService executor = Executors.newFixedThreadPool(1);
        boolean migrateNonSharedInc = command.isMigrateNonSharedInc() && !migrateStorageManaged;
        final Callable<Domain> worker = new MigrateKVMAsync(libvirtComputingResource, dm, dconn, xmlDesc, migrateStorage, migrateNonSharedInc, command.isAutoConvergence(), vmName, command.getDestinationIp());
        final Future<Domain> migrateThread = executor.submit(worker);
        executor.shutdown();
        long sleeptime = 0;
        while (!executor.isTerminated()) {
            Thread.sleep(100);
            sleeptime += 100;
            if (sleeptime == 1000) {
                // wait 1s before attempting to set downtime on migration, since I don't know of a VIR_DOMAIN_MIGRATING state
                final int migrateDowntime = libvirtComputingResource.getMigrateDowntime();
                if (migrateDowntime > 0) {
                    try {
                        final int setDowntime = dm.migrateSetMaxDowntime(migrateDowntime);
                        if (setDowntime == 0) {
                            s_logger.debug("Set max downtime for migration of " + vmName + " to " + String.valueOf(migrateDowntime) + "ms");
                        }
                    } catch (final LibvirtException e) {
                        s_logger.debug("Failed to set max downtime for migration, perhaps migration completed? Error: " + e.getMessage());
                    }
                }
            }
            if (sleeptime % 1000 == 0) {
                s_logger.info("Waiting for migration of " + vmName + " to complete, waited " + sleeptime + "ms");
            }
            // abort the vm migration if the job is executed more than vm.migrate.wait
            final int migrateWait = libvirtComputingResource.getMigrateWait();
            if (migrateWait > 0 && sleeptime > migrateWait * 1000) {
                DomainState state = null;
                try {
                    state = dm.getInfo().state;
                } catch (final LibvirtException e) {
                    s_logger.info("Couldn't get VM domain state after " + sleeptime + "ms: " + e.getMessage());
                }
                if (state != null && state == DomainState.VIR_DOMAIN_RUNNING) {
                    try {
                        DomainJobInfo job = dm.getJobInfo();
                        s_logger.info("Aborting " + vmName + " domain job: " + job);
                        dm.abortJob();
                        result = String.format("Migration of VM %s was cancelled by cloudstack due to time out after %d seconds", vmName, migrateWait);
                        s_logger.debug(result);
                        break;
                    } catch (final LibvirtException e) {
                        s_logger.info("Failed to abort the vm migration job of vm " + vmName + " : " + e.getMessage());
                    }
                }
            }
            // pause vm if we meet the vm.migrate.pauseafter threshold and not already paused
            final int migratePauseAfter = libvirtComputingResource.getMigratePauseAfter();
            if (migratePauseAfter > 0 && sleeptime > migratePauseAfter) {
                DomainState state = null;
                try {
                    state = dm.getInfo().state;
                } catch (final LibvirtException e) {
                    s_logger.info("Couldn't get VM domain state after " + sleeptime + "ms: " + e.getMessage());
                }
                if (state != null && state == DomainState.VIR_DOMAIN_RUNNING) {
                    try {
                        s_logger.info("Pausing VM " + vmName + " due to property vm.migrate.pauseafter setting to " + migratePauseAfter + "ms to complete migration");
                        dm.suspend();
                    } catch (final LibvirtException e) {
                        // pause could be racy if it attempts to pause right when vm is finished, simply warn
                        s_logger.info("Failed to pause vm " + vmName + " : " + e.getMessage());
                    }
                }
            }
        }
        s_logger.info("Migration thread for " + vmName + " is done");
        destDomain = migrateThread.get(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.VM_MIGRATE_DOMAIN_RETRIEVE_TIMEOUT), TimeUnit.SECONDS);
        if (destDomain != null) {
            deleteOrDisconnectDisksOnSourcePool(libvirtComputingResource, migrateDiskInfoList, disks);
        }
    } catch (final LibvirtException e) {
        s_logger.debug("Can't migrate domain: " + e.getMessage());
        result = e.getMessage();
        if (result.startsWith("unable to connect to server") && result.endsWith("refused")) {
            result = String.format("Migration was refused connection to destination: %s. Please check libvirt configuration compatibility and firewall rules on the source and destination hosts.", destinationUri);
        }
    } catch (final InterruptedException | ExecutionException | TimeoutException | IOException | ParserConfigurationException | SAXException | TransformerException | URISyntaxException e) {
        s_logger.debug(String.format("%s : %s", e.getClass().getSimpleName(), e.getMessage()));
        if (result == null) {
            result = "Exception during migrate: " + e.getMessage();
        }
    } finally {
        try {
            if (dm != null && result != null) {
                // restore vm snapshots in case of failed migration
                if (vmsnapshots != null) {
                    libvirtComputingResource.restoreVMSnapshotMetadata(dm, vmName, vmsnapshots);
                }
            }
            if (dm != null) {
                if (dm.isPersistent() == 1) {
                    dm.undefine();
                }
                dm.free();
            }
            if (dconn != null) {
                dconn.close();
            }
            if (destDomain != null) {
                destDomain.free();
            }
        } catch (final LibvirtException e) {
            s_logger.trace("Ignoring libvirt error.", e);
        }
    }
    if (result != null) {
    } else {
        libvirtComputingResource.destroyNetworkRulesForVM(conn, vmName);
        for (final InterfaceDef iface : ifaces) {
            String vlanId = libvirtComputingResource.getVlanIdFromBridgeName(iface.getBrName());
            // We don't know which "traffic type" is associated with
            // each interface at this point, so inform all vif drivers
            final List<VifDriver> allVifDrivers = libvirtComputingResource.getAllVifDrivers();
            for (final VifDriver vifDriver : allVifDrivers) {
                vifDriver.unplug(iface, libvirtComputingResource.shouldDeleteBridge(vlanToPersistenceMap, vlanId));
            }
        }
    }
    return new MigrateAnswer(command, result == null, result, null);
}