Examples with EndPoint com.cloud.engine.subsystem.api.storage.EndPoint used on opensource projects

Example 31 with EndPoint

use of com.cloud.engine.subsystem.api.storage.EndPoint in project cosmic by MissionCriticalCloud.

the class AncientDataMotionStrategy method copyVolumeFromSnapshot.

protected Answer copyVolumeFromSnapshot(final DataObject snapObj, final DataObject volObj) {
    final SnapshotInfo snapshot = (SnapshotInfo) snapObj;
    final StoragePool pool = (StoragePool) volObj.getDataStore();
    final String basicErrMsg = "Failed to create volume from " + snapshot.getName() + " on pool " + pool;
    final DataStore store = snapObj.getDataStore();
    final DataStoreTO storTO = store.getTO();
    DataObject srcData = snapObj;
    try {
        if (!(storTO instanceof NfsTO)) {
            // cache snapshot to zone-wide staging store for the volume to be created
            srcData = cacheSnapshotChain(snapshot, new ZoneScope(pool.getDataCenterId()));
        }
        final String value = configDao.getValue(Config.CreateVolumeFromSnapshotWait.toString());
        final int _createVolumeFromSnapshotWait = NumbersUtil.parseInt(value, Integer.parseInt(Config.CreateVolumeFromSnapshotWait.getDefaultValue()));
        EndPoint ep = null;
        if (srcData.getDataStore().getRole() == DataStoreRole.Primary) {
            ep = selector.select(volObj);
        } else {
            ep = selector.select(srcData, volObj);
        }
        final CopyCommand cmd = new CopyCommand(srcData.getTO(), volObj.getTO(), _createVolumeFromSnapshotWait, VirtualMachineManager.ExecuteInSequence.value());
        Answer answer = null;
        if (ep == null) {
            final String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
            s_logger.error(errMsg);
            answer = new Answer(cmd, false, errMsg);
        } else {
            answer = ep.sendMessage(cmd);
        }
        return answer;
    } catch (final Exception e) {
        s_logger.error(basicErrMsg, e);
        throw new CloudRuntimeException(basicErrMsg);
    } finally {
        if (!(storTO instanceof NfsTO)) {
            // still keep snapshot on cache which may be migrated from previous secondary storage
            releaseSnapshotCacheChain((SnapshotInfo) srcData);
        }
    }
}

Example 32 with EndPoint

use of com.cloud.engine.subsystem.api.storage.EndPoint in project cosmic by MissionCriticalCloud.

the class StorageManagerImpl method cleanupStorage.

@Override
public void cleanupStorage(final boolean recurring) {
    final GlobalLock scanLock = GlobalLock.getInternLock("storagemgr.cleanup");
    try {
        if (scanLock.lock(3)) {
            try {
                // Cleanup primary storage pools
                if (_templateCleanupEnabled) {
                    final List<StoragePoolVO> storagePools = _storagePoolDao.listAll();
                    for (final StoragePoolVO pool : storagePools) {
                        try {
                            final List<VMTemplateStoragePoolVO> unusedTemplatesInPool = _tmpltMgr.getUnusedTemplatesInPool(pool);
                            s_logger.debug("Storage pool garbage collector found " + unusedTemplatesInPool.size() + " templates to clean up in storage pool: " + pool.getName());
                            for (final VMTemplateStoragePoolVO templatePoolVO : unusedTemplatesInPool) {
                                if (templatePoolVO.getDownloadState() != VMTemplateStorageResourceAssoc.Status.DOWNLOADED) {
                                    s_logger.debug("Storage pool garbage collector is skipping template with ID: " + templatePoolVO.getTemplateId() + " on pool " + templatePoolVO.getPoolId() + " because it is not completely downloaded.");
                                    continue;
                                }
                                if (!templatePoolVO.getMarkedForGC()) {
                                    templatePoolVO.setMarkedForGC(true);
                                    _vmTemplatePoolDao.update(templatePoolVO.getId(), templatePoolVO);
                                    s_logger.debug("Storage pool garbage collector has marked template with ID: " + templatePoolVO.getTemplateId() + " on pool " + templatePoolVO.getPoolId() + " for garbage collection.");
                                    continue;
                                }
                                _tmpltMgr.evictTemplateFromStoragePool(templatePoolVO);
                            }
                        } catch (final Exception e) {
                            s_logger.warn("Problem cleaning up primary storage pool " + pool, e);
                        }
                    }
                }
                cleanupSecondaryStorage(recurring);
                final List<VolumeVO> vols = _volsDao.listNonRootVolumesToBeDestroyed(new Date(System.currentTimeMillis() - ((long) StorageCleanupDelay.value() << 10)));
                for (final VolumeVO vol : vols) {
                    try {
                        VolumeInfo volumeInfo = volFactory.getVolume(vol.getId());
                        if (volumeInfo != null) {
                            volService.expungeVolumeAsync(volumeInfo);
                        } else {
                            s_logger.debug("Volume " + vol.getUuid() + " is already destroyed");
                        }
                    } catch (final Exception e) {
                        s_logger.warn("Unable to destroy volume " + vol.getUuid(), e);
                    }
                }
                // remove snapshots in Error state
                final List<SnapshotVO> snapshots = _snapshotDao.listAllByStatus(Snapshot.State.Error);
                for (final SnapshotVO snapshotVO : snapshots) {
                    try {
                        final List<SnapshotDataStoreVO> storeRefs = _snapshotStoreDao.findBySnapshotId(snapshotVO.getId());
                        for (final SnapshotDataStoreVO ref : storeRefs) {
                            _snapshotStoreDao.expunge(ref.getId());
                        }
                        _snapshotDao.expunge(snapshotVO.getId());
                    } catch (final Exception e) {
                        s_logger.warn("Unable to destroy snapshot " + snapshotVO.getUuid(), e);
                    }
                }
                // destroy uploaded volumes in abandoned/error state
                final List<VolumeDataStoreVO> volumeDataStores = _volumeDataStoreDao.listByVolumeState(Volume.State.UploadError, Volume.State.UploadAbandoned);
                for (final VolumeDataStoreVO volumeDataStore : volumeDataStores) {
                    final VolumeVO volume = _volumeDao.findById(volumeDataStore.getVolumeId());
                    if (volume == null) {
                        s_logger.warn("Uploaded volume with id " + volumeDataStore.getVolumeId() + " not found, so cannot be destroyed");
                        continue;
                    }
                    try {
                        final DataStore dataStore = _dataStoreMgr.getDataStore(volumeDataStore.getDataStoreId(), DataStoreRole.Image);
                        final EndPoint ep = _epSelector.select(dataStore, volumeDataStore.getExtractUrl());
                        if (ep == null) {
                            s_logger.warn("There is no secondary storage VM for image store " + dataStore.getName() + ", cannot destroy uploaded volume " + volume.getUuid());
                            continue;
                        }
                        final Host host = _hostDao.findById(ep.getId());
                        if (host != null && host.getManagementServerId() != null) {
                            if (_serverId == host.getManagementServerId().longValue()) {
                                if (!volService.destroyVolume(volume.getId())) {
                                    s_logger.warn("Unable to destroy uploaded volume " + volume.getUuid());
                                    continue;
                                }
                                // decrement volume resource count
                                _resourceLimitMgr.decrementResourceCount(volume.getAccountId(), ResourceType.volume, volume.isDisplayVolume());
                                // expunge volume from secondary if volume is on image store
                                final VolumeInfo volOnSecondary = volFactory.getVolume(volume.getId(), DataStoreRole.Image);
                                if (volOnSecondary != null) {
                                    s_logger.info("Expunging volume " + volume.getUuid() + " uploaded using HTTP POST from secondary data store");
                                    final AsyncCallFuture<VolumeApiResult> future = volService.expungeVolumeAsync(volOnSecondary);
                                    final VolumeApiResult result = future.get();
                                    if (!result.isSuccess()) {
                                        s_logger.warn("Failed to expunge volume " + volume.getUuid() + " from the image store " + dataStore.getName() + " due to: " + result.getResult());
                                    }
                                }
                            }
                        }
                    } catch (InterruptedException | ExecutionException e) {
                        s_logger.warn("Unable to destroy uploaded volume " + volume.getUuid() + ". Error details: " + e.getMessage());
                    }
                }
                // destroy uploaded templates in abandoned/error state
                final List<TemplateDataStoreVO> templateDataStores = _templateStoreDao.listByTemplateState(VirtualMachineTemplate.State.UploadError, VirtualMachineTemplate.State.UploadAbandoned);
                for (final TemplateDataStoreVO templateDataStore : templateDataStores) {
                    final VMTemplateVO template = _templateDao.findById(templateDataStore.getTemplateId());
                    if (template == null) {
                        s_logger.warn("Uploaded template with id " + templateDataStore.getTemplateId() + " not found, so cannot be destroyed");
                        continue;
                    }
                    try {
                        final DataStore dataStore = _dataStoreMgr.getDataStore(templateDataStore.getDataStoreId(), DataStoreRole.Image);
                        final EndPoint ep = _epSelector.select(dataStore, templateDataStore.getExtractUrl());
                        if (ep == null) {
                            s_logger.warn("There is no secondary storage VM for image store " + dataStore.getName() + ", cannot destroy uploaded template " + template.getUuid());
                            continue;
                        }
                        final Host host = _hostDao.findById(ep.getId());
                        if (host != null && host.getManagementServerId() != null) {
                            if (_serverId == host.getManagementServerId().longValue()) {
                                final AsyncCallFuture<TemplateApiResult> future = _imageSrv.deleteTemplateAsync(tmplFactory.getTemplate(template.getId(), dataStore));
                                final TemplateApiResult result = future.get();
                                if (!result.isSuccess()) {
                                    s_logger.warn("Failed to delete template " + template.getUuid() + " from the image store " + dataStore.getName() + " due to: " + result.getResult());
                                    continue;
                                }
                                // remove from template_zone_ref
                                final List<VMTemplateZoneVO> templateZones = _vmTemplateZoneDao.listByZoneTemplate(((ImageStoreEntity) dataStore).getDataCenterId(), template.getId());
                                if (templateZones != null) {
                                    for (final VMTemplateZoneVO templateZone : templateZones) {
                                        _vmTemplateZoneDao.remove(templateZone.getId());
                                    }
                                }
                                // mark all the occurrences of this template in the given store as destroyed
                                _templateStoreDao.removeByTemplateStore(template.getId(), dataStore.getId());
                                // find all eligible image stores for this template
                                final List<DataStore> imageStores = _tmpltMgr.getImageStoreByTemplate(template.getId(), null);
                                if (imageStores == null || imageStores.size() == 0) {
                                    template.setState(VirtualMachineTemplate.State.Inactive);
                                    _templateDao.update(template.getId(), template);
                                    // decrement template resource count
                                    _resourceLimitMgr.decrementResourceCount(template.getAccountId(), ResourceType.template);
                                }
                            }
                        }
                    } catch (InterruptedException | ExecutionException e) {
                        s_logger.warn("Unable to destroy uploaded template " + template.getUuid() + ". Error details: " + e.getMessage());
                    }
                }
            } finally {
                scanLock.unlock();
            }
        }
    } finally {
        scanLock.releaseRef();
    }
}

Example 33 with EndPoint

use of com.cloud.engine.subsystem.api.storage.EndPoint in project cosmic by MissionCriticalCloud.

the class UploadMonitorImpl method createVolumeDownloadURL.

@Override
public void createVolumeDownloadURL(final Long entityId, final String path, final Type type, final Long dataCenterId, final Long uploadId, final ImageFormat format) {
    String errorString = "";
    boolean success = false;
    try {
        final List<HostVO> storageServers = _resourceMgr.listAllHostsInOneZoneByType(Host.Type.SecondaryStorage, dataCenterId);
        if (storageServers == null) {
            errorString = "No Storage Server found at the datacenter - " + dataCenterId;
            throw new CloudRuntimeException(errorString);
        }
        // Update DB for state = DOWNLOAD_URL_NOT_CREATED.
        final UploadVO uploadJob = _uploadDao.createForUpdate(uploadId);
        uploadJob.setUploadState(Status.DOWNLOAD_URL_NOT_CREATED);
        uploadJob.setLastUpdated(new Date());
        _uploadDao.update(uploadJob.getId(), uploadJob);
        // Create Symlink at ssvm
        final String uuid = UUID.randomUUID().toString() + "." + format.toString().toLowerCase();
        final DataStore secStore = storeMgr.getDataStore(ApiDBUtils.findUploadById(uploadId).getDataStoreId(), DataStoreRole.Image);
        final EndPoint ep = _epSelector.select(secStore);
        if (ep == null) {
            errorString = "There is no secondary storage VM for secondary storage host " + secStore.getName();
            throw new CloudRuntimeException(errorString);
        }
        final CreateEntityDownloadURLCommand cmd = new CreateEntityDownloadURLCommand(((ImageStoreEntity) secStore).getMountPoint(), path, uuid, null);
        final Answer ans = ep.sendMessage(cmd);
        if (ans == null || !ans.getResult()) {
            errorString = "Unable to create a link for " + type + " id:" + entityId + "," + (ans == null ? "" : ans.getDetails());
            s_logger.warn(errorString);
            throw new CloudRuntimeException(errorString);
        }
        final List<SecondaryStorageVmVO> ssVms = _secStorageVmDao.getSecStorageVmListInStates(SecondaryStorageVm.Role.templateProcessor, dataCenterId, State.Running);
        if (ssVms.size() > 0) {
            final SecondaryStorageVmVO ssVm = ssVms.get(0);
            if (ssVm.getPublicIpAddress() == null) {
                errorString = "A running secondary storage vm has a null public ip?";
                s_logger.error(errorString);
                throw new CloudRuntimeException(errorString);
            }
            // Construct actual URL locally now that the symlink exists at SSVM
            final String extractURL = generateCopyUrl(ssVm.getPublicIpAddress(), uuid);
            final UploadVO vo = _uploadDao.createForUpdate();
            vo.setLastUpdated(new Date());
            vo.setUploadUrl(extractURL);
            vo.setUploadState(Status.DOWNLOAD_URL_CREATED);
            _uploadDao.update(uploadId, vo);
            success = true;
            return;
        }
        errorString = "Couldnt find a running SSVM in the zone" + dataCenterId + ". Couldnt create the extraction URL.";
        throw new CloudRuntimeException(errorString);
    } finally {
        if (!success) {
            final UploadVO uploadJob = _uploadDao.createForUpdate(uploadId);
            uploadJob.setLastUpdated(new Date());
            uploadJob.setErrorString(errorString);
            uploadJob.setUploadState(Status.ERROR);
            _uploadDao.update(uploadId, uploadJob);
        }
    }
}

Example 34 with EndPoint

use of com.cloud.engine.subsystem.api.storage.EndPoint in project cosmic by MissionCriticalCloud.

the class UploadMonitorImpl method extractVolume.

@Override
public void extractVolume(final UploadVO uploadVolumeObj, final DataStore secStore, final VolumeVO volume, final String url, final Long dataCenterId, final String installPath, final long eventId, final long asyncJobId, final AsyncJobManager asyncMgr) {
    uploadVolumeObj.setUploadState(Upload.Status.NOT_UPLOADED);
    _uploadDao.update(uploadVolumeObj.getId(), uploadVolumeObj);
    start();
    final UploadCommand ucmd = new UploadCommand(url, volume.getId(), volume.getSize(), installPath, Type.VOLUME);
    final UploadListener ul = new UploadListener(secStore, _timer, _uploadDao, uploadVolumeObj, this, ucmd, volume.getAccountId(), volume.getName(), Type.VOLUME, eventId, asyncJobId, asyncMgr);
    _listenerMap.put(uploadVolumeObj, ul);
    try {
        final EndPoint ep = _epSelector.select(secStore);
        if (ep == null) {
            final String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
            s_logger.error(errMsg);
            return;
        }
        ep.sendMessageAsync(ucmd, new UploadListener.Callback(ep.getId(), ul));
    } catch (final Exception e) {
        s_logger.warn("Unable to start upload of volume " + volume.getName() + " from " + secStore.getName() + " to " + url, e);
        ul.setDisconnected();
        ul.scheduleStatusCheck(RequestType.GET_OR_RESTART);
    }
}

Example 35 with EndPoint

use of com.cloud.engine.subsystem.api.storage.EndPoint in project cosmic by MissionCriticalCloud.

the class DownloadMonitorImpl method initiateTemplateDownload.

private void initiateTemplateDownload(final DataObject template, final AsyncCompletionCallback<DownloadAnswer> callback) {
    boolean downloadJobExists = false;
    TemplateDataStoreVO vmTemplateStore = null;
    final DataStore store = template.getDataStore();
    vmTemplateStore = _vmTemplateStoreDao.findByStoreTemplate(store.getId(), template.getId());
    if (vmTemplateStore == null) {
        vmTemplateStore = new TemplateDataStoreVO(store.getId(), template.getId(), new Date(), 0, Status.NOT_DOWNLOADED, null, null, "jobid0000", null, template.getUri());
        vmTemplateStore.setDataStoreRole(store.getRole());
        vmTemplateStore = _vmTemplateStoreDao.persist(vmTemplateStore);
    } else if ((vmTemplateStore.getJobId() != null) && (vmTemplateStore.getJobId().length() > 2)) {
        downloadJobExists = true;
    }
    final Long maxTemplateSizeInBytes = getMaxTemplateSizeInBytes();
    if (vmTemplateStore != null) {
        start();
        final VirtualMachineTemplate tmpl = _templateDao.findById(template.getId());
        DownloadCommand dcmd = new DownloadCommand((TemplateObjectTO) (template.getTO()), maxTemplateSizeInBytes);
        dcmd.setProxy(getHttpProxy());
        if (downloadJobExists) {
            dcmd = new DownloadProgressCommand(dcmd, vmTemplateStore.getJobId(), RequestType.GET_OR_RESTART);
        }
        if (vmTemplateStore.isCopy()) {
            dcmd.setCreds(TemplateConstants.DEFAULT_HTTP_AUTH_USER, _copyAuthPasswd);
        }
        final EndPoint ep = _epSelector.select(template);
        if (ep == null) {
            final String errMsg = "There is no secondary storage VM for downloading template to image store " + store.getName();
            s_logger.warn(errMsg);
            throw new CloudRuntimeException(errMsg);
        }
        final DownloadListener dl = new DownloadListener(ep, store, template, _timer, this, dcmd, callback);
        // initialize those auto-wired field in download listener.
        ComponentContext.inject(dl);
        if (downloadJobExists) {
            // due to handling existing download job issues, we still keep
            // downloadState in template_store_ref to avoid big change in
            // DownloadListener to use
            // new ObjectInDataStore.State transition. TODO: fix this later
            // to be able to remove downloadState from template_store_ref.
            s_logger.info("found existing download job");
            dl.setCurrState(vmTemplateStore.getDownloadState());
        }
        try {
            ep.sendMessageAsync(dcmd, new UploadListener.Callback(ep.getId(), dl));
        } catch (final Exception e) {
            s_logger.warn("Unable to start /resume download of template " + template.getId() + " to " + store.getName(), e);
            dl.setDisconnected();
            dl.scheduleStatusCheck(RequestType.GET_OR_RESTART);
        }
    }
}