Examples with DataStore com.cloud.engine.subsystem.api.storage.DataStore used on opensource projects

Example 86 with DataStore

use of com.cloud.engine.subsystem.api.storage.DataStore in project cosmic by MissionCriticalCloud.

the class StorageManagerImpl method cleanupStorage.

@Override
public void cleanupStorage(final boolean recurring) {
    final GlobalLock scanLock = GlobalLock.getInternLock("storagemgr.cleanup");
    try {
        if (scanLock.lock(3)) {
            try {
                // Cleanup primary storage pools
                if (_templateCleanupEnabled) {
                    final List<StoragePoolVO> storagePools = _storagePoolDao.listAll();
                    for (final StoragePoolVO pool : storagePools) {
                        try {
                            final List<VMTemplateStoragePoolVO> unusedTemplatesInPool = _tmpltMgr.getUnusedTemplatesInPool(pool);
                            s_logger.debug("Storage pool garbage collector found " + unusedTemplatesInPool.size() + " templates to clean up in storage pool: " + pool.getName());
                            for (final VMTemplateStoragePoolVO templatePoolVO : unusedTemplatesInPool) {
                                if (templatePoolVO.getDownloadState() != VMTemplateStorageResourceAssoc.Status.DOWNLOADED) {
                                    s_logger.debug("Storage pool garbage collector is skipping template with ID: " + templatePoolVO.getTemplateId() + " on pool " + templatePoolVO.getPoolId() + " because it is not completely downloaded.");
                                    continue;
                                }
                                if (!templatePoolVO.getMarkedForGC()) {
                                    templatePoolVO.setMarkedForGC(true);
                                    _vmTemplatePoolDao.update(templatePoolVO.getId(), templatePoolVO);
                                    s_logger.debug("Storage pool garbage collector has marked template with ID: " + templatePoolVO.getTemplateId() + " on pool " + templatePoolVO.getPoolId() + " for garbage collection.");
                                    continue;
                                }
                                _tmpltMgr.evictTemplateFromStoragePool(templatePoolVO);
                            }
                        } catch (final Exception e) {
                            s_logger.warn("Problem cleaning up primary storage pool " + pool, e);
                        }
                    }
                }
                cleanupSecondaryStorage(recurring);
                final List<VolumeVO> vols = _volsDao.listNonRootVolumesToBeDestroyed(new Date(System.currentTimeMillis() - ((long) StorageCleanupDelay.value() << 10)));
                for (final VolumeVO vol : vols) {
                    try {
                        VolumeInfo volumeInfo = volFactory.getVolume(vol.getId());
                        if (volumeInfo != null) {
                            volService.expungeVolumeAsync(volumeInfo);
                        } else {
                            s_logger.debug("Volume " + vol.getUuid() + " is already destroyed");
                        }
                    } catch (final Exception e) {
                        s_logger.warn("Unable to destroy volume " + vol.getUuid(), e);
                    }
                }
                // remove snapshots in Error state
                final List<SnapshotVO> snapshots = _snapshotDao.listAllByStatus(Snapshot.State.Error);
                for (final SnapshotVO snapshotVO : snapshots) {
                    try {
                        final List<SnapshotDataStoreVO> storeRefs = _snapshotStoreDao.findBySnapshotId(snapshotVO.getId());
                        for (final SnapshotDataStoreVO ref : storeRefs) {
                            _snapshotStoreDao.expunge(ref.getId());
                        }
                        _snapshotDao.expunge(snapshotVO.getId());
                    } catch (final Exception e) {
                        s_logger.warn("Unable to destroy snapshot " + snapshotVO.getUuid(), e);
                    }
                }
                // destroy uploaded volumes in abandoned/error state
                final List<VolumeDataStoreVO> volumeDataStores = _volumeDataStoreDao.listByVolumeState(Volume.State.UploadError, Volume.State.UploadAbandoned);
                for (final VolumeDataStoreVO volumeDataStore : volumeDataStores) {
                    final VolumeVO volume = _volumeDao.findById(volumeDataStore.getVolumeId());
                    if (volume == null) {
                        s_logger.warn("Uploaded volume with id " + volumeDataStore.getVolumeId() + " not found, so cannot be destroyed");
                        continue;
                    }
                    try {
                        final DataStore dataStore = _dataStoreMgr.getDataStore(volumeDataStore.getDataStoreId(), DataStoreRole.Image);
                        final EndPoint ep = _epSelector.select(dataStore, volumeDataStore.getExtractUrl());
                        if (ep == null) {
                            s_logger.warn("There is no secondary storage VM for image store " + dataStore.getName() + ", cannot destroy uploaded volume " + volume.getUuid());
                            continue;
                        }
                        final Host host = _hostDao.findById(ep.getId());
                        if (host != null && host.getManagementServerId() != null) {
                            if (_serverId == host.getManagementServerId().longValue()) {
                                if (!volService.destroyVolume(volume.getId())) {
                                    s_logger.warn("Unable to destroy uploaded volume " + volume.getUuid());
                                    continue;
                                }
                                // decrement volume resource count
                                _resourceLimitMgr.decrementResourceCount(volume.getAccountId(), ResourceType.volume, volume.isDisplayVolume());
                                // expunge volume from secondary if volume is on image store
                                final VolumeInfo volOnSecondary = volFactory.getVolume(volume.getId(), DataStoreRole.Image);
                                if (volOnSecondary != null) {
                                    s_logger.info("Expunging volume " + volume.getUuid() + " uploaded using HTTP POST from secondary data store");
                                    final AsyncCallFuture<VolumeApiResult> future = volService.expungeVolumeAsync(volOnSecondary);
                                    final VolumeApiResult result = future.get();
                                    if (!result.isSuccess()) {
                                        s_logger.warn("Failed to expunge volume " + volume.getUuid() + " from the image store " + dataStore.getName() + " due to: " + result.getResult());
                                    }
                                }
                            }
                        }
                    } catch (InterruptedException | ExecutionException e) {
                        s_logger.warn("Unable to destroy uploaded volume " + volume.getUuid() + ". Error details: " + e.getMessage());
                    }
                }
                // destroy uploaded templates in abandoned/error state
                final List<TemplateDataStoreVO> templateDataStores = _templateStoreDao.listByTemplateState(VirtualMachineTemplate.State.UploadError, VirtualMachineTemplate.State.UploadAbandoned);
                for (final TemplateDataStoreVO templateDataStore : templateDataStores) {
                    final VMTemplateVO template = _templateDao.findById(templateDataStore.getTemplateId());
                    if (template == null) {
                        s_logger.warn("Uploaded template with id " + templateDataStore.getTemplateId() + " not found, so cannot be destroyed");
                        continue;
                    }
                    try {
                        final DataStore dataStore = _dataStoreMgr.getDataStore(templateDataStore.getDataStoreId(), DataStoreRole.Image);
                        final EndPoint ep = _epSelector.select(dataStore, templateDataStore.getExtractUrl());
                        if (ep == null) {
                            s_logger.warn("There is no secondary storage VM for image store " + dataStore.getName() + ", cannot destroy uploaded template " + template.getUuid());
                            continue;
                        }
                        final Host host = _hostDao.findById(ep.getId());
                        if (host != null && host.getManagementServerId() != null) {
                            if (_serverId == host.getManagementServerId().longValue()) {
                                final AsyncCallFuture<TemplateApiResult> future = _imageSrv.deleteTemplateAsync(tmplFactory.getTemplate(template.getId(), dataStore));
                                final TemplateApiResult result = future.get();
                                if (!result.isSuccess()) {
                                    s_logger.warn("Failed to delete template " + template.getUuid() + " from the image store " + dataStore.getName() + " due to: " + result.getResult());
                                    continue;
                                }
                                // remove from template_zone_ref
                                final List<VMTemplateZoneVO> templateZones = _vmTemplateZoneDao.listByZoneTemplate(((ImageStoreEntity) dataStore).getDataCenterId(), template.getId());
                                if (templateZones != null) {
                                    for (final VMTemplateZoneVO templateZone : templateZones) {
                                        _vmTemplateZoneDao.remove(templateZone.getId());
                                    }
                                }
                                // mark all the occurrences of this template in the given store as destroyed
                                _templateStoreDao.removeByTemplateStore(template.getId(), dataStore.getId());
                                // find all eligible image stores for this template
                                final List<DataStore> imageStores = _tmpltMgr.getImageStoreByTemplate(template.getId(), null);
                                if (imageStores == null || imageStores.size() == 0) {
                                    template.setState(VirtualMachineTemplate.State.Inactive);
                                    _templateDao.update(template.getId(), template);
                                    // decrement template resource count
                                    _resourceLimitMgr.decrementResourceCount(template.getAccountId(), ResourceType.template);
                                }
                            }
                        }
                    } catch (InterruptedException | ExecutionException e) {
                        s_logger.warn("Unable to destroy uploaded template " + template.getUuid() + ". Error details: " + e.getMessage());
                    }
                }
            } finally {
                scanLock.unlock();
            }
        }
    } finally {
        scanLock.releaseRef();
    }
}

Example 87 with DataStore

use of com.cloud.engine.subsystem.api.storage.DataStore in project cosmic by MissionCriticalCloud.

the class StorageManagerImpl method preparePrimaryStorageForMaintenance.

@Override
@DB
public PrimaryDataStoreInfo preparePrimaryStorageForMaintenance(final Long primaryStorageId) throws ResourceUnavailableException, InsufficientCapacityException {
    final StoragePoolVO primaryStorage;
    primaryStorage = _storagePoolDao.findById(primaryStorageId);
    if (primaryStorage == null) {
        final String msg = "Unable to obtain lock on the storage pool record in preparePrimaryStorageForMaintenance()";
        s_logger.error(msg);
        throw new InvalidParameterValueException(msg);
    }
    if (!primaryStorage.getStatus().equals(StoragePoolStatus.Up) && !primaryStorage.getStatus().equals(StoragePoolStatus.ErrorInMaintenance)) {
        throw new InvalidParameterValueException("Primary storage with id " + primaryStorageId + " is not ready for migration, as the status is:" + primaryStorage.getStatus().toString());
    }
    final DataStoreProvider provider = _dataStoreProviderMgr.getDataStoreProvider(primaryStorage.getStorageProviderName());
    final DataStoreLifeCycle lifeCycle = provider.getDataStoreLifeCycle();
    final DataStore store = _dataStoreMgr.getDataStore(primaryStorage.getId(), DataStoreRole.Primary);
    lifeCycle.maintain(store);
    return (PrimaryDataStoreInfo) _dataStoreMgr.getDataStore(primaryStorage.getId(), DataStoreRole.Primary);
}

Example 88 with DataStore

use of com.cloud.engine.subsystem.api.storage.DataStore in project cosmic by MissionCriticalCloud.

the class StorageManagerImpl method createSecondaryStagingStore.

@Override
public ImageStore createSecondaryStagingStore(final CreateSecondaryStagingStoreCmd cmd) {
    final String providerName = cmd.getProviderName();
    DataStoreProvider storeProvider = _dataStoreProviderMgr.getDataStoreProvider(providerName);
    if (storeProvider == null) {
        storeProvider = _dataStoreProviderMgr.getDefaultCacheDataStoreProvider();
        if (storeProvider == null) {
            throw new InvalidParameterValueException("can't find cache store provider: " + providerName);
        }
    }
    final Long dcId = cmd.getZoneId();
    ScopeType scopeType = null;
    final String scope = cmd.getScope();
    if (scope != null) {
        try {
            scopeType = Enum.valueOf(ScopeType.class, scope.toUpperCase());
        } catch (final Exception e) {
            throw new InvalidParameterValueException("invalid scope for cache store " + scope);
        }
        if (scopeType != ScopeType.ZONE) {
            throw new InvalidParameterValueException("Only zone wide cache storage is supported");
        }
    }
    if (scopeType == ScopeType.ZONE && dcId == null) {
        throw new InvalidParameterValueException("zone id can't be null, if scope is zone");
    }
    // Check if the zone exists in the system
    final DataCenterVO zone = _dcDao.findById(dcId);
    if (zone == null) {
        throw new InvalidParameterValueException("Can't find zone by id " + dcId);
    }
    final Account account = CallContext.current().getCallingAccount();
    if (AllocationState.Disabled == zone.getAllocationState() && !_accountMgr.isRootAdmin(account.getId())) {
        final PermissionDeniedException ex = new PermissionDeniedException("Cannot perform this operation, Zone with specified id is currently disabled");
        ex.addProxyObject(zone.getUuid(), "dcId");
        throw ex;
    }
    final Map<String, Object> params = new HashMap<>();
    params.put("zoneId", dcId);
    params.put("url", cmd.getUrl());
    params.put("name", cmd.getUrl());
    params.put("details", cmd.getDetails());
    params.put("scope", scopeType);
    params.put("providerName", storeProvider.getName());
    params.put("role", DataStoreRole.ImageCache);
    final DataStoreLifeCycle lifeCycle = storeProvider.getDataStoreLifeCycle();
    final DataStore store;
    try {
        store = lifeCycle.initialize(params);
    } catch (final Exception e) {
        s_logger.debug("Failed to add data store: " + e.getMessage(), e);
        throw new CloudRuntimeException("Failed to add data store: " + e.getMessage(), e);
    }
    return (ImageStore) _dataStoreMgr.getDataStore(store.getId(), DataStoreRole.ImageCache);
}

Example 89 with DataStore

use of com.cloud.engine.subsystem.api.storage.DataStore in project cosmic by MissionCriticalCloud.

the class UploadMonitorImpl method createVolumeDownloadURL.

@Override
public void createVolumeDownloadURL(final Long entityId, final String path, final Type type, final Long dataCenterId, final Long uploadId, final ImageFormat format) {
    String errorString = "";
    boolean success = false;
    try {
        final List<HostVO> storageServers = _resourceMgr.listAllHostsInOneZoneByType(Host.Type.SecondaryStorage, dataCenterId);
        if (storageServers == null) {
            errorString = "No Storage Server found at the datacenter - " + dataCenterId;
            throw new CloudRuntimeException(errorString);
        }
        // Update DB for state = DOWNLOAD_URL_NOT_CREATED.
        final UploadVO uploadJob = _uploadDao.createForUpdate(uploadId);
        uploadJob.setUploadState(Status.DOWNLOAD_URL_NOT_CREATED);
        uploadJob.setLastUpdated(new Date());
        _uploadDao.update(uploadJob.getId(), uploadJob);
        // Create Symlink at ssvm
        final String uuid = UUID.randomUUID().toString() + "." + format.toString().toLowerCase();
        final DataStore secStore = storeMgr.getDataStore(ApiDBUtils.findUploadById(uploadId).getDataStoreId(), DataStoreRole.Image);
        final EndPoint ep = _epSelector.select(secStore);
        if (ep == null) {
            errorString = "There is no secondary storage VM for secondary storage host " + secStore.getName();
            throw new CloudRuntimeException(errorString);
        }
        final CreateEntityDownloadURLCommand cmd = new CreateEntityDownloadURLCommand(((ImageStoreEntity) secStore).getMountPoint(), path, uuid, null);
        final Answer ans = ep.sendMessage(cmd);
        if (ans == null || !ans.getResult()) {
            errorString = "Unable to create a link for " + type + " id:" + entityId + "," + (ans == null ? "" : ans.getDetails());
            s_logger.warn(errorString);
            throw new CloudRuntimeException(errorString);
        }
        final List<SecondaryStorageVmVO> ssVms = _secStorageVmDao.getSecStorageVmListInStates(SecondaryStorageVm.Role.templateProcessor, dataCenterId, State.Running);
        if (ssVms.size() > 0) {
            final SecondaryStorageVmVO ssVm = ssVms.get(0);
            if (ssVm.getPublicIpAddress() == null) {
                errorString = "A running secondary storage vm has a null public ip?";
                s_logger.error(errorString);
                throw new CloudRuntimeException(errorString);
            }
            // Construct actual URL locally now that the symlink exists at SSVM
            final String extractURL = generateCopyUrl(ssVm.getPublicIpAddress(), uuid);
            final UploadVO vo = _uploadDao.createForUpdate();
            vo.setLastUpdated(new Date());
            vo.setUploadUrl(extractURL);
            vo.setUploadState(Status.DOWNLOAD_URL_CREATED);
            _uploadDao.update(uploadId, vo);
            success = true;
            return;
        }
        errorString = "Couldnt find a running SSVM in the zone" + dataCenterId + ". Couldnt create the extraction URL.";
        throw new CloudRuntimeException(errorString);
    } finally {
        if (!success) {
            final UploadVO uploadJob = _uploadDao.createForUpdate(uploadId);
            uploadJob.setLastUpdated(new Date());
            uploadJob.setErrorString(errorString);
            uploadJob.setUploadState(Status.ERROR);
            _uploadDao.update(uploadId, uploadJob);
        }
    }
}

Example 90 with DataStore

use of com.cloud.engine.subsystem.api.storage.DataStore in project cosmic by MissionCriticalCloud.

the class HypervisorTemplateAdapter method create.

@Override
public VMTemplateVO create(final TemplateProfile profile) {
    // persist entry in vm_template, vm_template_details and template_zone_ref tables, not that entry at template_store_ref is not created here, and created in
    // createTemplateAsync.
    final VMTemplateVO template = persistTemplate(profile, State.Active);
    if (template == null) {
        throw new CloudRuntimeException("Unable to persist the template " + profile.getTemplate());
    }
    // find all eligible image stores for this zone scope
    final List<DataStore> imageStores = storeMgr.getImageStoresByScope(new ZoneScope(profile.getZoneId()));
    if (imageStores == null || imageStores.size() == 0) {
        throw new CloudRuntimeException("Unable to find image store to download template " + profile.getTemplate());
    }
    final Set<Long> zoneSet = new HashSet<>();
    // For private templates choose a random store. TODO - Have a better algorithm based on size, no. of objects, load etc.
    Collections.shuffle(imageStores);
    for (final DataStore imageStore : imageStores) {
        // skip data stores for a disabled zone
        final Long zoneId = imageStore.getScope().getScopeId();
        if (zoneId != null) {
            final DataCenterVO zone = _dcDao.findById(zoneId);
            if (zone == null) {
                s_logger.warn("Unable to find zone by id " + zoneId + ", so skip downloading template to its image store " + imageStore.getId());
                continue;
            }
            // Check if zone is disabled
            if (AllocationState.Disabled == zone.getAllocationState()) {
                s_logger.info("Zone " + zoneId + " is disabled, so skip downloading template to its image store " + imageStore.getId());
                continue;
            }
            // We want to download private template to one of the image store in a zone
            if (isPrivateTemplate(template) && zoneSet.contains(zoneId)) {
                continue;
            } else {
                zoneSet.add(zoneId);
            }
        }
        final TemplateInfo tmpl = imageFactory.getTemplate(template.getId(), imageStore);
        final CreateTemplateContext<TemplateApiResult> context = new CreateTemplateContext<>(null, tmpl);
        final AsyncCallbackDispatcher<HypervisorTemplateAdapter, TemplateApiResult> caller = AsyncCallbackDispatcher.create(this);
        caller.setCallback(caller.getTarget().createTemplateAsyncCallBack(null, null));
        caller.setContext(context);
        imageService.createTemplateAsync(tmpl, imageStore, caller);
    }
    _resourceLimitMgr.incrementResourceCount(profile.getAccountId(), ResourceType.template);
    return template;
}