Examples with VolumeDataStoreVO org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO used on opensource projects

Example 1 with VolumeDataStoreVO

use of org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO in project cloudstack by apache.

the class VolumeServiceImpl method handleVolumeSync.

@Override
public void handleVolumeSync(DataStore store) {
    if (store == null) {
        s_logger.warn("Huh? image store is null");
        return;
    }
    long storeId = store.getId();
    // add lock to make template sync for a data store only be done once
    String lockString = "volumesync.storeId:" + storeId;
    GlobalLock syncLock = GlobalLock.getInternLock(lockString);
    try {
        if (syncLock.lock(3)) {
            try {
                Map<Long, TemplateProp> volumeInfos = listVolume(store);
                if (volumeInfos == null) {
                    return;
                }
                // find all the db volumes including those with NULL url column to avoid accidentally deleting volumes on image store later.
                List<VolumeDataStoreVO> dbVolumes = _volumeStoreDao.listByStoreId(storeId);
                List<VolumeDataStoreVO> toBeDownloaded = new ArrayList<VolumeDataStoreVO>(dbVolumes);
                for (VolumeDataStoreVO volumeStore : dbVolumes) {
                    VolumeVO volume = volDao.findById(volumeStore.getVolumeId());
                    if (volume == null) {
                        s_logger.warn("Volume_store_ref table shows that volume " + volumeStore.getVolumeId() + " is on image store " + storeId + ", but the volume is not found in volumes table, potentially some bugs in deleteVolume, so we just treat this volume to be deleted and mark it as destroyed");
                        volumeStore.setDestroyed(true);
                        _volumeStoreDao.update(volumeStore.getId(), volumeStore);
                        continue;
                    }
                    // Exists then don't download
                    if (volumeInfos.containsKey(volume.getId())) {
                        TemplateProp volInfo = volumeInfos.remove(volume.getId());
                        toBeDownloaded.remove(volumeStore);
                        s_logger.info("Volume Sync found " + volume.getUuid() + " already in the volume image store table");
                        if (volumeStore.getDownloadState() != Status.DOWNLOADED) {
                            volumeStore.setErrorString("");
                        }
                        if (volInfo.isCorrupted()) {
                            volumeStore.setDownloadState(Status.DOWNLOAD_ERROR);
                            String msg = "Volume " + volume.getUuid() + " is corrupted on image store";
                            volumeStore.setErrorString(msg);
                            s_logger.info(msg);
                            if (volume.getState() == State.NotUploaded || volume.getState() == State.UploadInProgress) {
                                s_logger.info("Volume Sync found " + volume.getUuid() + " uploaded using SSVM on image store " + storeId + " as corrupted, marking it as failed");
                                _volumeStoreDao.update(volumeStore.getId(), volumeStore);
                                // mark volume as failed, so that storage GC will clean it up
                                VolumeObject volObj = (VolumeObject) volFactory.getVolume(volume.getId());
                                volObj.processEvent(Event.OperationFailed);
                            } else if (volumeStore.getDownloadUrl() == null) {
                                msg = "Volume (" + volume.getUuid() + ") with install path " + volInfo.getInstallPath() + " is corrupted, please check in image store: " + volumeStore.getDataStoreId();
                                s_logger.warn(msg);
                            } else {
                                s_logger.info("Removing volume_store_ref entry for corrupted volume " + volume.getName());
                                _volumeStoreDao.remove(volumeStore.getId());
                                toBeDownloaded.add(volumeStore);
                            }
                        } else {
                            // Put them in right status
                            volumeStore.setDownloadPercent(100);
                            volumeStore.setDownloadState(Status.DOWNLOADED);
                            volumeStore.setState(ObjectInDataStoreStateMachine.State.Ready);
                            volumeStore.setInstallPath(volInfo.getInstallPath());
                            volumeStore.setSize(volInfo.getSize());
                            volumeStore.setPhysicalSize(volInfo.getPhysicalSize());
                            volumeStore.setLastUpdated(new Date());
                            _volumeStoreDao.update(volumeStore.getId(), volumeStore);
                            if (volume.getSize() == 0) {
                                // Set volume size in volumes table
                                volume.setSize(volInfo.getSize());
                                volDao.update(volumeStore.getVolumeId(), volume);
                            }
                            if (volume.getState() == State.NotUploaded || volume.getState() == State.UploadInProgress) {
                                VolumeObject volObj = (VolumeObject) volFactory.getVolume(volume.getId());
                                volObj.processEvent(Event.OperationSuccessed);
                            }
                            if (volInfo.getSize() > 0) {
                                try {
                                    _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(volume.getAccountId()), com.cloud.configuration.Resource.ResourceType.secondary_storage, volInfo.getSize() - volInfo.getPhysicalSize());
                                } catch (ResourceAllocationException e) {
                                    s_logger.warn(e.getMessage());
                                    _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED, volume.getDataCenterId(), volume.getPodId(), e.getMessage(), e.getMessage());
                                } finally {
                                    _resourceLimitMgr.recalculateResourceCount(volume.getAccountId(), volume.getDomainId(), com.cloud.configuration.Resource.ResourceType.secondary_storage.getOrdinal());
                                }
                            }
                        }
                        continue;
                    } else if (volume.getState() == State.NotUploaded || volume.getState() == State.UploadInProgress) {
                        // failed uploads through SSVM
                        s_logger.info("Volume Sync did not find " + volume.getUuid() + " uploaded using SSVM on image store " + storeId + ", marking it as failed");
                        toBeDownloaded.remove(volumeStore);
                        volumeStore.setDownloadState(Status.DOWNLOAD_ERROR);
                        String msg = "Volume " + volume.getUuid() + " is corrupted on image store";
                        volumeStore.setErrorString(msg);
                        _volumeStoreDao.update(volumeStore.getId(), volumeStore);
                        // mark volume as failed, so that storage GC will clean it up
                        VolumeObject volObj = (VolumeObject) volFactory.getVolume(volume.getId());
                        volObj.processEvent(Event.OperationFailed);
                        continue;
                    }
                    // Volume is not on secondary but we should download.
                    if (volumeStore.getDownloadState() != Status.DOWNLOADED) {
                        s_logger.info("Volume Sync did not find " + volume.getName() + " ready on image store " + storeId + ", will request download to start/resume shortly");
                    }
                }
                // Download volumes which haven't been downloaded yet.
                if (toBeDownloaded.size() > 0) {
                    for (VolumeDataStoreVO volumeHost : toBeDownloaded) {
                        if (volumeHost.getDownloadUrl() == null) {
                            // If url is null, skip downloading
                            s_logger.info("Skip downloading volume " + volumeHost.getVolumeId() + " since no download url is specified.");
                            continue;
                        }
                        // means that this is a duplicate entry from migration of previous NFS to staging.
                        if (store.getScope().getScopeType() == ScopeType.REGION) {
                            if (volumeHost.getDownloadState() == VMTemplateStorageResourceAssoc.Status.DOWNLOADED && volumeHost.getInstallPath() == null) {
                                s_logger.info("Skip sync volume for migration of previous NFS to object store");
                                continue;
                            }
                        }
                        s_logger.debug("Volume " + volumeHost.getVolumeId() + " needs to be downloaded to " + store.getName());
                        // reset volume status back to Allocated
                        VolumeObject vol = (VolumeObject) volFactory.getVolume(volumeHost.getVolumeId());
                        // reset back volume status
                        vol.processEvent(Event.OperationFailed);
                        // remove leftover volume_store_ref entry since re-download will create it again
                        _volumeStoreDao.remove(volumeHost.getId());
                        // get an updated volumeVO
                        vol = (VolumeObject) volFactory.getVolume(volumeHost.getVolumeId());
                        RegisterVolumePayload payload = new RegisterVolumePayload(volumeHost.getDownloadUrl(), volumeHost.getChecksum(), vol.getFormat().toString());
                        vol.addPayload(payload);
                        createVolumeAsync(vol, store);
                    }
                }
                // Delete volumes which are not present on DB.
                for (Map.Entry<Long, TemplateProp> entry : volumeInfos.entrySet()) {
                    Long uniqueName = entry.getKey();
                    TemplateProp tInfo = entry.getValue();
                    // we cannot directly call expungeVolumeAsync here to reuse delete logic since in this case db does not have this volume at all.
                    VolumeObjectTO tmplTO = new VolumeObjectTO();
                    tmplTO.setDataStore(store.getTO());
                    tmplTO.setPath(tInfo.getInstallPath());
                    tmplTO.setId(tInfo.getId());
                    DeleteCommand dtCommand = new DeleteCommand(tmplTO);
                    EndPoint ep = _epSelector.select(store);
                    Answer answer = null;
                    if (ep == null) {
                        String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
                        s_logger.error(errMsg);
                        answer = new Answer(dtCommand, false, errMsg);
                    } else {
                        answer = ep.sendMessage(dtCommand);
                    }
                    if (answer == null || !answer.getResult()) {
                        s_logger.info("Failed to deleted volume at store: " + store.getName());
                    } else {
                        String description = "Deleted volume " + tInfo.getTemplateName() + " on secondary storage " + storeId;
                        s_logger.info(description);
                    }
                }
            } finally {
                syncLock.unlock();
            }
        } else {
            s_logger.info("Couldn't get global lock on " + lockString + ", another thread may be doing volume sync on data store " + storeId + " now.");
        }
    } finally {
        syncLock.releaseRef();
    }
}

Example 2 with VolumeDataStoreVO

use of org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO in project cloudstack by apache.

the class ObjectInDataStoreManagerImpl method create.

@Override
public DataObject create(DataObject obj, DataStore dataStore) {
    if (dataStore.getRole() == DataStoreRole.Primary) {
        if (obj.getType() == DataObjectType.TEMPLATE) {
            VMTemplateStoragePoolVO vo = new VMTemplateStoragePoolVO(dataStore.getId(), obj.getId());
            vo = templatePoolDao.persist(vo);
        } else if (obj.getType() == DataObjectType.SNAPSHOT) {
            SnapshotInfo snapshotInfo = (SnapshotInfo) obj;
            SnapshotDataStoreVO ss = new SnapshotDataStoreVO();
            ss.setSnapshotId(obj.getId());
            ss.setDataStoreId(dataStore.getId());
            ss.setRole(dataStore.getRole());
            ss.setVolumeId(snapshotInfo.getVolumeId());
            // this is the virtual size of snapshot in primary storage.
            ss.setSize(snapshotInfo.getSize());
            // this physical size will get updated with actual size once the snapshot backup is done.
            ss.setPhysicalSize(snapshotInfo.getSize());
            SnapshotDataStoreVO snapshotDataStoreVO = snapshotDataStoreDao.findParent(dataStore.getRole(), dataStore.getId(), snapshotInfo.getVolumeId());
            if (snapshotDataStoreVO != null) {
                //Double check the snapshot is removed or not
                SnapshotVO parentSnap = snapshotDao.findById(snapshotDataStoreVO.getSnapshotId());
                if (parentSnap != null) {
                    ss.setParentSnapshotId(snapshotDataStoreVO.getSnapshotId());
                } else {
                    s_logger.debug("find inconsistent db for snapshot " + snapshotDataStoreVO.getSnapshotId());
                }
            }
            ss.setState(ObjectInDataStoreStateMachine.State.Allocated);
            ss = snapshotDataStoreDao.persist(ss);
        }
    } else {
        // Image store
        switch(obj.getType()) {
            case TEMPLATE:
                TemplateDataStoreVO ts = new TemplateDataStoreVO();
                ts.setTemplateId(obj.getId());
                ts.setDataStoreId(dataStore.getId());
                ts.setDataStoreRole(dataStore.getRole());
                String installPath = TemplateConstants.DEFAULT_TMPLT_ROOT_DIR + "/" + TemplateConstants.DEFAULT_TMPLT_FIRST_LEVEL_DIR + templateDao.findById(obj.getId()).getAccountId() + "/" + obj.getId();
                if (dataStore.getTO() instanceof S3TO) {
                    TemplateInfo tmpl = (TemplateInfo) obj;
                    // for S3, we
                    installPath += "/" + tmpl.getUniqueName();
                // append
                // template name
                // in the path
                // for template
                // sync since we
                // don't have
                // template.properties
                // there
                }
                ts.setInstallPath(installPath);
                ts.setState(ObjectInDataStoreStateMachine.State.Allocated);
                ts = templateDataStoreDao.persist(ts);
                break;
            case SNAPSHOT:
                SnapshotInfo snapshot = (SnapshotInfo) obj;
                SnapshotDataStoreVO ss = new SnapshotDataStoreVO();
                ss.setSnapshotId(obj.getId());
                ss.setDataStoreId(dataStore.getId());
                ss.setRole(dataStore.getRole());
                ss.setSize(snapshot.getSize());
                ss.setVolumeId(snapshot.getVolumeId());
                SnapshotDataStoreVO snapshotDataStoreVO = snapshotDataStoreDao.findParent(dataStore.getRole(), dataStore.getId(), snapshot.getVolumeId());
                if (snapshotDataStoreVO != null) {
                    ss.setParentSnapshotId(snapshotDataStoreVO.getSnapshotId());
                }
                ss.setInstallPath(TemplateConstants.DEFAULT_SNAPSHOT_ROOT_DIR + "/" + snapshotDao.findById(obj.getId()).getAccountId() + "/" + snapshot.getVolumeId());
                ss.setState(ObjectInDataStoreStateMachine.State.Allocated);
                ss = snapshotDataStoreDao.persist(ss);
                break;
            case VOLUME:
                VolumeDataStoreVO vs = new VolumeDataStoreVO();
                vs.setVolumeId(obj.getId());
                vs.setDataStoreId(dataStore.getId());
                vs.setInstallPath(TemplateConstants.DEFAULT_VOLUME_ROOT_DIR + "/" + volumeDao.findById(obj.getId()).getAccountId() + "/" + obj.getId());
                vs.setState(ObjectInDataStoreStateMachine.State.Allocated);
                vs = volumeDataStoreDao.persist(vs);
                break;
        }
    }
    return this.get(obj, dataStore);
}

Example 3 with VolumeDataStoreVO

use of org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO in project cloudstack by apache.

the class ObjectInDataStoreManagerImpl method delete.

@Override
public boolean delete(DataObject dataObj) {
    long objId = dataObj.getId();
    DataStore dataStore = dataObj.getDataStore();
    if (dataStore.getRole() == DataStoreRole.Primary) {
        if (dataObj.getType() == DataObjectType.TEMPLATE) {
            VMTemplateStoragePoolVO destTmpltPool = templatePoolDao.findByPoolTemplate(dataStore.getId(), objId);
            if (destTmpltPool != null) {
                return templatePoolDao.remove(destTmpltPool.getId());
            } else {
                s_logger.warn("Template " + objId + " is not found on storage pool " + dataStore.getId() + ", so no need to delete");
                return true;
            }
        }
    } else {
        // Image store
        switch(dataObj.getType()) {
            case TEMPLATE:
                TemplateDataStoreVO destTmpltStore = templateDataStoreDao.findByStoreTemplate(dataStore.getId(), objId);
                if (destTmpltStore != null) {
                    return templateDataStoreDao.remove(destTmpltStore.getId());
                } else {
                    s_logger.warn("Template " + objId + " is not found on image store " + dataStore.getId() + ", so no need to delete");
                    return true;
                }
            case SNAPSHOT:
                SnapshotDataStoreVO destSnapshotStore = snapshotDataStoreDao.findByStoreSnapshot(dataStore.getRole(), dataStore.getId(), objId);
                if (destSnapshotStore != null) {
                    return snapshotDataStoreDao.remove(destSnapshotStore.getId());
                } else {
                    s_logger.warn("Snapshot " + objId + " is not found on image store " + dataStore.getId() + ", so no need to delete");
                    return true;
                }
            case VOLUME:
                VolumeDataStoreVO destVolumeStore = volumeDataStoreDao.findByStoreVolume(dataStore.getId(), objId);
                if (destVolumeStore != null) {
                    return volumeDataStoreDao.remove(destVolumeStore.getId());
                } else {
                    s_logger.warn("Volume " + objId + " is not found on image store " + dataStore.getId() + ", so no need to delete");
                    return true;
                }
        }
    }
    s_logger.warn("Unsupported data object (" + dataObj.getType() + ", " + dataObj.getDataStore() + ")");
    return false;
}

Example 4 with VolumeDataStoreVO

use of org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO in project cloudstack by apache.

the class ObjectInDataStoreManagerImpl method deleteIfNotReady.

@Override
public boolean deleteIfNotReady(DataObject dataObj) {
    long objId = dataObj.getId();
    DataStore dataStore = dataObj.getDataStore();
    if (dataStore.getRole() == DataStoreRole.Primary) {
        if (dataObj.getType() == DataObjectType.TEMPLATE) {
            VMTemplateStoragePoolVO destTmpltPool = templatePoolDao.findByPoolTemplate(dataStore.getId(), objId);
            if (destTmpltPool != null && destTmpltPool.getState() != ObjectInDataStoreStateMachine.State.Ready) {
                return templatePoolDao.remove(destTmpltPool.getId());
            } else {
                s_logger.warn("Template " + objId + " is not found on storage pool " + dataStore.getId() + ", so no need to delete");
                return true;
            }
        } else if (dataObj.getType() == DataObjectType.SNAPSHOT) {
            SnapshotDataStoreVO destSnapshotStore = snapshotDataStoreDao.findByStoreSnapshot(dataStore.getRole(), dataStore.getId(), objId);
            if (destSnapshotStore != null && destSnapshotStore.getState() != ObjectInDataStoreStateMachine.State.Ready) {
                snapshotDataStoreDao.remove(destSnapshotStore.getId());
            }
            return true;
        }
    } else {
        // Image store
        switch(dataObj.getType()) {
            case TEMPLATE:
                return true;
            case SNAPSHOT:
                SnapshotDataStoreVO destSnapshotStore = snapshotDataStoreDao.findByStoreSnapshot(dataStore.getRole(), dataStore.getId(), objId);
                if (destSnapshotStore != null && destSnapshotStore.getState() != ObjectInDataStoreStateMachine.State.Ready) {
                    return snapshotDataStoreDao.remove(destSnapshotStore.getId());
                } else {
                    s_logger.warn("Snapshot " + objId + " is not found on image store " + dataStore.getId() + ", so no need to delete");
                    return true;
                }
            case VOLUME:
                VolumeDataStoreVO destVolumeStore = volumeDataStoreDao.findByStoreVolume(dataStore.getId(), objId);
                if (destVolumeStore != null && destVolumeStore.getState() != ObjectInDataStoreStateMachine.State.Ready) {
                    return volumeDataStoreDao.remove(destVolumeStore.getId());
                } else {
                    s_logger.warn("Volume " + objId + " is not found on image store " + dataStore.getId() + ", so no need to delete");
                    return true;
                }
        }
    }
    s_logger.warn("Unsupported data object (" + dataObj.getType() + ", " + dataObj.getDataStore() + "), no need to delete from object in store ref table");
    return false;
}

Example 5 with VolumeDataStoreVO

use of org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO in project cloudstack by apache.

the class BaseImageStoreDriverImpl method createVolumeAsyncCallback.

protected Void createVolumeAsyncCallback(AsyncCallbackDispatcher<? extends BaseImageStoreDriverImpl, DownloadAnswer> callback, CreateContext<CreateCmdResult> context) {
    DownloadAnswer answer = callback.getResult();
    DataObject obj = context.data;
    DataStore store = obj.getDataStore();
    VolumeDataStoreVO volStoreVO = _volumeStoreDao.findByStoreVolume(store.getId(), obj.getId());
    if (volStoreVO != null) {
        if (volStoreVO.getDownloadState() == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) {
            if (s_logger.isDebugEnabled()) {
                s_logger.debug("Volume is already in DOWNLOADED state, ignore further incoming DownloadAnswer");
            }
            return null;
        }
        VolumeDataStoreVO updateBuilder = _volumeStoreDao.createForUpdate();
        updateBuilder.setDownloadPercent(answer.getDownloadPct());
        updateBuilder.setDownloadState(answer.getDownloadStatus());
        updateBuilder.setLastUpdated(new Date());
        updateBuilder.setErrorString(answer.getErrorString());
        updateBuilder.setJobId(answer.getJobId());
        updateBuilder.setLocalDownloadPath(answer.getDownloadPath());
        updateBuilder.setInstallPath(answer.getInstallPath());
        updateBuilder.setSize(answer.getTemplateSize());
        updateBuilder.setPhysicalSize(answer.getTemplatePhySicalSize());
        _volumeStoreDao.update(volStoreVO.getId(), updateBuilder);
        // update size in volume table
        VolumeVO volUpdater = volumeDao.createForUpdate();
        volUpdater.setSize(answer.getTemplateSize());
        volumeDao.update(obj.getId(), volUpdater);
    }
    AsyncCompletionCallback<CreateCmdResult> caller = context.getParentCallback();
    if (answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR || answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.ABANDONED || answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.UNKNOWN) {
        CreateCmdResult result = new CreateCmdResult(null, null);
        result.setSuccess(false);
        result.setResult(answer.getErrorString());
        caller.complete(result);
        String msg = "Failed to upload volume: " + obj.getUuid() + " with error: " + answer.getErrorString();
        _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_UPLOAD_FAILED, (volStoreVO == null ? -1L : volStoreVO.getZoneId()), null, msg, msg);
        s_logger.error(msg);
    } else if (answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) {
        CreateCmdResult result = new CreateCmdResult(null, null);
        caller.complete(result);
    }
    return null;
}