Examples with VolumeIngestionContext com.emc.storageos.api.service.impl.resource.blockingestorchestration.context.VolumeIngestionContext used on opensource projects

Example 1 with VolumeIngestionContext

use of com.emc.storageos.api.service.impl.resource.blockingestorchestration.context.VolumeIngestionContext in project coprhd-controller by CoprHD.

the class IngestVolumesExportedSchedulingThread method run.

@Override
public void run() {
    try {
        _requestContext.reset();
        URI varrayId = null;
        while (_requestContext.hasNext()) {
            UnManagedVolume unManagedVolume = _requestContext.next();
            _logger.info("Ingestion starting for exported unmanaged volume {}", unManagedVolume.getNativeGuid());
            if (null == varrayId) {
                varrayId = _requestContext.getVarray(unManagedVolume).getId();
            }
            TaskResourceRep resourceRep = _taskMap.get(unManagedVolume.getId().toString());
            String taskId = resourceRep != null ? resourceRep.getOpId() : null;
            try {
                URI storageSystemUri = unManagedVolume.getStorageSystemUri();
                StorageSystem system = _requestContext.getStorageSystemCache().get(storageSystemUri.toString());
                if (null == system) {
                    system = _dbClient.queryObject(StorageSystem.class, storageSystemUri);
                    _requestContext.getStorageSystemCache().put(storageSystemUri.toString(), system);
                }
                // Build the Strategy , which contains reference to Block object & export orchestrators
                IngestStrategy ingestStrategy = _ingestStrategyFactory.buildIngestStrategy(unManagedVolume, !IngestStrategyFactory.DISREGARD_PROTECTION);
                @SuppressWarnings("unchecked") BlockObject blockObject = ingestStrategy.ingestBlockObjects(_requestContext, VolumeIngestionUtil.getBlockObjectClass(unManagedVolume));
                if (null == blockObject) {
                    throw IngestionException.exceptions.generalVolumeException(unManagedVolume.getLabel(), "check the logs for more details");
                }
                _requestContext.getBlockObjectsToBeCreatedMap().put(blockObject.getNativeGuid(), blockObject);
                _requestContext.getProcessedUnManagedVolumeMap().put(unManagedVolume.getNativeGuid(), _requestContext.getVolumeContext());
                _logger.info("Volume ingestion completed successfully for exported unmanaged volume {} (export ingestion will follow)", unManagedVolume.getNativeGuid());
            } catch (APIException ex) {
                _logger.error("error: " + ex.getLocalizedMessage(), ex);
                _dbClient.error(UnManagedVolume.class, _requestContext.getCurrentUnManagedVolumeUri(), taskId, ex);
                _requestContext.getVolumeContext().rollback();
            } catch (Exception ex) {
                _logger.error("error: " + ex.getLocalizedMessage(), ex);
                _dbClient.error(UnManagedVolume.class, _requestContext.getCurrentUnManagedVolumeUri(), taskId, IngestionException.exceptions.generalVolumeException(unManagedVolume.getLabel(), ex.getLocalizedMessage()));
                _requestContext.getVolumeContext().rollback();
            }
        }
        _logger.info("Ingestion of all the unmanaged volumes has completed.");
        // next ingest the export masks for the unmanaged volumes which have been fully ingested
        _logger.info("Ingestion of unmanaged export masks for all requested volumes starting.");
        ingestBlockExportMasks(_requestContext, _taskMap);
        for (VolumeIngestionContext volumeContext : _requestContext.getProcessedUnManagedVolumeMap().values()) {
            // If there is a CG involved in the ingestion, organize, pollenate, and commit.
            _unManagedVolumeService.commitIngestedCG(_requestContext, volumeContext.getUnmanagedVolume());
            // commit the volume itself
            volumeContext.commit();
        }
        for (BlockObject bo : _requestContext.getObjectsIngestedByExportProcessing()) {
            _logger.info("Ingestion Wrap Up: Creating BlockObject {} (hash {})", bo.forDisplay(), bo.hashCode());
            _dbClient.createObject(bo);
        }
        for (UnManagedVolume umv : _requestContext.getUnManagedVolumesToBeDeleted()) {
            _logger.info("Ingestion Wrap Up: Deleting UnManagedVolume {} (hash {})", umv.forDisplay(), umv.hashCode());
            _dbClient.updateObject(umv);
        }
        // Update the related objects if any after successful export mask ingestion
        for (Entry<String, Set<DataObject>> updatedObjectsEntry : _requestContext.getDataObjectsToBeUpdatedMap().entrySet()) {
            if (updatedObjectsEntry != null) {
                _logger.info("Ingestion Wrap Up: Updating objects for UnManagedVolume URI " + updatedObjectsEntry.getKey());
                for (DataObject dob : updatedObjectsEntry.getValue()) {
                    if (dob.getInactive()) {
                        _logger.info("Ingestion Wrap Up: Deleting DataObject {} (hash {})", dob.forDisplay(), dob.hashCode());
                    } else {
                        _logger.info("Ingestion Wrap Up: Updating DataObject {} (hash {})", dob.forDisplay(), dob.hashCode());
                    }
                    _dbClient.updateObject(dob);
                }
            }
        }
        // Create the related objects if any after successful export mask ingestion
        for (Set<DataObject> createdObjects : _requestContext.getDataObjectsToBeCreatedMap().values()) {
            if (createdObjects != null && !createdObjects.isEmpty()) {
                for (DataObject dob : createdObjects) {
                    _logger.info("Ingestion Wrap Up: Creating DataObject {} (hash {})", dob.forDisplay(), dob.hashCode());
                    _dbClient.createObject(dob);
                }
            }
        }
        ExportGroup exportGroup = _requestContext.getExportGroup();
        if (_requestContext.isExportGroupCreated()) {
            _logger.info("Ingestion Wrap Up: Creating ExportGroup {} (hash {})", exportGroup.forDisplay(), exportGroup.hashCode());
            _dbClient.createObject(exportGroup);
        } else {
            _logger.info("Ingestion Wrap Up: Updating ExportGroup {} (hash {})", exportGroup.forDisplay(), exportGroup.hashCode());
            _dbClient.updateObject(exportGroup);
        }
        // record the events after they have been persisted
        for (BlockObject volume : _requestContext.getObjectsIngestedByExportProcessing()) {
            _unManagedVolumeService.recordVolumeOperation(_dbClient, _unManagedVolumeService.getOpByBlockObjectType(volume), Status.ready, volume.getId());
        }
    } catch (InternalException e) {
        _logger.error("InternalException occurred due to: {}", e);
        throw e;
    } catch (Exception e) {
        _logger.error("Unexpected exception occurred due to: {}", e);
        throw APIException.internalServerErrors.genericApisvcError(ExceptionUtils.getExceptionMessage(e), e);
    } finally {
        // it, then we should clean it up in the database (CTRL-8520)
        if ((null != _requestContext) && _requestContext.isExportGroupCreated() && _requestContext.getObjectsIngestedByExportProcessing().isEmpty()) {
            _logger.info("Ingestion Wrap Up: an export group was created, but no volumes were ingested into it");
            if (_requestContext.getExportGroup().getVolumes() == null || _requestContext.getExportGroup().getVolumes().isEmpty()) {
                _logger.info("Ingestion Wrap Up: since no volumes are present, marking {} for deletion", _requestContext.getExportGroup().getLabel());
                _dbClient.markForDeletion(_requestContext.getExportGroup());
            }
        }
    }
}

Example 2 with VolumeIngestionContext

use of com.emc.storageos.api.service.impl.resource.blockingestorchestration.context.VolumeIngestionContext in project coprhd-controller by CoprHD.

the class IngestVolumesExportedSchedulingThread method ingestBlockExportMasks.

/**
 * Ingest block export masks for the already-ingested Volumes.
 *
 * @param requestContext the IngestionRequestContext
 * @param taskMap a Map of UnManagedVolume ids to TaskResourceReps
 */
private void ingestBlockExportMasks(IngestionRequestContext requestContext, Map<String, TaskResourceRep> taskMap) {
    for (String unManagedVolumeGUID : requestContext.getProcessedUnManagedVolumeMap().keySet()) {
        BlockObject processedBlockObject = requestContext.getProcessedBlockObject(unManagedVolumeGUID);
        VolumeIngestionContext volumeContext = requestContext.getVolumeContext(unManagedVolumeGUID);
        UnManagedVolume processedUnManagedVolume = volumeContext.getUnmanagedVolume();
        URI unManagedVolumeUri = processedUnManagedVolume.getId();
        TaskResourceRep resourceRep = taskMap.get(processedUnManagedVolume.getId().toString());
        String taskId = resourceRep != null ? resourceRep.getOpId() : null;
        try {
            if (processedBlockObject == null) {
                _logger.warn("The ingested block object is null. Skipping ingestion of export masks for unmanaged volume {}", unManagedVolumeGUID);
                throw IngestionException.exceptions.generalVolumeException(processedUnManagedVolume.getLabel(), "check the logs for more details");
            }
            // Build the Strategy , which contains reference to Block object & export orchestrators
            IngestExportStrategy ingestStrategy = _ingestStrategyFactory.buildIngestExportStrategy(processedUnManagedVolume);
            BlockObject blockObject = ingestStrategy.ingestExportMasks(processedUnManagedVolume, processedBlockObject, requestContext);
            if (null == blockObject) {
                throw IngestionException.exceptions.generalVolumeException(processedUnManagedVolume.getLabel(), "check the logs for more details");
            }
            if (null == blockObject.getCreationTime()) {
                // only add objects to create if they were created this round of ingestion,
                // creationTime will be null unless the object has already been saved to the db
                requestContext.getObjectsIngestedByExportProcessing().add(blockObject);
            }
            // If the ingested object is internal, flag an error. If it's an RP volume, it's exempt from this check.
            if (blockObject.checkInternalFlags(Flag.PARTIALLY_INGESTED) && !(blockObject instanceof Volume && ((Volume) blockObject).getRpCopyName() != null)) {
                StringBuffer taskStatus = requestContext.getTaskStatusMap().get(processedUnManagedVolume.getNativeGuid());
                String taskMessage = "";
                if (taskStatus == null) {
                    // No task status found. Put in a default message.
                    taskMessage = String.format("Not all the parent/replicas of unmanaged volume %s have been ingested", processedUnManagedVolume.getLabel());
                } else {
                    taskMessage = taskStatus.toString();
                }
                _dbClient.error(UnManagedVolume.class, processedUnManagedVolume.getId(), taskId, IngestionException.exceptions.unmanagedVolumeIsNotVisible(processedUnManagedVolume.getLabel(), taskMessage));
            } else {
                _dbClient.ready(UnManagedVolume.class, processedUnManagedVolume.getId(), taskId, INGESTION_SUCCESSFUL_MSG);
            }
        } catch (APIException ex) {
            _logger.warn(ex.getLocalizedMessage(), ex);
            _dbClient.error(UnManagedVolume.class, unManagedVolumeUri, taskId, ex);
            volumeContext.rollback();
        } catch (Exception ex) {
            _logger.warn(ex.getLocalizedMessage(), ex);
            _dbClient.error(UnManagedVolume.class, unManagedVolumeUri, taskId, IngestionException.exceptions.generalVolumeException(processedUnManagedVolume.getLabel(), ex.getLocalizedMessage()));
            volumeContext.rollback();
        }
    }
}

Example 3 with VolumeIngestionContext

use of com.emc.storageos.api.service.impl.resource.blockingestorchestration.context.VolumeIngestionContext in project coprhd-controller by CoprHD.

the class IngestVolumesUnexportedSchedulingThread method run.

@Override
public void run() {
    _requestContext.reset();
    while (_requestContext.hasNext()) {
        UnManagedVolume unManagedVolume = _requestContext.next();
        String taskId = _taskMap.get(unManagedVolume.getId().toString());
        try {
            _logger.info("Ingestion starting for unmanaged volume {}", unManagedVolume.getNativeGuid());
            List<URI> volList = new ArrayList<URI>();
            volList.add(_requestContext.getCurrentUnManagedVolumeUri());
            VolumeIngestionUtil.checkIngestionRequestValidForUnManagedVolumes(volList, _requestContext.getVpool(unManagedVolume), _dbClient);
            IngestStrategy ingestStrategy = _ingestStrategyFactory.buildIngestStrategy(unManagedVolume, !IngestStrategyFactory.DISREGARD_PROTECTION);
            @SuppressWarnings("unchecked") BlockObject blockObject = ingestStrategy.ingestBlockObjects(_requestContext, VolumeIngestionUtil.getBlockObjectClass(unManagedVolume));
            if (null == blockObject) {
                throw IngestionException.exceptions.generalVolumeException(unManagedVolume.getLabel(), "check the logs for more details");
            }
            _logger.info("Ingestion completed successfully for unmanaged volume {}", unManagedVolume.getNativeGuid());
            _requestContext.getBlockObjectsToBeCreatedMap().put(blockObject.getNativeGuid(), blockObject);
            _requestContext.getProcessedUnManagedVolumeMap().put(unManagedVolume.getNativeGuid(), _requestContext.getVolumeContext());
        } catch (APIException ex) {
            _logger.error("APIException occurred", ex);
            _dbClient.error(UnManagedVolume.class, _requestContext.getCurrentUnManagedVolumeUri(), taskId, ex);
            _requestContext.getVolumeContext().rollback();
        } catch (Exception ex) {
            _logger.error("Exception occurred", ex);
            _dbClient.error(UnManagedVolume.class, _requestContext.getCurrentUnManagedVolumeUri(), taskId, IngestionException.exceptions.generalVolumeException(unManagedVolume.getLabel(), ex.getLocalizedMessage()));
            _requestContext.getVolumeContext().rollback();
        }
    }
    try {
        // update the task status
        for (String unManagedVolumeGUID : _requestContext.getProcessedUnManagedVolumeMap().keySet()) {
            VolumeIngestionContext volumeContext = _requestContext.getProcessedUnManagedVolumeMap().get(unManagedVolumeGUID);
            UnManagedVolume unManagedVolume = volumeContext.getUnmanagedVolume();
            String taskMessage = "";
            String taskId = _taskMap.get(unManagedVolume.getId().toString());
            boolean ingestedSuccessfully = false;
            if (unManagedVolume.getInactive()) {
                ingestedSuccessfully = true;
                taskMessage = INGESTION_SUCCESSFUL_MSG;
            } else {
                // check in the created objects for corresponding block object without any internal flags set
                BlockObject createdObject = _requestContext.findCreatedBlockObject(unManagedVolumeGUID.replace(VolumeIngestionUtil.UNMANAGEDVOLUME, VolumeIngestionUtil.VOLUME));
                _logger.info("checking partial ingestion status of block object " + createdObject);
                if ((null != createdObject) && (!createdObject.checkInternalFlags(Flag.PARTIALLY_INGESTED) || // If this is an ingested RP volume in an uningested protection set, the ingest is successful.
                (createdObject instanceof Volume && ((Volume) createdObject).checkForRp() && ((Volume) createdObject).getProtectionSet() == null)) || // If this is a successfully processed VPLEX backend volume, it will have the INTERNAL_OBJECT Flag
                (VolumeIngestionUtil.isVplexBackendVolume(unManagedVolume) && createdObject.checkInternalFlags(Flag.INTERNAL_OBJECT))) {
                    _logger.info("successfully partially ingested block object {} ", createdObject.forDisplay());
                    ingestedSuccessfully = true;
                    taskMessage = INGESTION_SUCCESSFUL_MSG;
                } else {
                    _logger.info("block object {} was not (partially) ingested successfully", createdObject);
                    ingestedSuccessfully = false;
                    StringBuffer taskStatus = _requestContext.getTaskStatusMap().get(unManagedVolume.getNativeGuid());
                    if (taskStatus == null) {
                        // No task status found. Put in a default message.
                        taskMessage = String.format("Not all the parent/replicas of unmanaged volume %s have been ingested", unManagedVolume.getLabel());
                    } else {
                        taskMessage = taskStatus.toString();
                    }
                }
            }
            if (ingestedSuccessfully) {
                _dbClient.ready(UnManagedVolume.class, unManagedVolume.getId(), taskId, taskMessage);
            } else {
                _dbClient.error(UnManagedVolume.class, unManagedVolume.getId(), taskId, IngestionException.exceptions.unmanagedVolumeIsNotVisible(unManagedVolume.getLabel(), taskMessage));
            }
            // Commit any ingested CG
            _unManagedVolumeService.commitIngestedCG(_requestContext, unManagedVolume);
            // Commit the volume's internal resources
            volumeContext.commit();
            // Commit this volume's updated data objects if any after ingestion
            Set<DataObject> updatedObjects = _requestContext.getDataObjectsToBeUpdatedMap().get(unManagedVolumeGUID);
            if (updatedObjects != null && !updatedObjects.isEmpty()) {
                for (DataObject dob : updatedObjects) {
                    _logger.info("Ingestion Wrap Up: Updating DataObject {} (hash {})", dob.forDisplay(), dob.hashCode());
                    _dbClient.updateObject(dob);
                }
            }
            // Commit this volume's created data objects if any after ingestion
            Set<DataObject> createdObjects = _requestContext.getDataObjectsToBeCreatedMap().get(unManagedVolumeGUID);
            if (createdObjects != null && !createdObjects.isEmpty()) {
                for (DataObject dob : createdObjects) {
                    _logger.info("Ingestion Wrap Up: Creating DataObject {} (hash {})", dob.forDisplay(), dob.hashCode());
                    _dbClient.createObject(dob);
                }
            }
        }
    } catch (InternalException e) {
        throw e;
    } catch (Exception e) {
        _logger.debug("Unexpected ingestion exception:", e);
        throw APIException.internalServerErrors.genericApisvcError(ExceptionUtils.getExceptionMessage(e), e);
    }
    for (BlockObject bo : _requestContext.getBlockObjectsToBeCreatedMap().values()) {
        _logger.info("Ingestion Wrap Up: Creating BlockObject {} (hash {})", bo.forDisplay(), bo.hashCode());
        _dbClient.createObject(bo);
    }
    for (UnManagedVolume umv : _requestContext.getUnManagedVolumesToBeDeleted()) {
        _logger.info("Ingestion Wrap Up: Deleting UnManagedVolume {} (hash {})", umv.forDisplay(), umv.hashCode());
        _dbClient.updateObject(umv);
    }
    // record the events after they have been persisted
    for (BlockObject volume : _requestContext.getBlockObjectsToBeCreatedMap().values()) {
        _unManagedVolumeService.recordVolumeOperation(_dbClient, _unManagedVolumeService.getOpByBlockObjectType(volume), Status.ready, volume.getId());
    }
}

Example 4 with VolumeIngestionContext

use of com.emc.storageos.api.service.impl.resource.blockingestorchestration.context.VolumeIngestionContext in project coprhd-controller by CoprHD.

the class BaseIngestionRequestContext method findDataObjectByType.

/*
     * (non-Javadoc)
     * 
     * @see
     * com.emc.storageos.api.service.impl.resource.blockingestorchestration.context.IngestionRequestContext#findObjectAnywhere(java.lang.
     * Class, java.net.URI)
     */
@Override
public <T extends DataObject> T findDataObjectByType(Class<T> clazz, URI id, boolean fallbackToDatabase) {
    _logger.info("looking for {} object with id {}", clazz.toString(), id);
    // check for DataObjects in already-loaded Updated Objects first
    DataObject dob = this.findInUpdatedObjects(id);
    if (clazz.isInstance(dob)) {
        _logger.info("\tfound in updated objects: " + dob.forDisplay());
        return clazz.cast(dob);
    }
    // check for Mirrors/Snapshots/Volumes that have been created
    if (clazz.equals(BlockMirror.class) || clazz.equals(Volume.class) || clazz.equals(BlockSnapshot.class)) {
        BlockObject bo = this.findCreatedBlockObject(id);
        if (clazz.isInstance(bo)) {
            _logger.info("\tfound in created objects: " + bo.forDisplay());
            return clazz.cast(bo);
        }
    }
    // search for any already-loaded UnManagedVolume instances
    if (clazz.equals(UnManagedVolume.class)) {
        for (UnManagedVolume umv : this.findAllUnManagedVolumesToBeDeleted()) {
            if (umv != null && umv.getId().equals(id)) {
                _logger.info("\tfound in volumes to be deleted: " + umv.forDisplay());
                return clazz.cast(umv);
            }
        }
        VolumeIngestionContext currentVolumeContext = getVolumeContext();
        if (currentVolumeContext != null && currentVolumeContext instanceof IngestionRequestContext) {
            UnManagedVolume umv = currentVolumeContext.getUnmanagedVolume();
            if (umv != null && umv.getId().equals(id)) {
                _logger.info("\tfound in current volume context: " + umv.forDisplay());
                return clazz.cast(umv);
            }
        }
        for (VolumeIngestionContext volumeContext : this.getProcessedUnManagedVolumeMap().values()) {
            if (volumeContext instanceof IngestionRequestContext) {
                UnManagedVolume umv = volumeContext.getUnmanagedVolume();
                if (umv != null && umv.getId().equals(id)) {
                    _logger.info("\tfound in already-processed volume context: " + umv.forDisplay());
                    return clazz.cast(umv);
                }
            }
        }
    }
    // search for any already-loaded UnManagedProtectionSet instances
    if (clazz.equals(UnManagedProtectionSet.class)) {
        VolumeIngestionContext currentVolumeContext = getVolumeContext();
        if (currentVolumeContext != null && currentVolumeContext instanceof RecoverPointVolumeIngestionContext) {
            UnManagedProtectionSet umpset = ((RecoverPointVolumeIngestionContext) currentVolumeContext).getUnManagedProtectionSetLocal();
            if (umpset != null && umpset.getId().equals(id)) {
                _logger.info("\tfound in current volume context: " + umpset.forDisplay());
                return clazz.cast(umpset);
            }
        }
        for (VolumeIngestionContext volumeContext : this.getProcessedUnManagedVolumeMap().values()) {
            if (volumeContext != null && volumeContext instanceof RecoverPointVolumeIngestionContext) {
                UnManagedProtectionSet umpset = ((RecoverPointVolumeIngestionContext) volumeContext).getUnManagedProtectionSetLocal();
                if (umpset != null && umpset.getId().equals(id)) {
                    _logger.info("\tfound in already-processed volume context: " + umpset.forDisplay());
                    return clazz.cast(umpset);
                }
            }
        }
    }
    if (fallbackToDatabase) {
        // if we still haven't found it, load it from the database
        T dataObject = _dbClient.queryObject(clazz, id);
        if (dataObject != null) {
            _logger.info("\tloaded object from database: " + dataObject.forDisplay());
            return clazz.cast(dataObject);
        }
    }
    return null;
}

Example 5 with VolumeIngestionContext

use of com.emc.storageos.api.service.impl.resource.blockingestorchestration.context.VolumeIngestionContext in project coprhd-controller by CoprHD.

the class BaseIngestionRequestContext method findUnManagedConsistencyGroup.

/*
     * (non-Javadoc)
     *
     * @see
     * com.emc.storageos.api.service.impl.resource.blockingestorchestration.context.IngestionRequestContext#findUnManagedConsistencyGroup(
     * com.emc.storageos.db.client.model.BlockConsistencyGroup)
     */
@Override
public UnManagedConsistencyGroup findUnManagedConsistencyGroup(String cgName) {
    VolumeIngestionContext currentVolumeContext = getVolumeContext();
    if (currentVolumeContext != null) {
        _logger.info("checking current volume ingestion context {}", currentVolumeContext.getUnmanagedVolume().forDisplay());
        List<UnManagedConsistencyGroup> umcgList = currentVolumeContext.getUmCGObjectsToUpdate();
        if (null != umcgList && !umcgList.isEmpty()) {
            for (UnManagedConsistencyGroup umcg : umcgList) {
                if (umcg.getLabel().equalsIgnoreCase(cgName)) {
                    return umcg;
                }
            }
        }
    }
    for (VolumeIngestionContext volumeContext : this.getProcessedUnManagedVolumeMap().values()) {
        _logger.info("checking already-ingested volume ingestion context {}", volumeContext.getUnmanagedVolume().forDisplay());
        List<UnManagedConsistencyGroup> umcgList = volumeContext.getUmCGObjectsToUpdate();
        if (null != umcgList && !umcgList.isEmpty()) {
            for (UnManagedConsistencyGroup umcg : umcgList) {
                if (umcg.getLabel().equalsIgnoreCase(cgName)) {
                    return umcg;
                }
            }
        }
    }
    return null;
}