use of com.emc.storageos.db.client.model.DataObject in project coprhd-controller by CoprHD.
the class BlockRecoverPointIngestOrchestrator method decorateVolumeInformationFinalIngest.
/**
* This method will perform all of the final decorations (attribute setting) on the Volume
* object after creating the required BlockConsistencyGroup and ProtectionSet objects.
*
* Fields such as rpCopyName and rSetName were already filled in when we did the ingest of
* the volume itself. In this method, we worry about stitching together all of the object
* references within the Volume object so it will act like a native CoprHD-created RP volume.
*
* @param volumeContext the RecoverPointVolumeIngestionContext for the volume currently being ingested
* @param unManagedVolume the currently ingesting UnManagedVolume
*/
private void decorateVolumeInformationFinalIngest(IngestionRequestContext requestContext, UnManagedVolume unManagedVolume) {
RecoverPointVolumeIngestionContext volumeContext = (RecoverPointVolumeIngestionContext) requestContext.getVolumeContext();
ProtectionSet pset = volumeContext.getManagedProtectionSet();
BlockConsistencyGroup cg = volumeContext.getManagedBlockConsistencyGroup();
if (pset.getVolumes() == null) {
_logger.error("No volumes found in protection set: " + pset.getLabel() + ", cannot process ingestion");
throw IngestionException.exceptions.noVolumesFoundInProtectionSet(pset.getLabel());
}
List<Volume> volumes = new ArrayList<Volume>();
for (String volId : pset.getVolumes()) {
BlockObject volume = requestContext.getRootIngestionRequestContext().findCreatedBlockObject(URI.create(volId));
if (volume != null && volume instanceof Volume) {
volumes.add((Volume) volume);
}
}
// Make sure all of the changed managed block objects get updated
volumes.add((Volume) volumeContext.getManagedBlockObject());
Set<DataObject> updatedObjects = new HashSet<DataObject>();
VolumeIngestionUtil.decorateRPVolumesCGInfo(volumes, pset, cg, updatedObjects, _dbClient, requestContext);
VolumeIngestionUtil.clearPersistedReplicaFlags(requestContext, volumes, updatedObjects, _dbClient);
clearReplicaFlagsInIngestionContext(volumeContext, volumes);
for (DataObject volume : updatedObjects) {
if (volumeContext.getManagedBlockObject().getId().equals(volume.getId()) && (null == _dbClient.queryObject(Volume.class, volume.getId()))) {
// this is the managed block object and it hasn't been saved to the db yet
continue;
} else {
// add all volumes except the newly ingested one to the update list
volumeContext.addDataObjectToUpdate(volume, unManagedVolume);
}
}
}
use of com.emc.storageos.db.client.model.DataObject in project coprhd-controller by CoprHD.
the class BlockVplexVolumeIngestOrchestrator method decorateCGInfoInVolumes.
/**
* Decorates the CG uri & replicationGroupName for the VPLEX & its Backend volumes.
* This information will be used when decorating CG at the end of the ingestion process.
*
* For each backend unmanaged volume,
* 1. We verify whether the BlockObject is available in the current createdBlockObjects in context or not.
* If it is available, then set the CG properties
* Else, verify in the current updatedBlockObjects in context.
* 2. If the backend blockObject is available in updateBlockObjects, then update CG properties.
* Else, blockObject might have ingested in previous requests, so, we should check from DB.
* If blockObject is in DB, update CG properties else log a warning message.
*/
@Override
protected void decorateCGInfoInVolumes(BlockConsistencyGroup vplexCG, BlockObject volume, IngestionRequestContext requestContext, UnManagedVolume unManagedVolume) {
updateCG(vplexCG, volume, requestContext.getStorageSystem(), unManagedVolume);
StringSet vplexBackendVolumes = PropertySetterUtil.extractValuesFromStringSet(SupportedVolumeInformation.VPLEX_BACKEND_VOLUMES.toString(), unManagedVolume.getVolumeInformation());
if (null != vplexBackendVolumes && !vplexBackendVolumes.isEmpty()) {
Set<DataObject> toUpdateSet = new HashSet<DataObject>();
for (String vplexBackendUmvNativeGuid : vplexBackendVolumes) {
String backendVolumeNativeGuid = vplexBackendUmvNativeGuid.replace(VolumeIngestionUtil.UNMANAGEDVOLUME, VolumeIngestionUtil.VOLUME);
BlockObject blockObject = requestContext.getRootIngestionRequestContext().findCreatedBlockObject(backendVolumeNativeGuid);
if (blockObject == null) {
// Next look in the updated objects.
blockObject = (BlockObject) requestContext.findInUpdatedObjects(URI.create(backendVolumeNativeGuid));
}
if (blockObject == null) {
// Finally look in the DB itself. It may be from a previous ingestion operation.
blockObject = VolumeIngestionUtil.getBlockObject(backendVolumeNativeGuid, _dbClient);
// If blockObject is still not exists
if (null == blockObject) {
_logger.warn("Unmanaged Volume {} is not yet ingested. Hence skipping", vplexBackendUmvNativeGuid);
continue;
}
toUpdateSet.add(blockObject);
}
blockObject.setConsistencyGroup(vplexCG.getId());
}
if (!toUpdateSet.isEmpty()) {
// Since I pulled this in from the database, we need to add it to the list of objects to update.
((VplexVolumeIngestionContext) requestContext.getVolumeContext()).getDataObjectsToBeUpdatedMap().put(unManagedVolume.getNativeGuid(), toUpdateSet);
}
}
volume.setConsistencyGroup(vplexCG.getId());
volume.setReplicationGroupInstance(vplexCG.getLabel());
}
use of com.emc.storageos.db.client.model.DataObject in project coprhd-controller by CoprHD.
the class IngestExportStrategy method ingestExportMasks.
/**
* After volume object gets created successfully locally, now start
* running ingest associated masks of the volume
*/
public <T extends BlockObject> T ingestExportMasks(UnManagedVolume unManagedVolume, T blockObject, IngestionRequestContext requestContext) throws IngestionException {
_logger.info("ingesting export masks for requestContext " + requestContext.getCurrentUnmanagedVolume());
if (null != requestContext.getExportGroup()) {
if (null != unManagedVolume.getUnmanagedExportMasks() && !unManagedVolume.getUnmanagedExportMasks().isEmpty()) {
List<URI> unManagedMaskUris = new ArrayList<URI>(Collections2.transform(unManagedVolume.getUnmanagedExportMasks(), CommonTransformerFunctions.FCTN_STRING_TO_URI));
List<UnManagedExportMask> unManagedMasks = _dbClient.queryObject(UnManagedExportMask.class, unManagedMaskUris);
int originalSize = unManagedMasks.size();
MutableInt masksIngestedCount = new MutableInt(0);
// Ingest Associated Masks
ingestExportOrchestrator.ingestExportMasks(requestContext, unManagedVolume, blockObject, unManagedMasks, masksIngestedCount);
_logger.info("{} of {} unmanaged export masks were ingested", masksIngestedCount, originalSize);
List<String> errorMessages = requestContext.getErrorMessagesForVolume(unManagedVolume.getNativeGuid());
// If the internal flags are set, return the block object
if (blockObject.checkInternalFlags(Flag.PARTIALLY_INGESTED)) {
_logger.info("block object {} is partially ingested", blockObject.forDisplay());
// check if none of the export masks are ingested
if (masksIngestedCount.intValue() == 0) {
if (null != errorMessages && !errorMessages.isEmpty()) {
throw IngestionException.exceptions.unmanagedVolumeMasksNotIngestedAdditionalInfo(unManagedVolume.getLabel(), Joiner.on(", ").join(errorMessages));
} else {
throw IngestionException.exceptions.unmanagedVolumeMasksNotIngested(unManagedVolume.getLabel());
}
} else {
// If the unmanaged volume is not marked for deletion, then it should be updated with the changes done.
requestContext.addDataObjectToUpdate(unManagedVolume, unManagedVolume);
_logger.info("all export masks of unmanaged volume {} have been ingested, " + "but the volume is still marked as partially ingested, returning block object {}", unManagedVolume.forDisplay(), blockObject.forDisplay());
return blockObject;
}
}
if (unManagedVolume.getUnmanagedExportMasks().size() != originalSize) {
// delete this volume only if the masks are ingested.
if (VolumeIngestionUtil.canDeleteUnManagedVolume(unManagedVolume)) {
_logger.info("Marking UnManaged Volume {} inactive as it doesn't have any associated unmanaged export masks ", unManagedVolume.getNativeGuid());
boolean isRPVolume = VolumeIngestionUtil.checkUnManagedResourceIsRecoverPointEnabled(unManagedVolume);
// if its RP volume and non RP exported, then check whether the RP CG is fully ingested
if (isRPVolume && VolumeIngestionUtil.checkUnManagedResourceIsNonRPExported(unManagedVolume)) {
_logger.info("unmanaged volume {} is both RecoverPoint protected and exported to another Host or Cluster", unManagedVolume.forDisplay());
Set<DataObject> updateObjects = requestContext.getDataObjectsToBeUpdatedMap().get(unManagedVolume.getNativeGuid());
if (updateObjects == null) {
updateObjects = new HashSet<DataObject>();
requestContext.getDataObjectsToBeUpdatedMap().put(unManagedVolume.getNativeGuid(), updateObjects);
}
List<UnManagedVolume> ingestedUnManagedVolumes = requestContext.findAllUnManagedVolumesToBeDeleted();
ingestedUnManagedVolumes.add(unManagedVolume);
UnManagedProtectionSet umpset = VolumeIngestionUtil.getUnManagedProtectionSetForUnManagedVolume(requestContext, unManagedVolume, _dbClient);
// unmanaged volume has already been ingested. In this case, try to get it from the managed volume
if (umpset == null) {
umpset = VolumeIngestionUtil.getUnManagedProtectionSetForManagedVolume(requestContext, blockObject, _dbClient);
}
// If fully ingested, then setup the RP CG too.
if (VolumeIngestionUtil.validateAllVolumesInCGIngested(ingestedUnManagedVolumes, umpset, requestContext, _dbClient)) {
VolumeIngestionUtil.validateRPVolumesAlignWithIngestVpool(requestContext, umpset, _dbClient);
VolumeIngestionUtil.setupRPCG(requestContext, umpset, unManagedVolume, updateObjects, _dbClient);
} else {
// else mark the volume as internal. This will be marked visible when the RP CG is ingested
blockObject.addInternalFlags(BlockRecoverPointIngestOrchestrator.INTERNAL_VOLUME_FLAGS);
}
}
unManagedVolume.setInactive(true);
requestContext.getUnManagedVolumesToBeDeleted().add(unManagedVolume);
} else {
// If the unmanaged volume is not marked for deletion, then it should be updated with the changes done.
requestContext.addDataObjectToUpdate(unManagedVolume, unManagedVolume);
}
return blockObject;
} else {
if (null != errorMessages && !errorMessages.isEmpty()) {
Collections.sort(errorMessages);
throw IngestionException.exceptions.unmanagedVolumeMasksNotIngestedAdditionalInfo(unManagedVolume.getLabel(), "\n\n" + Joiner.on("\n\n").join(errorMessages));
} else {
throw IngestionException.exceptions.unmanagedVolumeMasksNotIngested(unManagedVolume.getLabel());
}
}
}
}
return blockObject;
}
use of com.emc.storageos.db.client.model.DataObject in project coprhd-controller by CoprHD.
the class IngestVolumesExportedSchedulingThread method run.
@Override
public void run() {
try {
_requestContext.reset();
URI varrayId = null;
while (_requestContext.hasNext()) {
UnManagedVolume unManagedVolume = _requestContext.next();
_logger.info("Ingestion starting for exported unmanaged volume {}", unManagedVolume.getNativeGuid());
if (null == varrayId) {
varrayId = _requestContext.getVarray(unManagedVolume).getId();
}
TaskResourceRep resourceRep = _taskMap.get(unManagedVolume.getId().toString());
String taskId = resourceRep != null ? resourceRep.getOpId() : null;
try {
URI storageSystemUri = unManagedVolume.getStorageSystemUri();
StorageSystem system = _requestContext.getStorageSystemCache().get(storageSystemUri.toString());
if (null == system) {
system = _dbClient.queryObject(StorageSystem.class, storageSystemUri);
_requestContext.getStorageSystemCache().put(storageSystemUri.toString(), system);
}
// Build the Strategy , which contains reference to Block object & export orchestrators
IngestStrategy ingestStrategy = _ingestStrategyFactory.buildIngestStrategy(unManagedVolume, !IngestStrategyFactory.DISREGARD_PROTECTION);
@SuppressWarnings("unchecked") BlockObject blockObject = ingestStrategy.ingestBlockObjects(_requestContext, VolumeIngestionUtil.getBlockObjectClass(unManagedVolume));
if (null == blockObject) {
throw IngestionException.exceptions.generalVolumeException(unManagedVolume.getLabel(), "check the logs for more details");
}
_requestContext.getBlockObjectsToBeCreatedMap().put(blockObject.getNativeGuid(), blockObject);
_requestContext.getProcessedUnManagedVolumeMap().put(unManagedVolume.getNativeGuid(), _requestContext.getVolumeContext());
_logger.info("Volume ingestion completed successfully for exported unmanaged volume {} (export ingestion will follow)", unManagedVolume.getNativeGuid());
} catch (APIException ex) {
_logger.error("error: " + ex.getLocalizedMessage(), ex);
_dbClient.error(UnManagedVolume.class, _requestContext.getCurrentUnManagedVolumeUri(), taskId, ex);
_requestContext.getVolumeContext().rollback();
} catch (Exception ex) {
_logger.error("error: " + ex.getLocalizedMessage(), ex);
_dbClient.error(UnManagedVolume.class, _requestContext.getCurrentUnManagedVolumeUri(), taskId, IngestionException.exceptions.generalVolumeException(unManagedVolume.getLabel(), ex.getLocalizedMessage()));
_requestContext.getVolumeContext().rollback();
}
}
_logger.info("Ingestion of all the unmanaged volumes has completed.");
// next ingest the export masks for the unmanaged volumes which have been fully ingested
_logger.info("Ingestion of unmanaged export masks for all requested volumes starting.");
ingestBlockExportMasks(_requestContext, _taskMap);
for (VolumeIngestionContext volumeContext : _requestContext.getProcessedUnManagedVolumeMap().values()) {
// If there is a CG involved in the ingestion, organize, pollenate, and commit.
_unManagedVolumeService.commitIngestedCG(_requestContext, volumeContext.getUnmanagedVolume());
// commit the volume itself
volumeContext.commit();
}
for (BlockObject bo : _requestContext.getObjectsIngestedByExportProcessing()) {
_logger.info("Ingestion Wrap Up: Creating BlockObject {} (hash {})", bo.forDisplay(), bo.hashCode());
_dbClient.createObject(bo);
}
for (UnManagedVolume umv : _requestContext.getUnManagedVolumesToBeDeleted()) {
_logger.info("Ingestion Wrap Up: Deleting UnManagedVolume {} (hash {})", umv.forDisplay(), umv.hashCode());
_dbClient.updateObject(umv);
}
// Update the related objects if any after successful export mask ingestion
for (Entry<String, Set<DataObject>> updatedObjectsEntry : _requestContext.getDataObjectsToBeUpdatedMap().entrySet()) {
if (updatedObjectsEntry != null) {
_logger.info("Ingestion Wrap Up: Updating objects for UnManagedVolume URI " + updatedObjectsEntry.getKey());
for (DataObject dob : updatedObjectsEntry.getValue()) {
if (dob.getInactive()) {
_logger.info("Ingestion Wrap Up: Deleting DataObject {} (hash {})", dob.forDisplay(), dob.hashCode());
} else {
_logger.info("Ingestion Wrap Up: Updating DataObject {} (hash {})", dob.forDisplay(), dob.hashCode());
}
_dbClient.updateObject(dob);
}
}
}
// Create the related objects if any after successful export mask ingestion
for (Set<DataObject> createdObjects : _requestContext.getDataObjectsToBeCreatedMap().values()) {
if (createdObjects != null && !createdObjects.isEmpty()) {
for (DataObject dob : createdObjects) {
_logger.info("Ingestion Wrap Up: Creating DataObject {} (hash {})", dob.forDisplay(), dob.hashCode());
_dbClient.createObject(dob);
}
}
}
ExportGroup exportGroup = _requestContext.getExportGroup();
if (_requestContext.isExportGroupCreated()) {
_logger.info("Ingestion Wrap Up: Creating ExportGroup {} (hash {})", exportGroup.forDisplay(), exportGroup.hashCode());
_dbClient.createObject(exportGroup);
} else {
_logger.info("Ingestion Wrap Up: Updating ExportGroup {} (hash {})", exportGroup.forDisplay(), exportGroup.hashCode());
_dbClient.updateObject(exportGroup);
}
// record the events after they have been persisted
for (BlockObject volume : _requestContext.getObjectsIngestedByExportProcessing()) {
_unManagedVolumeService.recordVolumeOperation(_dbClient, _unManagedVolumeService.getOpByBlockObjectType(volume), Status.ready, volume.getId());
}
} catch (InternalException e) {
_logger.error("InternalException occurred due to: {}", e);
throw e;
} catch (Exception e) {
_logger.error("Unexpected exception occurred due to: {}", e);
throw APIException.internalServerErrors.genericApisvcError(ExceptionUtils.getExceptionMessage(e), e);
} finally {
// it, then we should clean it up in the database (CTRL-8520)
if ((null != _requestContext) && _requestContext.isExportGroupCreated() && _requestContext.getObjectsIngestedByExportProcessing().isEmpty()) {
_logger.info("Ingestion Wrap Up: an export group was created, but no volumes were ingested into it");
if (_requestContext.getExportGroup().getVolumes() == null || _requestContext.getExportGroup().getVolumes().isEmpty()) {
_logger.info("Ingestion Wrap Up: since no volumes are present, marking {} for deletion", _requestContext.getExportGroup().getLabel());
_dbClient.markForDeletion(_requestContext.getExportGroup());
}
}
}
}
use of com.emc.storageos.db.client.model.DataObject in project coprhd-controller by CoprHD.
the class IngestVolumesUnexportedSchedulingThread method run.
@Override
public void run() {
_requestContext.reset();
while (_requestContext.hasNext()) {
UnManagedVolume unManagedVolume = _requestContext.next();
String taskId = _taskMap.get(unManagedVolume.getId().toString());
try {
_logger.info("Ingestion starting for unmanaged volume {}", unManagedVolume.getNativeGuid());
List<URI> volList = new ArrayList<URI>();
volList.add(_requestContext.getCurrentUnManagedVolumeUri());
VolumeIngestionUtil.checkIngestionRequestValidForUnManagedVolumes(volList, _requestContext.getVpool(unManagedVolume), _dbClient);
IngestStrategy ingestStrategy = _ingestStrategyFactory.buildIngestStrategy(unManagedVolume, !IngestStrategyFactory.DISREGARD_PROTECTION);
@SuppressWarnings("unchecked") BlockObject blockObject = ingestStrategy.ingestBlockObjects(_requestContext, VolumeIngestionUtil.getBlockObjectClass(unManagedVolume));
if (null == blockObject) {
throw IngestionException.exceptions.generalVolumeException(unManagedVolume.getLabel(), "check the logs for more details");
}
_logger.info("Ingestion completed successfully for unmanaged volume {}", unManagedVolume.getNativeGuid());
_requestContext.getBlockObjectsToBeCreatedMap().put(blockObject.getNativeGuid(), blockObject);
_requestContext.getProcessedUnManagedVolumeMap().put(unManagedVolume.getNativeGuid(), _requestContext.getVolumeContext());
} catch (APIException ex) {
_logger.error("APIException occurred", ex);
_dbClient.error(UnManagedVolume.class, _requestContext.getCurrentUnManagedVolumeUri(), taskId, ex);
_requestContext.getVolumeContext().rollback();
} catch (Exception ex) {
_logger.error("Exception occurred", ex);
_dbClient.error(UnManagedVolume.class, _requestContext.getCurrentUnManagedVolumeUri(), taskId, IngestionException.exceptions.generalVolumeException(unManagedVolume.getLabel(), ex.getLocalizedMessage()));
_requestContext.getVolumeContext().rollback();
}
}
try {
// update the task status
for (String unManagedVolumeGUID : _requestContext.getProcessedUnManagedVolumeMap().keySet()) {
VolumeIngestionContext volumeContext = _requestContext.getProcessedUnManagedVolumeMap().get(unManagedVolumeGUID);
UnManagedVolume unManagedVolume = volumeContext.getUnmanagedVolume();
String taskMessage = "";
String taskId = _taskMap.get(unManagedVolume.getId().toString());
boolean ingestedSuccessfully = false;
if (unManagedVolume.getInactive()) {
ingestedSuccessfully = true;
taskMessage = INGESTION_SUCCESSFUL_MSG;
} else {
// check in the created objects for corresponding block object without any internal flags set
BlockObject createdObject = _requestContext.findCreatedBlockObject(unManagedVolumeGUID.replace(VolumeIngestionUtil.UNMANAGEDVOLUME, VolumeIngestionUtil.VOLUME));
_logger.info("checking partial ingestion status of block object " + createdObject);
if ((null != createdObject) && (!createdObject.checkInternalFlags(Flag.PARTIALLY_INGESTED) || // If this is an ingested RP volume in an uningested protection set, the ingest is successful.
(createdObject instanceof Volume && ((Volume) createdObject).checkForRp() && ((Volume) createdObject).getProtectionSet() == null)) || // If this is a successfully processed VPLEX backend volume, it will have the INTERNAL_OBJECT Flag
(VolumeIngestionUtil.isVplexBackendVolume(unManagedVolume) && createdObject.checkInternalFlags(Flag.INTERNAL_OBJECT))) {
_logger.info("successfully partially ingested block object {} ", createdObject.forDisplay());
ingestedSuccessfully = true;
taskMessage = INGESTION_SUCCESSFUL_MSG;
} else {
_logger.info("block object {} was not (partially) ingested successfully", createdObject);
ingestedSuccessfully = false;
StringBuffer taskStatus = _requestContext.getTaskStatusMap().get(unManagedVolume.getNativeGuid());
if (taskStatus == null) {
// No task status found. Put in a default message.
taskMessage = String.format("Not all the parent/replicas of unmanaged volume %s have been ingested", unManagedVolume.getLabel());
} else {
taskMessage = taskStatus.toString();
}
}
}
if (ingestedSuccessfully) {
_dbClient.ready(UnManagedVolume.class, unManagedVolume.getId(), taskId, taskMessage);
} else {
_dbClient.error(UnManagedVolume.class, unManagedVolume.getId(), taskId, IngestionException.exceptions.unmanagedVolumeIsNotVisible(unManagedVolume.getLabel(), taskMessage));
}
// Commit any ingested CG
_unManagedVolumeService.commitIngestedCG(_requestContext, unManagedVolume);
// Commit the volume's internal resources
volumeContext.commit();
// Commit this volume's updated data objects if any after ingestion
Set<DataObject> updatedObjects = _requestContext.getDataObjectsToBeUpdatedMap().get(unManagedVolumeGUID);
if (updatedObjects != null && !updatedObjects.isEmpty()) {
for (DataObject dob : updatedObjects) {
_logger.info("Ingestion Wrap Up: Updating DataObject {} (hash {})", dob.forDisplay(), dob.hashCode());
_dbClient.updateObject(dob);
}
}
// Commit this volume's created data objects if any after ingestion
Set<DataObject> createdObjects = _requestContext.getDataObjectsToBeCreatedMap().get(unManagedVolumeGUID);
if (createdObjects != null && !createdObjects.isEmpty()) {
for (DataObject dob : createdObjects) {
_logger.info("Ingestion Wrap Up: Creating DataObject {} (hash {})", dob.forDisplay(), dob.hashCode());
_dbClient.createObject(dob);
}
}
}
} catch (InternalException e) {
throw e;
} catch (Exception e) {
_logger.debug("Unexpected ingestion exception:", e);
throw APIException.internalServerErrors.genericApisvcError(ExceptionUtils.getExceptionMessage(e), e);
}
for (BlockObject bo : _requestContext.getBlockObjectsToBeCreatedMap().values()) {
_logger.info("Ingestion Wrap Up: Creating BlockObject {} (hash {})", bo.forDisplay(), bo.hashCode());
_dbClient.createObject(bo);
}
for (UnManagedVolume umv : _requestContext.getUnManagedVolumesToBeDeleted()) {
_logger.info("Ingestion Wrap Up: Deleting UnManagedVolume {} (hash {})", umv.forDisplay(), umv.hashCode());
_dbClient.updateObject(umv);
}
// record the events after they have been persisted
for (BlockObject volume : _requestContext.getBlockObjectsToBeCreatedMap().values()) {
_unManagedVolumeService.recordVolumeOperation(_dbClient, _unManagedVolumeService.getOpByBlockObjectType(volume), Status.ready, volume.getId());
}
}
Aggregations