use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedVolume in project coprhd-controller by CoprHD.
the class BlockIngestOrchestrator method runReplicasIngestedCheck.
/**
* Invoke RunReplicasIngestedCheck
* a. Get the replicas of the ROOT UMV
* b. Find if all replicas Ingested
* c. If Yes, then update parent Replica Map [Parent --> Child]
* d. For each Replica unmanaged volume, RUN STEP 5.
* e. Else Clear Parent Replica and come out.
*
* @param unmanagedVolume
* @param blockObject
* @param processingUnManagedVolume
* @param processingBlockObject
* @param unManagedVolumeGUIDs
* @param parentReplicaMap
* @param requestContext
*/
protected void runReplicasIngestedCheck(UnManagedVolume rootUnmanagedVolume, BlockObject rootBlockObject, UnManagedVolume currentUnManagedVolume, BlockObject currentBlockObject, StringSet unManagedVolumeGUIDs, Map<BlockObject, List<BlockObject>> parentReplicaMap, IngestionRequestContext requestContext) {
if (rootBlockObject == null) {
_logger.warn("parent object {} not ingested yet.", rootUnmanagedVolume.getNativeGuid());
parentReplicaMap.clear();
StringBuffer taskStatus = requestContext.getTaskStatusMap().get(currentUnManagedVolume.getNativeGuid());
if (taskStatus == null) {
taskStatus = new StringBuffer();
requestContext.getTaskStatusMap().put(currentUnManagedVolume.getNativeGuid(), taskStatus);
}
taskStatus.append(String.format("Parent object %s not ingested yet for unmanaged volume %s.", rootUnmanagedVolume.getLabel(), currentUnManagedVolume.getLabel()));
return;
}
boolean traverseTree = true;
String unManagedVolumeNativeGUID = rootUnmanagedVolume.getNativeGuid();
StringSetMap unManagedVolumeInformation = rootUnmanagedVolume.getVolumeInformation();
StringSet unmanagedReplicaGUIDs = new StringSet();
StringSet expectedIngestedReplicas = new StringSet();
List<BlockObject> foundIngestedReplicas = new ArrayList<BlockObject>();
StringSet foundIngestedReplicaNativeGuids = new StringSet();
StringSet mirrors = PropertySetterUtil.extractValuesFromStringSet(SupportedVolumeInformation.MIRRORS.toString(), unManagedVolumeInformation);
if (mirrors != null && !mirrors.isEmpty()) {
unmanagedReplicaGUIDs.addAll(mirrors);
StringSet mirrorGUIDs = VolumeIngestionUtil.getListofVolumeIds(mirrors);
expectedIngestedReplicas.addAll(mirrorGUIDs);
foundIngestedReplicas.addAll(VolumeIngestionUtil.getMirrorObjects(mirrorGUIDs, requestContext, _dbClient));
}
StringSet clones = PropertySetterUtil.extractValuesFromStringSet(SupportedVolumeInformation.FULL_COPIES.toString(), unManagedVolumeInformation);
if (clones != null && !clones.isEmpty()) {
unmanagedReplicaGUIDs.addAll(clones);
StringSet cloneGUIDs = VolumeIngestionUtil.getListofVolumeIds(clones);
expectedIngestedReplicas.addAll(cloneGUIDs);
foundIngestedReplicas.addAll(VolumeIngestionUtil.getVolumeObjects(cloneGUIDs, requestContext, _dbClient));
}
StringSet snaps = PropertySetterUtil.extractValuesFromStringSet(SupportedVolumeInformation.SNAPSHOTS.toString(), unManagedVolumeInformation);
if (snaps != null && !snaps.isEmpty()) {
unmanagedReplicaGUIDs.addAll(snaps);
StringSet snapGUIDs = VolumeIngestionUtil.getListofVolumeIds(snaps);
expectedIngestedReplicas.addAll(snapGUIDs);
foundIngestedReplicas.addAll(VolumeIngestionUtil.getSnapObjects(snapGUIDs, requestContext, _dbClient));
}
StringSet remoteMirrors = PropertySetterUtil.extractValuesFromStringSet(SupportedVolumeInformation.REMOTE_MIRRORS.toString(), unManagedVolumeInformation);
if (remoteMirrors != null && !remoteMirrors.isEmpty()) {
unmanagedReplicaGUIDs.addAll(remoteMirrors);
StringSet remoteMirrorGUIDs = VolumeIngestionUtil.getListofVolumeIds(remoteMirrors);
expectedIngestedReplicas.addAll(remoteMirrorGUIDs);
foundIngestedReplicas.addAll(VolumeIngestionUtil.getVolumeObjects(remoteMirrorGUIDs, requestContext, _dbClient));
}
StringSet vplexBackendVolumes = PropertySetterUtil.extractValuesFromStringSet(SupportedVolumeInformation.VPLEX_BACKEND_VOLUMES.toString(), unManagedVolumeInformation);
StringSet vplexBackendVolumeGUIDs = null;
if (vplexBackendVolumes != null && !vplexBackendVolumes.isEmpty()) {
unmanagedReplicaGUIDs.addAll(vplexBackendVolumes);
vplexBackendVolumeGUIDs = VolumeIngestionUtil.getListofVolumeIds(vplexBackendVolumes);
expectedIngestedReplicas.addAll(vplexBackendVolumeGUIDs);
foundIngestedReplicas.addAll(VolumeIngestionUtil.getVolumeObjects(vplexBackendVolumeGUIDs, requestContext, _dbClient));
}
if (unmanagedReplicaGUIDs.contains(currentUnManagedVolume.getNativeGuid())) {
foundIngestedReplicas.add(currentBlockObject);
}
getFoundIngestedReplicaURIs(foundIngestedReplicas, foundIngestedReplicaNativeGuids);
_logger.info("Expected replicas : {} -->Found replica URIs : {}", expectedIngestedReplicas.size(), foundIngestedReplicaNativeGuids.size());
_logger.info("Expected replicas {} : Found {} : ", Joiner.on(", ").join(expectedIngestedReplicas), Joiner.on(", ").join(foundIngestedReplicaNativeGuids));
if (foundIngestedReplicas.size() == expectedIngestedReplicas.size()) {
if (null != rootBlockObject && !foundIngestedReplicas.isEmpty()) {
parentReplicaMap.put(rootBlockObject, foundIngestedReplicas);
unManagedVolumeGUIDs.add(unManagedVolumeNativeGUID);
unManagedVolumeGUIDs.addAll(unmanagedReplicaGUIDs);
traverseTree = true;
}
} else {
Set<String> unIngestedReplicas = VolumeIngestionUtil.getUnIngestedReplicas(expectedIngestedReplicas, foundIngestedReplicas);
_logger.info("The replicas {} not ingested for volume {}", Joiner.on(", ").join(unIngestedReplicas), unManagedVolumeNativeGUID);
StringBuffer taskStatus = requestContext.getTaskStatusMap().get(currentUnManagedVolume.getNativeGuid());
if (taskStatus == null) {
taskStatus = new StringBuffer();
requestContext.getTaskStatusMap().put(currentUnManagedVolume.getNativeGuid(), taskStatus);
}
// volume guids in the list returned to the user
if (vplexBackendVolumeGUIDs != null) {
_logger.info("removing the subset of vplex backend volume GUIDs from the error message: " + vplexBackendVolumeGUIDs);
// have to convert this because getUningestedReplicas returns an immutable set
Set<String> mutableSet = new HashSet<String>();
mutableSet.addAll(unIngestedReplicas);
mutableSet.removeAll(vplexBackendVolumeGUIDs);
unIngestedReplicas = mutableSet;
}
taskStatus.append(String.format("The umanaged volume %s has been partially ingested, but not all replicas " + "have been ingested. Uningested replicas: %s.", currentUnManagedVolume.getLabel(), Joiner.on(", ").join(unIngestedReplicas)));
// clear the map and stop traversing
parentReplicaMap.clear();
traverseTree = false;
}
if (traverseTree && !unmanagedReplicaGUIDs.isEmpty()) {
// get all the unmanaged volume replicas and traverse through them
List<UnManagedVolume> replicaUnManagedVolumes = _dbClient.queryObject(UnManagedVolume.class, VolumeIngestionUtil.getUnManagedVolumeUris(unmanagedReplicaGUIDs, _dbClient));
for (UnManagedVolume replica : replicaUnManagedVolumes) {
BlockObject replicaBlockObject = null;
if (replica.getNativeGuid().equals(currentUnManagedVolume.getNativeGuid())) {
replicaBlockObject = currentBlockObject;
} else {
_logger.info("Checking for replica object in created object map");
String replicaGUID = replica.getNativeGuid().replace(VolumeIngestionUtil.UNMANAGEDVOLUME, VolumeIngestionUtil.VOLUME);
replicaBlockObject = requestContext.getRootIngestionRequestContext().findCreatedBlockObject(replicaGUID);
if (replicaBlockObject == null) {
_logger.info("Checking if the replica is ingested");
replicaBlockObject = VolumeIngestionUtil.getBlockObject(replicaGUID, _dbClient);
}
}
runReplicasIngestedCheck(replica, replicaBlockObject, currentUnManagedVolume, currentBlockObject, unManagedVolumeGUIDs, parentReplicaMap, requestContext);
// TODO- break out if the parent-replica map is empty
}
}
}
use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedVolume in project coprhd-controller by CoprHD.
the class BlockRecoverPointIngestOrchestrator method decorateVolumeInformationFinalIngest.
/**
* This method will perform all of the final decorations (attribute setting) on the Volume
* object after creating the required BlockConsistencyGroup and ProtectionSet objects.
*
* Fields such as rpCopyName and rSetName were already filled in when we did the ingest of
* the volume itself. In this method, we worry about stitching together all of the object
* references within the Volume object so it will act like a native CoprHD-created RP volume.
*
* @param volumeContext the RecoverPointVolumeIngestionContext for the volume currently being ingested
* @param unManagedVolume the currently ingesting UnManagedVolume
*/
private void decorateVolumeInformationFinalIngest(IngestionRequestContext requestContext, UnManagedVolume unManagedVolume) {
RecoverPointVolumeIngestionContext volumeContext = (RecoverPointVolumeIngestionContext) requestContext.getVolumeContext();
ProtectionSet pset = volumeContext.getManagedProtectionSet();
BlockConsistencyGroup cg = volumeContext.getManagedBlockConsistencyGroup();
if (pset.getVolumes() == null) {
_logger.error("No volumes found in protection set: " + pset.getLabel() + ", cannot process ingestion");
throw IngestionException.exceptions.noVolumesFoundInProtectionSet(pset.getLabel());
}
List<Volume> volumes = new ArrayList<Volume>();
for (String volId : pset.getVolumes()) {
BlockObject volume = requestContext.getRootIngestionRequestContext().findCreatedBlockObject(URI.create(volId));
if (volume != null && volume instanceof Volume) {
volumes.add((Volume) volume);
}
}
// Make sure all of the changed managed block objects get updated
volumes.add((Volume) volumeContext.getManagedBlockObject());
Set<DataObject> updatedObjects = new HashSet<DataObject>();
VolumeIngestionUtil.decorateRPVolumesCGInfo(volumes, pset, cg, updatedObjects, _dbClient, requestContext);
VolumeIngestionUtil.clearPersistedReplicaFlags(requestContext, volumes, updatedObjects, _dbClient);
clearReplicaFlagsInIngestionContext(volumeContext, volumes);
for (DataObject volume : updatedObjects) {
if (volumeContext.getManagedBlockObject().getId().equals(volume.getId()) && (null == _dbClient.queryObject(Volume.class, volume.getId()))) {
// this is the managed block object and it hasn't been saved to the db yet
continue;
} else {
// add all volumes except the newly ingested one to the update list
volumeContext.addDataObjectToUpdate(volume, unManagedVolume);
}
}
}
use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedVolume in project coprhd-controller by CoprHD.
the class BlockRecoverPointIngestOrchestrator method performRPExportIngestion.
/**
* RecoverPoint volumes are expected to have export masks where the volume is exported to
* a RecoverPoint site. Therefore every RP volume (sources, targets, journals) will need to
* go through this code and have their export mask ingested. Even if the mask has already been
* ingested by a previous volume ingestion, this method still needs to update the ExportGroup and
* ExportMask objects to reflect the newly ingested volume as part of its management.
*
* @param volumeContext the RecoverPointVolumeIngestionContext for the volume currently being ingested
* @param unManagedVolume unmanaged volume
* @param volume managed volume
* @return managed volume with export ingested
*/
private void performRPExportIngestion(IngestionRequestContext parentRequestContext, RecoverPointVolumeIngestionContext volumeContext, UnManagedVolume unManagedVolume, Volume volume) {
_logger.info("starting RecoverPoint export ingestion for volume {}", volume.forDisplay());
Project project = volumeContext.getProject();
ProtectionSystem protectionSystem = _dbClient.queryObject(ProtectionSystem.class, volume.getProtectionController());
StorageSystem storageSystem = _dbClient.queryObject(StorageSystem.class, volume.getStorageController());
List<UnManagedExportMask> unManagedRPExportMasks = findUnManagedRPExportMask(protectionSystem, unManagedVolume);
if (unManagedRPExportMasks.isEmpty()) {
_logger.error("Could not find any unmanaged export masks associated with volume: " + unManagedVolume.getLabel());
throw IngestionException.exceptions.noUnManagedExportMaskFound(unManagedVolume.getNativeGuid());
}
// Keep a map for internal site name name and varray
Map<String, VirtualArray> internalSiteToVarrayMap = new HashMap<String, VirtualArray>();
internalSiteToVarrayMap.put(volume.getInternalSiteName(), volumeContext.getVarray(unManagedVolume));
// If this is a MetroPoint volume we're going to have multiple ExportMasks/ExportGroups to deal with.
// We'll need to query the backend volumes for extra info to populate internalSiteToVarrayMap so
// we can properly line up the ExportMasks/ExportGroups.
boolean metropoint = RPHelper.isMetroPointVolume(_dbClient, volume);
if (metropoint) {
// We need the VPLEX ingest context to get the backend volume info
VplexVolumeIngestionContext vplexVolumeContext = ((RpVplexVolumeIngestionContext) volumeContext.getVolumeContext()).getVplexVolumeIngestionContext();
for (String associatedVolumeIdStr : vplexVolumeContext.getAssociatedVolumeIds(volume)) {
// Find the associated volumes using the context maps or the db if they are already there
Volume associatedVolume = VolumeIngestionUtil.findVolume(_dbClient, vplexVolumeContext.getBlockObjectsToBeCreatedMap(), vplexVolumeContext.getDataObjectsToBeUpdatedMap(), associatedVolumeIdStr);
String internalSiteName = associatedVolume.getInternalSiteName();
// If we don't already have an entry for this internal site name, let's add it now.
if (!internalSiteToVarrayMap.containsKey(internalSiteName)) {
internalSiteToVarrayMap.put(internalSiteName, _dbClient.queryObject(VirtualArray.class, associatedVolume.getVirtualArray()));
}
}
}
// this will more than likely only loop once.
for (Entry<String, VirtualArray> entry : internalSiteToVarrayMap.entrySet()) {
String internalSiteName = entry.getKey();
VirtualArray virtualArray = entry.getValue();
UnManagedExportMask em = null;
if (metropoint) {
// Since we're flagged for MetroPoint we need to determine which ExportMask to use.
// We need the MetroPoint volume to be added to BOTH ExportGroups that represent the
// two Storage Views on VPLEX for cluster-1 and cluster-2.
// So let's use the varray to find the cluster we're looking for on this pass and match
// it to the maskingViewParth of the UnManagedExportMask.
// This should line things up roughly as:
// VPLEX Storage View 1 -> VPLEX Cluster1 + RPA1
// VPLEX Storage View 2 -> VPLEX Cluster2 + RPA2
String vplexCluster = ConnectivityUtil.getVplexClusterForVarray(virtualArray.getId(), storageSystem.getId(), _dbClient);
// First try and match based on UnManagedExportMask ports
for (UnManagedExportMask exportMask : unManagedRPExportMasks) {
for (String portUri : exportMask.getKnownStoragePortUris()) {
StoragePort port = _dbClient.queryObject(StoragePort.class, URI.create(portUri));
if (port != null && !port.getInactive()) {
String vplexClusterForMask = ConnectivityUtil.getVplexClusterOfPort(port);
if (vplexCluster.equals(vplexClusterForMask)) {
em = exportMask;
break;
}
}
}
if (em != null) {
break;
}
}
if (em == null) {
// It really shouldn't come to this, but leaving this code just in case.
for (UnManagedExportMask exportMask : unManagedRPExportMasks) {
if (exportMask.getMaskingViewPath().contains("cluster-" + vplexCluster)) {
em = exportMask;
break;
}
}
}
} else {
em = unManagedRPExportMasks.get(0);
}
// If the mask for ingested volume is in a mask that contains JOURNAL keyword, make sure the ExportGroup created contains
// that internal flag.
boolean isJournalExport = false;
if (em.getMaskName().toLowerCase().contains(VolumeIngestionUtil.RP_JOURNAL)) {
isJournalExport = true;
}
String exportGroupGeneratedName = RPHelper.generateExportGroupName(protectionSystem, storageSystem, internalSiteName, virtualArray, isJournalExport);
ExportGroup exportGroup = VolumeIngestionUtil.verifyExportGroupExists(parentRequestContext, exportGroupGeneratedName, project.getId(), em.getKnownInitiatorUris(), virtualArray.getId(), _dbClient);
boolean exportGroupCreated = false;
if (null == exportGroup) {
exportGroupCreated = true;
Integer numPaths = em.getZoningMap().size();
_logger.info("Creating Export Group with label {}", em.getMaskName());
exportGroup = RPHelper.createRPExportGroup(exportGroupGeneratedName, virtualArray, project, numPaths, isJournalExport);
}
if (null != exportGroup) {
// check if the ExportGroup has already been fetched
ExportGroup loadedExportGroup = parentRequestContext.findExportGroup(exportGroup.getLabel(), project.getId(), virtualArray.getId(), null, null);
if (null != loadedExportGroup) {
exportGroup = loadedExportGroup;
}
}
volumeContext.setExportGroup(exportGroup);
volumeContext.setExportGroupCreated(exportGroupCreated);
volumeContext.getRpExportGroupMap().put(exportGroup, exportGroupCreated);
// set RP device initiators to be used as the "host" for export mask ingestion
List<Initiator> initiators = new ArrayList<Initiator>();
Iterator<Initiator> initiatorItr = _dbClient.queryIterativeObjects(Initiator.class, URIUtil.toURIList(em.getKnownInitiatorUris()));
while (initiatorItr.hasNext()) {
initiators.add(initiatorItr.next());
}
volumeContext.setDeviceInitiators(initiators);
// find the ingest export strategy and call into for this unmanaged export mask
IngestExportStrategy ingestStrategy = ingestStrategyFactory.buildIngestExportStrategy(unManagedVolume);
volume = ingestStrategy.ingestExportMasks(unManagedVolume, volume, volumeContext);
if (null == volume) {
// ingestion did not succeed, but in case it wasn't, throw one
throw IngestionException.exceptions.generalVolumeException(unManagedVolume.getLabel(), "check the logs for more details");
}
}
}
use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedVolume in project coprhd-controller by CoprHD.
the class BlockRemoteReplicationIngestOrchestrator method validateSourceVolumeVarrayWithTargetVPool.
/**
* Validate the SourceVolume VArray details with ingested target volumes
* VArray.
*
* @param unManagedVolume
* @param VirtualPool
* @return
*/
private void validateSourceVolumeVarrayWithTargetVPool(UnManagedVolume unManagedVolume, VirtualPool sourceVPool) {
StringSetMap unManagedVolumeInformation = unManagedVolume.getVolumeInformation();
// find whether all targets are ingested
StringSet targetUnManagedVolumeGuids = unManagedVolumeInformation.get(SupportedVolumeInformation.REMOTE_MIRRORS.toString());
if (null != targetUnManagedVolumeGuids && !targetUnManagedVolumeGuids.isEmpty()) {
StringSet targetVolumeNativeGuids = VolumeIngestionUtil.getListofVolumeIds(targetUnManagedVolumeGuids);
// check whether target exists
List<URI> targetUris = VolumeIngestionUtil.getVolumeUris(targetVolumeNativeGuids, _dbClient);
if (null == targetUris || targetUris.isEmpty()) {
_logger.info("None of the targets ingested for source volume: {}", unManagedVolume.getNativeGuid());
} else {
List<Volume> targetVolumes = _dbClient.queryObject(Volume.class, targetUris);
for (Volume targetVolume : targetVolumes) {
Map<URI, VpoolRemoteCopyProtectionSettings> settings = sourceVPool.getRemoteProtectionSettings(sourceVPool, _dbClient);
if (null == settings || settings.size() == 0 || !settings.containsKey(targetVolume.getVirtualArray())) {
_logger.info("Target Volume's VArray {} is not matching already ingested source volume virtual pool's remote VArray {}", targetVolume.getVirtualArray(), Joiner.on(",").join(settings.keySet()));
throw IngestionException.exceptions.unmanagedSRDFSourceVolumeVArrayMismatch(unManagedVolume.getLabel(), targetVolume.getVirtualArray().toString());
}
}
}
}
}
use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedVolume in project coprhd-controller by CoprHD.
the class BlockSnapIngestOrchestrator method ingestBlockObjects.
@Override
public <T extends BlockObject> T ingestBlockObjects(IngestionRequestContext requestContext, Class<T> clazz) throws IngestionException {
UnManagedVolume unManagedVolume = requestContext.getCurrentUnmanagedVolume();
boolean unManagedVolumeExported = requestContext.getVolumeContext().isVolumeExported();
BlockSnapshot snapShot = null;
String snapNativeGuid = unManagedVolume.getNativeGuid().replace(NativeGUIDGenerator.UN_MANAGED_VOLUME, NativeGUIDGenerator.VOLUME);
snapShot = VolumeIngestionUtil.checkSnapShotExistsInDB(snapNativeGuid, _dbClient);
// Check if ingested volume has exportmasks pending for ingestion.
if (isExportIngestionPending(snapShot, unManagedVolume.getId(), unManagedVolumeExported)) {
return clazz.cast(snapShot);
}
if (null == snapShot) {
// @TODO Need to revisit this. In 8.x provider, Replication Group is
// automatically created when a volume is associated to a
// StorageGroup.
// checkUnManagedVolumeAddedToCG(unManagedVolume, virtualArray,
// tenant, project, vPool);
checkVolumeExportState(unManagedVolume, unManagedVolumeExported);
VolumeIngestionUtil.checkUnManagedResourceIngestable(unManagedVolume);
snapShot = createSnapshot(requestContext, snapNativeGuid);
// See if this is a linked target for existing block snapshot sessions.
if (!NullColumnValueGetter.isNullValue(snapShot.getSettingsInstance())) {
URIQueryResultList queryResults = new URIQueryResultList();
_dbClient.queryByConstraint(AlternateIdConstraint.Factory.getBlockSnapshotSessionBySessionInstance(snapShot.getSettingsInstance()), queryResults);
Iterator<URI> queryResultsIter = queryResults.iterator();
while (queryResultsIter.hasNext()) {
BlockSnapshotSession snapSession = _dbClient.queryObject(BlockSnapshotSession.class, queryResultsIter.next());
StringSet linkedTargets = snapSession.getLinkedTargets();
if ((linkedTargets != null)) {
linkedTargets.add(snapShot.getId().toString());
} else {
linkedTargets = new StringSet();
linkedTargets.add(snapShot.getId().toString());
snapSession.setLinkedTargets(linkedTargets);
}
_dbClient.updateObject(snapSession);
}
}
}
// Run this logic always when Volume is NO_PUBLIC_ACCESS
if (markUnManagedVolumeInactive(requestContext, snapShot)) {
_logger.info("All the related replicas and parent of unManagedVolume {} has been ingested ", unManagedVolume.getNativeGuid());
// mark inactive if this is not to be exported. Else, mark as inactive after successful export
if (!unManagedVolumeExported) {
unManagedVolume.setInactive(true);
requestContext.getUnManagedVolumesToBeDeleted().add(unManagedVolume);
}
} else {
_logger.info("Not all the parent/replicas of unManagedVolume {} have been ingested , hence marking as internal", unManagedVolume.getNativeGuid());
snapShot.addInternalFlags(INTERNAL_VOLUME_FLAGS);
}
return clazz.cast(snapShot);
}
Aggregations