Examples with BlockConsistencyGroup com.emc.storageos.db.client.model.BlockConsistencyGroup used on opensource projects

Example 71 with BlockConsistencyGroup

use of com.emc.storageos.db.client.model.BlockConsistencyGroup in project coprhd-controller by CoprHD.

the class ExternalBlockStorageDevice method doCreateGroupClone.

@Override
public void doCreateGroupClone(StorageSystem storageSystem, List<URI> cloneURIs, Boolean createInactive, TaskCompleter taskCompleter) {
    BlockStorageDriver driver = getDriver(storageSystem.getSystemType());
    List<VolumeClone> driverClones = new ArrayList<>();
    Map<VolumeClone, Volume> driverCloneToCloneMap = new HashMap<>();
    Set<URI> consistencyGroups = new HashSet<>();
    List<Volume> clones = null;
    DriverTask task = null;
    try {
        clones = dbClient.queryObject(Volume.class, cloneURIs);
        // We assume here that all volumes belong to the same consistency group
        URI parentUri = clones.get(0).getAssociatedSourceVolume();
        Volume parentVolume = dbClient.queryObject(Volume.class, parentUri);
        BlockConsistencyGroup cg = null;
        if (!NullColumnValueGetter.isNullURI(parentVolume.getConsistencyGroup())) {
            cg = dbClient.queryObject(BlockConsistencyGroup.class, parentVolume.getConsistencyGroup());
        } else {
            String errorMsg = String.format("doCreateGroupClone -- Failed to create group clone, parent volumes do not belong to consistency group." + " Clones: %s .", cloneURIs);
            _log.error(errorMsg);
            ServiceError serviceError = ExternalDeviceException.errors.createGroupCloneFailed("doCreateGroupClone", errorMsg);
            taskCompleter.error(dbClient, serviceError);
            return;
        }
        // Prepare driver consistency group of parent volume
        VolumeConsistencyGroup driverCG = new VolumeConsistencyGroup();
        driverCG.setDisplayName(cg.getLabel());
        driverCG.setNativeId(cg.getNativeId());
        driverCG.setStorageSystemId(storageSystem.getNativeId());
        // Prepare driver clones
        for (Volume clone : clones) {
            URI sourceVolumeUri = clone.getAssociatedSourceVolume();
            Volume sourceVolume = dbClient.queryObject(Volume.class, sourceVolumeUri);
            VolumeClone driverClone = new VolumeClone();
            driverClone.setParentId(sourceVolume.getNativeId());
            driverClone.setStorageSystemId(storageSystem.getNativeId());
            driverClone.setDisplayName(clone.getLabel());
            driverClone.setRequestedCapacity(clone.getCapacity());
            driverClone.setThinlyProvisioned(clone.getThinlyProvisioned());
            driverClones.add(driverClone);
            driverCloneToCloneMap.put(driverClone, clone);
        }
        // Call driver to create group snapshot
        task = driver.createConsistencyGroupClone(driverCG, Collections.unmodifiableList(driverClones), null);
        if (!isTaskInTerminalState(task.getStatus())) {
            // If the task is not in a terminal state and will be completed asynchronously
            // create a job to monitor the progress of the request and update the clone
            // volume and call the completer as appropriate based on the result of the request.
            CreateGroupCloneExternalDeviceJob job = new CreateGroupCloneExternalDeviceJob(storageSystem.getId(), cloneURIs, parentVolume.getConsistencyGroup(), task.getTaskId(), taskCompleter);
            ControllerServiceImpl.enqueueJob(new QueueJob(job));
        } else if (task.getStatus() == DriverTask.TaskStatus.READY) {
            // Update clone object with driver data
            String msg = String.format("doCreateGroupClone -- Created group clone: %s .", task.getMessage());
            _log.info(msg);
            List<Volume> cloneObjects = new ArrayList<>();
            for (VolumeClone driverCloneResult : driverClones) {
                Volume cloneObject = driverCloneToCloneMap.get(driverCloneResult);
                ExternalDeviceUtils.updateNewlyCreatedGroupClone(cloneObject, driverCloneResult, parentVolume.getConsistencyGroup(), dbClient);
                cloneObjects.add(cloneObject);
            }
            dbClient.updateObject(cloneObjects);
            taskCompleter.ready(dbClient);
        } else {
            // Process failure
            for (Volume cloneObject : clones) {
                cloneObject.setInactive(true);
            }
            dbClient.updateObject(clones);
            String errorMsg = String.format("doCreateGroupClone -- Failed to create group clone: %s .", task.getMessage());
            _log.error(errorMsg);
            ServiceError serviceError = ExternalDeviceException.errors.createGroupCloneFailed("doCreateGroupClone", errorMsg);
            taskCompleter.error(dbClient, serviceError);
        }
    } catch (Exception e) {
        if (clones != null) {
            // Process failure
            for (Volume cloneObject : clones) {
                cloneObject.setInactive(true);
            }
            dbClient.updateObject(clones);
        }
        _log.error("Failed to create group clone. ", e);
        ServiceError serviceError = ExternalDeviceException.errors.createGroupCloneFailed("doCreateGroupClone", e.getMessage());
        taskCompleter.error(dbClient, serviceError);
    } finally {
        try {
            if (task == null || isTaskInTerminalState(task.getStatus())) {
                // post process storage pool capacity for clone's pools
                // map clones to their storage pool
                Map<URI, List<URI>> dbPoolToClone = new HashMap<>();
                for (Volume clone : clones) {
                    URI dbPoolUri = clone.getPool();
                    List<URI> poolClones = dbPoolToClone.get(dbPoolUri);
                    if (poolClones == null) {
                        poolClones = new ArrayList<>();
                        dbPoolToClone.put(dbPoolUri, poolClones);
                    }
                    poolClones.add(clone.getId());
                }
                for (URI dbPoolUri : dbPoolToClone.keySet()) {
                    StoragePool dbPool = dbClient.queryObject(StoragePool.class, dbPoolUri);
                    updateStoragePoolCapacity(dbPool, storageSystem, dbPoolToClone.get(dbPoolUri), dbClient);
                }
            }
        } catch (Exception ex) {
            _log.error("Failed to update storage pool after create group clone operation completion.", ex);
        }
    }
}

Example 72 with BlockConsistencyGroup

use of com.emc.storageos.db.client.model.BlockConsistencyGroup in project coprhd-controller by CoprHD.

the class ReplicaDeviceController method addStepsForCreateVolumes.

@Override
public String addStepsForCreateVolumes(Workflow workflow, String waitFor, List<VolumeDescriptor> volumes, String taskId) throws InternalException {
    // Get the list of descriptors which represent source volumes that have
    // just been created and added to CG possibly
    List<VolumeDescriptor> volumeDescriptors = VolumeDescriptor.filterByType(volumes, new VolumeDescriptor.Type[] { VolumeDescriptor.Type.BLOCK_DATA, VolumeDescriptor.Type.SRDF_SOURCE, VolumeDescriptor.Type.SRDF_EXISTING_SOURCE }, null);
    // If no source volumes, just return
    if (volumeDescriptors.isEmpty()) {
        log.info("No replica steps required");
        return waitFor;
    }
    // Get the consistency group. If no consistency group for
    // any source volumes, just return. Get CG from any descriptor.
    URI cgURI = null;
    for (VolumeDescriptor descriptor : volumeDescriptors) {
        Volume volume = _dbClient.queryObject(Volume.class, descriptor.getVolumeURI());
        if (volume == null || !volume.isInCG() || !(ControllerUtils.isVmaxVolumeUsing803SMIS(volume, _dbClient) || ControllerUtils.isNotInRealVNXRG(volume, _dbClient))) {
            log.info("No replica steps required for volume: " + descriptor.getVolumeURI());
            continue;
        }
        log.info("CG URI:{}", volume.getConsistencyGroup());
        cgURI = volume.getConsistencyGroup();
    }
    // Reason:Provisioning new volumes for VPLEX/RP CG in Application does not add backend volume to RG
    if (!NullColumnValueGetter.isNullURI(cgURI)) {
        BlockConsistencyGroup cg = _dbClient.queryObject(BlockConsistencyGroup.class, cgURI);
        if (!cg.getArrayConsistency() && isBackendVolumeForVplexOrRp(volumes)) {
            log.info("No replica steps required for CG {} as array consistency is disabled.", cg.getLabel());
            return waitFor;
        }
    } else {
        log.info("No replica steps required because no volumes had CG");
        return waitFor;
    }
    List<VolumeDescriptor> nonSrdfVolumeDescriptors = VolumeDescriptor.filterByType(volumes, new VolumeDescriptor.Type[] { VolumeDescriptor.Type.BLOCK_DATA }, null);
    List<VolumeDescriptor> srdfSourceVolumeDescriptors = VolumeDescriptor.filterByType(volumes, new VolumeDescriptor.Type[] { VolumeDescriptor.Type.SRDF_SOURCE, VolumeDescriptor.Type.SRDF_EXISTING_SOURCE }, null);
    if (srdfSourceVolumeDescriptors.isEmpty()) {
        waitFor = createReplicaIfCGHasReplica(workflow, waitFor, nonSrdfVolumeDescriptors, cgURI);
    } else {
        // Create Replica for SRDF R1 and R2 if any replica available already
        log.debug("srdfSourceVolumeDescriptors :{}", srdfSourceVolumeDescriptors);
        List<VolumeDescriptor> srdfTargetVolumeDescriptors = VolumeDescriptor.filterByType(volumes, new VolumeDescriptor.Type[] { VolumeDescriptor.Type.SRDF_TARGET }, null);
        log.debug("srdfTargetVolumeDescriptors :{}", srdfTargetVolumeDescriptors);
        // Create replica for R1
        waitFor = createReplicaIfCGHasReplica(workflow, waitFor, srdfSourceVolumeDescriptors, cgURI);
        // Create replica for R2
        URI targetVolumeCG = SRDFUtils.getTargetVolumeCGFromSourceCG(_dbClient, cgURI);
        if (targetVolumeCG != null) {
            waitFor = createReplicaIfCGHasReplica(workflow, waitFor, srdfTargetVolumeDescriptors, targetVolumeCG);
        }
    }
    return waitFor;
}

Example 73 with BlockConsistencyGroup

use of com.emc.storageos.db.client.model.BlockConsistencyGroup in project coprhd-controller by CoprHD.

the class ScaleIOStorageDevice method doDeleteConsistencyGroup.

@Override
public void doDeleteConsistencyGroup(StorageSystem storage, URI consistencyGroup, String replicationGroupName, Boolean keepRGName, Boolean markInactive, TaskCompleter taskCompleter) throws DeviceControllerException {
    log.info("Going to delete BlockConsistency Group {}", consistencyGroup);
    if (markInactive && consistencyGroup != null) {
        BlockConsistencyGroup cg = dbClient.queryObject(BlockConsistencyGroup.class, consistencyGroup);
        if (cg != null) {
            dbClient.markForDeletion(cg);
        }
    }
    taskCompleter.ready(dbClient);
}

Example 74 with BlockConsistencyGroup

use of com.emc.storageos.db.client.model.BlockConsistencyGroup in project coprhd-controller by CoprHD.

the class ScaleIOStorageDevice method updateConsistencyGroupsWithStorageSystem.

private void updateConsistencyGroupsWithStorageSystem(Set<URI> consistencyGroups, StorageSystem storageSystem) {
    List<BlockConsistencyGroup> updateCGs = new ArrayList<>();
    Iterator<BlockConsistencyGroup> consistencyGroupIterator = dbClient.queryIterativeObjects(BlockConsistencyGroup.class, consistencyGroups, true);
    while (consistencyGroupIterator.hasNext()) {
        BlockConsistencyGroup consistencyGroup = consistencyGroupIterator.next();
        consistencyGroup.setStorageController(storageSystem.getId());
        consistencyGroup.addConsistencyGroupTypes(BlockConsistencyGroup.Types.LOCAL.name());
        consistencyGroup.addSystemConsistencyGroup(storageSystem.getId().toString(), consistencyGroup.getLabel());
        updateCGs.add(consistencyGroup);
    }
    dbClient.updateAndReindexObject(updateCGs);
}

Example 75 with BlockConsistencyGroup

use of com.emc.storageos.db.client.model.BlockConsistencyGroup in project coprhd-controller by CoprHD.

the class RecoverPointScheduler method scheduleStorageSourcePoolConstraint.

/**
 * Schedule storage based on the incoming storage pools for source volumes. (New version)
 *
 * @param varray varray requested for source
 * @param protectionVarrays Neighborhood to protect this volume to.
 * @param vpool vpool requested
 * @param capabilities parameters
 * @param candidatePools List of StoragePools already populated to choose from. RP+VPLEX.
 * @param vpoolChangeVolume vpool change volume, if applicable
 * @param preSelectedCandidateProtectionPoolsMap pre-populated map for tgt varray to storage pools, use null if not needed
 * @return list of Recommendation objects to satisfy the request
 */
protected List<Recommendation> scheduleStorageSourcePoolConstraint(VirtualArray varray, List<VirtualArray> protectionVarrays, VirtualPool vpool, VirtualPoolCapabilityValuesWrapper capabilities, List<StoragePool> candidatePools, Project project, Volume vpoolChangeVolume, Map<VirtualArray, List<StoragePool>> preSelectedCandidateProtectionPoolsMap) {
    // Initialize a list of recommendations to be returned.
    List<Recommendation> recommendations = new ArrayList<Recommendation>();
    String candidateSourceInternalSiteName = "";
    placementStatus = new PlacementStatus();
    // Attempt to use these pools for selection based on protection
    StringBuffer sb = new StringBuffer("Determining if protection is possible from " + varray.getId() + " to: ");
    for (VirtualArray protectionVarray : protectionVarrays) {
        sb.append(protectionVarray.getId()).append(" ");
    }
    _log.info(sb.toString());
    // BEGIN: Put the local varray first in the list. We want to give him pick of internal site name.
    int index = -1;
    for (VirtualArray targetVarray : protectionVarrays) {
        if (targetVarray.getId().equals(varray.getId())) {
            index = protectionVarrays.indexOf(targetVarray);
            break;
        }
    }
    if (index > 0) {
        VirtualArray localVarray = protectionVarrays.get(index);
        VirtualArray swapVarray = protectionVarrays.get(0);
        protectionVarrays.set(0, localVarray);
        protectionVarrays.set(index, swapVarray);
    }
    // END: Put the local varray first in the list. We want to give him pick of internal site name.
    List<URI> protectionVarrayURIs = new ArrayList<URI>();
    for (VirtualArray vArray : protectionVarrays) {
        protectionVarrayURIs.add(vArray.getId());
        placementStatus.getProcessedProtectionVArrays().put(vArray.getId(), false);
    }
    // Fetch the list of pools for the source journal if a journal virtual pool is specified to be used for journal volumes.
    VirtualArray journalVarray = varray;
    if (NullColumnValueGetter.isNotNullValue(vpool.getJournalVarray())) {
        journalVarray = dbClient.queryObject(VirtualArray.class, URI.create(vpool.getJournalVarray()));
    }
    VirtualPool journalVpool = vpool;
    if (NullColumnValueGetter.isNotNullValue(vpool.getJournalVpool())) {
        journalVpool = dbClient.queryObject(VirtualPool.class, URI.create(vpool.getJournalVpool()));
    }
    // The attributes below will not change throughout the placement process
    placementStatus.setSrcVArray(varray.getLabel());
    placementStatus.setSrcVPool(vpool.getLabel());
    BlockConsistencyGroup cg = dbClient.queryObject(BlockConsistencyGroup.class, capabilities.getBlockConsistencyGroup());
    int totalRequestedCount = capabilities.getResourceCount();
    int totalSatisfiedCount = 0;
    int requestedCount = totalRequestedCount;
    int satisfiedCount = 0;
    boolean isChangeVpool = (vpoolChangeVolume != null);
    RPProtectionRecommendation rpProtectionRecommendation = new RPProtectionRecommendation();
    rpProtectionRecommendation.setVpoolChangeVolume(vpoolChangeVolume != null ? vpoolChangeVolume.getId() : null);
    rpProtectionRecommendation.setVpoolChangeNewVpool(vpoolChangeVolume != null ? vpool.getId() : null);
    rpProtectionRecommendation.setVpoolChangeProtectionAlreadyExists(vpoolChangeVolume != null ? vpoolChangeVolume.checkForRp() : false);
    List<Recommendation> sourcePoolRecommendations = new ArrayList<Recommendation>();
    if (isChangeVpool) {
        Recommendation changeVpoolSourceRecommendation = new Recommendation();
        URI existingStoragePoolId = null;
        // valid source pool, the existing one. Get that pool and add it to the list.
        if (RPHelper.isVPlexVolume(vpoolChangeVolume, dbClient)) {
            if (null == vpoolChangeVolume.getAssociatedVolumes() || vpoolChangeVolume.getAssociatedVolumes().isEmpty()) {
                _log.error("VPLEX volume {} has no backend volumes.", vpoolChangeVolume.forDisplay());
                throw InternalServerErrorException.internalServerErrors.noAssociatedVolumesForVPLEXVolume(vpoolChangeVolume.forDisplay());
            }
            for (String associatedVolume : vpoolChangeVolume.getAssociatedVolumes()) {
                Volume assocVol = dbClient.queryObject(Volume.class, URI.create(associatedVolume));
                if (assocVol.getVirtualArray().equals(varray.getId())) {
                    existingStoragePoolId = assocVol.getPool();
                    break;
                }
            }
        } else {
            existingStoragePoolId = vpoolChangeVolume.getPool();
        }
        // This is the existing active source backing volume
        changeVpoolSourceRecommendation.setSourceStoragePool(existingStoragePoolId);
        StoragePool pool = dbClient.queryObject(StoragePool.class, existingStoragePoolId);
        changeVpoolSourceRecommendation.setSourceStorageSystem(pool.getStorageDevice());
        changeVpoolSourceRecommendation.setResourceCount(1);
        sourcePoolRecommendations.add(changeVpoolSourceRecommendation);
        _log.info(String.format("RP Placement : Change Virtual Pool - Active source pool already exists, reuse pool: [%s] [%s].", pool.getLabel().toString(), pool.getId().toString()));
    } else {
        // Recommendation analysis:
        // Each recommendation returned will indicate the number of resources of specified size that it can accommodate in ascending order.
        // Go through each recommendation, map to storage system from the recommendation to find connectivity
        // If we get through the process and couldn't achieve full protection, we should try with the next pool in the list until
        // we either find a successful solution or failure.
        sourcePoolRecommendations = getRecommendedPools(rpProtectionRecommendation, varray, vpool, null, null, capabilities, RPHelper.SOURCE, null);
        if (sourcePoolRecommendations == null || sourcePoolRecommendations.isEmpty()) {
            _log.error(String.format("RP Placement : No matching storage pools found for the source varray: [%s]. " + "There are no storage pools that " + "match the passed vpool parameters and protocols and/or there are " + "no pools that have enough capacity to hold at least one resource of the requested size.", varray.getLabel()));
            throw APIException.badRequests.noMatchingStoragePoolsForVpoolAndVarray(vpool.getLabel(), varray.getLabel());
        }
    }
    for (Recommendation sourcePoolRecommendation : sourcePoolRecommendations) {
        satisfiedCount = ((sourcePoolRecommendation.getResourceCount()) >= requestedCount) ? requestedCount : sourcePoolRecommendation.getResourceCount();
        _log.info("Looking to place " + satisfiedCount + " resources...");
        // Start with the top of the list of source pools, find a solution based on that.
        // Given the candidatePools.get(0), what protection systems and internal sites protect it?
        Set<ProtectionSystem> protectionSystems = new HashSet<ProtectionSystem>();
        ProtectionSystem cgProtectionSystem = getCgProtectionSystem(capabilities.getBlockConsistencyGroup());
        StoragePool sourcePool = dbClient.queryObject(StoragePool.class, sourcePoolRecommendation.getSourceStoragePool());
        // used by other volumes in it.
        if (cgProtectionSystem != null) {
            _log.info(String.format("RP Placement : Narrowing down placement to use ProtectionSystem %s, " + "which is currently used by RecoverPoint consistency group %s.", cgProtectionSystem.getLabel(), cg));
            protectionSystems.add(cgProtectionSystem);
        } else {
            protectionSystems = getProtectionSystemsForStoragePool(sourcePool, varray, VirtualPool.vPoolSpecifiesHighAvailability(vpool));
            // Verify that the candidate pool can be protected
            if (protectionSystems.isEmpty()) {
                continue;
            }
        }
        // Sort the ProtectionSystems based on the last time a CG was created. Always use the
        // ProtectionSystem with the oldest cgLastCreated timestamp to support a round-robin
        // style of load balancing.
        List<ProtectionSystem> protectionSystemsLst = sortProtectionSystems(protectionSystems);
        for (ProtectionSystem candidateProtectionSystem : protectionSystemsLst) {
            Calendar cgLastCreated = candidateProtectionSystem.getCgLastCreatedTime();
            _log.info(String.format("RP Placement : Attempting to use ProtectionSystem %s, which was last used to create a CG on %s.", candidateProtectionSystem.getLabel(), cgLastCreated != null ? cgLastCreated.getTime().toString() : "N/A"));
            List<String> associatedStorageSystems = new ArrayList<String>();
            String internalSiteNameandAssocStorageSystem = getCgSourceInternalSiteNameAndAssociatedStorageSystem(capabilities.getBlockConsistencyGroup());
            // source internal site.
            if (internalSiteNameandAssocStorageSystem != null) {
                _log.info(String.format("RP Placement : Narrowing down placement to use internal site %s for source, " + "which is currently used by RecoverPoint consistency group %s.", internalSiteNameandAssocStorageSystem, cg));
                associatedStorageSystems.add(internalSiteNameandAssocStorageSystem);
            } else {
                associatedStorageSystems = getCandidateVisibleStorageSystems(sourcePool, candidateProtectionSystem, varray, protectionVarrays, VirtualPool.vPoolSpecifiesHighAvailability(vpool));
            }
            // make sure you check RP topology to see if the sites can protect that many targets
            if (associatedStorageSystems.isEmpty()) {
                // no rp site clusters connected to this storage system, should not hit this, but just to be safe we'll catch it
                _log.info(String.format("RP Placement: Protection System %s does not have an RP internal site connected to Storage pool %s ", candidateProtectionSystem.getLabel(), sourcePool.getLabel()));
                continue;
            }
            for (String associatedStorageSystem : associatedStorageSystems) {
                _log.info(String.format("RP Placement : Attempting to find solution using StorageSystem : %s for RP source", associatedStorageSystem));
                rpProtectionRecommendation.setProtectionDevice(candidateProtectionSystem.getId());
                _log.info(String.format("RP Placement : Build RP Source Recommendation..."));
                RPRecommendation rpSourceRecommendation = buildSourceRecommendation(associatedStorageSystem, varray, vpool, candidateProtectionSystem, sourcePool, capabilities, satisfiedCount, placementStatus, vpoolChangeVolume, false);
                if (rpSourceRecommendation == null) {
                    // No placement found for the associatedStorageSystem, so continue.
                    _log.warn(String.format("RP Placement : Could not create Source Recommendation using [%s], continuing...", associatedStorageSystem));
                    continue;
                }
                candidateSourceInternalSiteName = rpSourceRecommendation.getInternalSiteName();
                String siteName = candidateProtectionSystem.getRpSiteNames().get(candidateSourceInternalSiteName);
                _log.info(String.format("RP Placement : Choosing RP internal site %s %s for source", siteName, candidateSourceInternalSiteName));
                // Build the HA recommendation if HA is specified
                VirtualPoolCapabilityValuesWrapper haCapabilities = new VirtualPoolCapabilityValuesWrapper(capabilities);
                haCapabilities.put(VirtualPoolCapabilityValuesWrapper.RESOURCE_COUNT, satisfiedCount);
                RPRecommendation haRecommendation = this.getHaRecommendation(varray, vpool, project, haCapabilities);
                if (haRecommendation != null) {
                    rpSourceRecommendation.setHaRecommendation(haRecommendation);
                }
                // Build Source Journal Recommendation
                RPRecommendation sourceJournalRecommendation = null;
                if (rpProtectionRecommendation.getSourceJournalRecommendation() == null) {
                    _log.info(String.format("RP Placement : Build RP Source Journal Recommendation..."));
                    sourceJournalRecommendation = buildJournalRecommendation(rpProtectionRecommendation, candidateSourceInternalSiteName, vpool.getJournalSize(), journalVarray, journalVpool, candidateProtectionSystem, capabilities, totalRequestedCount, vpoolChangeVolume, false);
                    if (sourceJournalRecommendation == null) {
                        _log.warn(String.format("RP Placement : Could not create Source Journal Recommendation using [%s], continuing...", associatedStorageSystem));
                        continue;
                    }
                }
                rpProtectionRecommendation.getSourceRecommendations().add(rpSourceRecommendation);
                rpProtectionRecommendation.setSourceJournalRecommendation(sourceJournalRecommendation);
                // If we made it this far we know that our source virtual pool and associated source virtual array
                // has a storage pool with enough capacity for the requested resources and which is accessible to an rp
                // cluster site
                rpProtectionRecommendation.setPlacementStepsCompleted(PlacementProgress.IDENTIFIED_SOLUTION_FOR_SOURCE);
                if (placementStatus.isBestSolutionToDate(rpProtectionRecommendation)) {
                    placementStatus.setLatestInvalidRecommendation(rpProtectionRecommendation);
                }
                // TODO Joe: need this when we are creating multiple recommendations
                placementStatus.setLatestInvalidRecommendation(null);
                // Find a solution, given this vpool, and the target varrays
                if (findSolution(rpProtectionRecommendation, rpSourceRecommendation, varray, vpool, protectionVarrays, capabilities, satisfiedCount, false, null, project)) {
                    // Found Source, Source Journal, Target, Target Journals...we're good to go.
                    totalSatisfiedCount += satisfiedCount;
                    requestedCount = requestedCount - totalSatisfiedCount;
                    if ((totalSatisfiedCount >= totalRequestedCount)) {
                        // Check to ensure the protection system can handle the new resources about to come down
                        if (!verifyPlacement(candidateProtectionSystem, rpProtectionRecommendation, rpProtectionRecommendation.getResourceCount())) {
                            // Did not pass placement verification, back out and try again...
                            rpProtectionRecommendation.getSourceRecommendations().remove(rpSourceRecommendation);
                            rpProtectionRecommendation.setSourceJournalRecommendation(null);
                            _log.warn(String.format("RP Placement : Placement could not be verified with " + "current resources, trying placement again...", associatedStorageSystem));
                            continue;
                        }
                        rpProtectionRecommendation.setResourceCount(totalSatisfiedCount);
                        recommendations.add(rpProtectionRecommendation);
                        return recommendations;
                    } else {
                        break;
                    }
                } else {
                    // Not sure there's anything to do here. Just go to the next candidate protection system or Protection System
                    _log.info(String.format("RP Placement : Could not find a solution against ProtectionSystem %s " + "and internal site %s", candidateProtectionSystem.getLabel(), candidateSourceInternalSiteName));
                    rpProtectionRecommendation = getNewProtectionRecommendation(vpoolChangeVolume, vpool);
                }
            }
            // end of for loop trying to find solution using possible rp cluster sites
            rpProtectionRecommendation = getNewProtectionRecommendation(vpoolChangeVolume, vpool);
        }
    // end of protection systems for loop
    }
    // we went through all the candidate pools and there are still some of the volumes that haven't been placed, then we failed to find
    // a solution
    _log.error("RP Placement : ViPR could not find matching storage pools that could be protected via RecoverPoint");
    throw APIException.badRequests.cannotFindSolutionForRP(placementStatus.toString(dbClient));
}