Examples with VirtualArray com.emc.storageos.db.client.model.VirtualArray used on opensource projects

Example 91 with VirtualArray

use of com.emc.storageos.db.client.model.VirtualArray in project coprhd-controller by CoprHD.

the class VmaxMaskingOrchestrator method suggestExportMasksForPlacement.

@Override
protected void suggestExportMasksForPlacement(StorageSystem storage, BlockStorageDevice device, List<Initiator> initiators, ExportMaskPlacementDescriptor descriptor) {
    // VMAX can have multiple ExportMasks (MaskingViews) that contain the same set of initiators.
    // So, it's up to this implementation to determine where to best place the volumes based
    // on volume and ExportMask characteristics. To that end, we will hint that the placement
    // will be separating the volumes per ExportMask
    descriptor.setPlacementHint(ExportMaskPlacementDescriptor.PlacementHint.VOLUMES_TO_SEPARATE_MASKS);
    // Find all the ExportMasks on the array that have the initiators (or a subset of them)
    Map<URI, ExportMask> matchingMasks = readExistingExportMasks(storage, device, initiators);
    // filter out export masks which do not have at least one port from the virtual array
    Set<URI> invalidMasks = new HashSet<URI>();
    VirtualArray virtualArray = _dbClient.queryObject(VirtualArray.class, descriptor.getVirtualArray());
    for (Entry<URI, ExportMask> entry : matchingMasks.entrySet()) {
        ExportMask mask = entry.getValue();
        boolean matched = maskHasStoragePortsInExportVarray(virtualArray, mask);
        if (!matched) {
            invalidMasks.add(entry.getKey());
            _log.info("Mask does not have valid ports from varray: {}", mask.getLabel());
        }
    }
    for (URI maskUri : invalidMasks) {
        matchingMasks.remove(maskUri);
    }
    // set matching masks in the descriptor
    descriptor.setMasks(matchingMasks);
    if (matchingMasks.isEmpty()) {
        return;
    }
    // Dummy/non-essential data
    ExportGroup dummyExportGroup = new ExportGroup();
    dummyExportGroup.setType(ExportGroupType.Host.name());
    // InitiatorHelper for processing the initiators
    InitiatorHelper initiatorHelper = new InitiatorHelper(initiators).process(dummyExportGroup);
    // Create and fill in:
    // -- Mapping of initiators to ExportMask URIs
    // -- Set of ExportMasks that match partially to the initiator set
    // -- Mapping of ExportMask URI to ExportMaskPolicy
    Set<URI> partialMasks = new HashSet<>();
    Map<String, Set<URI>> initiatorToExportMaskMap = new HashMap<>();
    Map<URI, ExportMaskPolicy> policyCache = new HashMap<>();
    Collection<String> portNames = Collections2.transform(initiators, CommonTransformerFunctions.fctnInitiatorToPortName());
    for (Map.Entry<URI, ExportMask> entry : matchingMasks.entrySet()) {
        URI exportMaskURI = entry.getKey();
        ExportMask exportMask = entry.getValue();
        // Get the ExportMaskPolicy, thereby saving it to the policyCache. The policyCache is a mapping of the ExportMask URI to
        // its ExportMaskPolicy object. The ExportMaskPolicy is a transient object that is used to hold meta data about the ExportMask.
        // This meta data is mostly there to describe the AutoTieringPolicy, Host IO parameters, and InitiatorGroup usage.
        // There could be other information, as well. It suffices to understand that this data is relevant for the rules applicator
        // that we're invoking below. The rules applicator will use this as a way to determine which ExportMask is best
        // suited to hold the volumes.
        getExportMaskPolicy(policyCache, device, storage, exportMask);
        // Populate the mapping of Initiator portname (WWN/IQN) to the ExportMask URI
        for (String portName : portNames) {
            Set<URI> masks = initiatorToExportMaskMap.get(portName);
            if (masks == null) {
                masks = new HashSet<>();
                initiatorToExportMaskMap.put(portName, masks);
            }
            masks.add(exportMaskURI);
        }
        // export to, then we need to put it in the set of masks that have a partial match
        if (!ExportMaskUtils.hasExactlyTheseInitiators(exportMask, portNames, _dbClient)) {
            partialMasks.add(exportMaskURI);
        }
        // Determine which ExportMasks are equivalent in terms of attributes, other than
        // the number of volumes that they contain. The preference is for the rules
        // applicator (below) to choose, from equivalent masks, the one with the least
        // volumes. But we'd like to still know which are equivalent in case the mask
        // that is selected in the code here, is invalid in some higher level validation.
        descriptor.addToEquivalentMasks(exportMask, policyCache.get(exportMaskURI));
    }
    // Populate the Volume URI to Volume HLU mapping. We will let the array decide the HLUs (i.e., set it to -1)
    Map<URI, Integer> volumeMap = new HashMap<>();
    for (URI volumeURI : descriptor.getVolumesToPlace().keySet()) {
        volumeMap.put(volumeURI, -1);
    }
    // Mapping of ExportMask URI to Volume-HLU: the basic output that we're expecting to be filled in by the rules applicator
    Map<URI, Map<URI, Integer>> maskToUpdateWithNewVolumes = new HashMap<>();
    // All data structures should have been filled in at this point; create the context and ruleApplicator for it
    VmaxVolumeToExportMaskApplicatorContext context = createVPlexBackendApplicatorContext(dummyExportGroup, storage, policyCache, initiatorHelper, initiatorToExportMaskMap, partialMasks, volumeMap, maskToUpdateWithNewVolumes);
    VplexBackendVolumeToExportMaskRuleApplicator rulesApplicator = new VplexBackendVolumeToExportMaskRuleApplicator(_dbClient, context);
    try {
        rulesApplicator.run();
        if (context.resultSuccess) {
            // Get configuration value for how many volumes are allowed in MaskingView. If the number
            // of volumes exceeds this amount for a particular ExportMask, then it cannot be a candidate
            // for reuse.
            customConfigHandler = (CustomConfigHandler) ControllerServiceImpl.getBean(CUSTOM_CONFIG_HANDLER);
            int maxVolumesAllowedByConfig = Integer.valueOf(customConfigHandler.getComputedCustomConfigValue(CustomConfigConstants.VPLEX_VMAX_MASKING_VIEW_MAXIMUM_VOLUMES, storage.getSystemType(), null));
            // Use a local cache in case the same volumes are selected
            // to be placed into different ExportMasks
            ObjectLocalCache cache = new ObjectLocalCache(_dbClient);
            // Process each entry in the mapping of ExportMask to Volumes ...
            for (Map.Entry<URI, Map<URI, Integer>> entry : maskToUpdateWithNewVolumes.entrySet()) {
                URI exportMaskURI = entry.getKey();
                Set<URI> volumeURIs = entry.getValue().keySet();
                // The ExportMaskPolicy is a transient object that is used to hold meta data about the ExportMask.
                // This meta data is mostly there to describe the AutoTieringPolicy, Host IO parameters, and InitiatorGroup usage.
                // There could be other information, as well.
                ExportMaskPolicy policy = policyCache.get(exportMaskURI);
                // Translate the Volume URI to Volume HLU map to a Volume URI to Volume object map:
                Map<URI, Volume> volumes = new HashMap<>();
                List<Volume> queriedVols = cache.queryObject(Volume.class, volumeURIs);
                for (Volume volume : queriedVols) {
                    volumes.put(volume.getId(), volume);
                }
                // TODO: We need to explore if we should/can make the volume count check (done below) another rule run as
                // part of the RuleApplicator. The one concern with doing this is what would happen if another ExportMask
                // is selected. Would we end up selecting an ExportMask that can support the volumes, but is undesirable
                // through some other considerations. For now, we will let the rules engine decide the appropriate
                // ExportMasks and then evaluate that selection for the volume count.
                // Validate the number of volumes that will be in the ExportMask if 'volumes' were added to it.
                // If there are more the maximum number of volumes allowed, then we should not place these 'volumes'.
                // The volumes would show up in the descriptor as being unplaced. The VplexBackendManager would
                // to take care of this case by creating another ExportMask to contain these volumes.
                ExportMask exportMask = matchingMasks.get(exportMaskURI);
                int totalVolumesWhenAddedToExportMask = exportMask.returnTotalVolumeCount() + volumes.size();
                boolean moreVolumesThanAllowedByConfig = totalVolumesWhenAddedToExportMask > maxVolumesAllowedByConfig;
                boolean moreVolumesThanAllowedByArray = totalVolumesWhenAddedToExportMask > policy.getMaxVolumesAllowed();
                if (moreVolumesThanAllowedByArray || moreVolumesThanAllowedByConfig) {
                    _log.info(String.format("ExportMask %s (%s) is matching, but already %d volumes associated with it. " + "Adding %d volumes to it will make it go over its limit, hence it will not be used for placement.%n" + "The configuration allows %d volumes and the array allows %d as the max number of volumes to a MaskingView", exportMask.getMaskName(), exportMask.getId(), exportMask.returnTotalVolumeCount(), volumes.size(), maxVolumesAllowedByConfig, policy.getMaxVolumesAllowed()));
                    continue;
                }
                // Fill in the descriptor to be used for VPlex backend placement
                descriptor.placeVolumes(exportMaskURI, volumes);
                // be invalid (by some higher-level validation).
                for (URI volumeURI : volumes.keySet()) {
                    for (URI equivalentExport : descriptor.getEquivalentExportMasks(exportMaskURI)) {
                        descriptor.addAsAlternativeExportForVolume(volumeURI, exportMaskURI);
                    }
                }
            }
        }
    } catch (Exception e) {
        _log.error("Exception while trying to get suggestions for ExportMasks to used for volumes", e);
    }
}

Example 92 with VirtualArray

use of com.emc.storageos.db.client.model.VirtualArray in project coprhd-controller by CoprHD.

the class SRDFOperations method updateVolumeLabels.

/**
 * Updates the label field of the invalidTgt, and if the volume is fronted by
 * a Vplex Volume, also updates the target vplex volume label.
 * @param trustedSrc -- source volume with correct label (vmax)
 * @param invalidTgt -- target volume with incorrect label (vmax)
 */
private void updateVolumeLabels(Volume trustedSrc, Volume invalidTgt) {
    // Update the label of the invalid target to match it's new source.
    VirtualArray invalidTgtVA = dbClient.queryObject(VirtualArray.class, invalidTgt.getVirtualArray());
    StringBuilder newLabel = new StringBuilder();
    newLabel.append(trustedSrc.getLabel());
    newLabel.append("-target-");
    newLabel.append(invalidTgtVA.getLabel());
    log.info("Revised name for target: " + newLabel.toString());
    invalidTgt.setLabel(newLabel.toString());
    NamedURI projectURI = invalidTgt.getProject();
    projectURI.setName(newLabel.toString());
    invalidTgt.setProject(projectURI);
    NamedURI tenantURI = invalidTgt.getTenant();
    tenantURI.setName(newLabel.toString());
    invalidTgt.setTenant(tenantURI);
    // See if there is a corresponding Vplex volume. If so update its label as well.
    Volume tgtVplexVolume = VPlexSrdfUtil.getVplexVolumeFromSrdfVolume(dbClient, invalidTgt);
    if (tgtVplexVolume != null) {
        // If the target volume is fronted by Vplex, the source volume should also be Vplex fronted
        Volume srcVplexVolume = VPlexSrdfUtil.getVplexVolumeFromSrdfVolume(dbClient, trustedSrc);
        if (srcVplexVolume != null) {
            newLabel.setLength(0);
            newLabel.append(srcVplexVolume.getLabel());
            newLabel.append("-target-");
            newLabel.append(invalidTgtVA.getLabel());
            log.info("Revised name for VPlex target: " + newLabel.toString());
            tgtVplexVolume.setLabel(newLabel.toString());
            projectURI = tgtVplexVolume.getProject();
            projectURI.setName(newLabel.toString());
            tgtVplexVolume.setProject(projectURI);
            tenantURI = tgtVplexVolume.getTenant();
            tenantURI.setName(newLabel.toString());
            tgtVplexVolume.setTenant(tenantURI);
            dbClient.updateAndReindexObject(tgtVplexVolume);
        }
    }
}

Example 93 with VirtualArray

use of com.emc.storageos.db.client.model.VirtualArray in project coprhd-controller by CoprHD.

the class SRDFOperations method findOrCreateTargetBlockConsistencyGroup.

/**
 * Given a list of target volumes, find a source volume and its associated BlockConsistencyGroup.
 *
 * Using this BlockConsistencyGroup, find or create a group for the targets and apply to it all
 * targets.
 *
 * @param targetVolumes List of target volumes
 */
private void findOrCreateTargetBlockConsistencyGroup(List<? extends BlockObject> targetVolumes) {
    log.info("Find or create target BlockConsistencyGroup...");
    final String LABEL_SUFFIX_FOR_46X = "T";
    // Get a target
    Volume target = (Volume) targetVolumes.get(0);
    // Get its SRDF parent
    Volume source = dbClient.queryObject(Volume.class, target.getSrdfParent().getURI());
    // Get source system
    StorageSystem sourceSystem = dbClient.queryObject(StorageSystem.class, source.getStorageController());
    // Get the SRDF parents' BlockConsistencyGroup
    BlockConsistencyGroup sourceGroup = dbClient.queryObject(BlockConsistencyGroup.class, source.getConsistencyGroup());
    // Get the VirtualArray associated with the target volume
    VirtualArray virtualArray = dbClient.queryObject(VirtualArray.class, target.getVirtualArray());
    // Get the Project
    Project project = dbClient.queryObject(Project.class, target.getProject().getURI());
    // Generate the target BlockConsistencyGroup name
    String CG_NAME_FORMAT = "%s-Target-%s";
    String cgName = format(CG_NAME_FORMAT, sourceGroup.getLabel(), virtualArray.getLabel());
    // Check for existing target group
    List<BlockConsistencyGroup> groups = queryActiveResourcesByConstraint(dbClient, BlockConsistencyGroup.class, PrefixConstraint.Factory.getFullMatchConstraint(BlockConsistencyGroup.class, "label", cgName));
    BlockConsistencyGroup newConsistencyGroup = null;
    if (groups.isEmpty()) {
        log.info("Creating target group: {}", cgName);
        // create CG
        newConsistencyGroup = new BlockConsistencyGroup();
        newConsistencyGroup.setId(URIUtil.createId(BlockConsistencyGroup.class));
        newConsistencyGroup.setLabel(cgName);
        newConsistencyGroup.setProject(new NamedURI(project.getId(), project.getLabel()));
        newConsistencyGroup.setTenant(new NamedURI(project.getTenantOrg().getURI(), project.getTenantOrg().getName()));
        // ModifyReplica on GroupSync for swap operation will try to create CG with same name on target provider.
        // For 4.6.x, better to use a different name for target CG.
        StringBuffer label = new StringBuffer(sourceGroup.getLabel());
        if (!sourceSystem.getUsingSmis80()) {
            label.append(LABEL_SUFFIX_FOR_46X);
        }
        newConsistencyGroup.setAlternateLabel(label.toString());
        // Persist the new BCG
        dbClient.createObject(newConsistencyGroup);
    } else {
        newConsistencyGroup = groups.get(0);
        log.info("Using existing target group: {}", newConsistencyGroup.getLabel());
    }
    // Update and persist target volumes with BCG
    for (BlockObject targetObj : targetVolumes) {
        targetObj.setConsistencyGroup(newConsistencyGroup.getId());
        targetObj.setReplicationGroupInstance(newConsistencyGroup.getAlternateLabel());
    }
    dbClient.updateObject(targetVolumes);
}

Example 94 with VirtualArray

use of com.emc.storageos.db.client.model.VirtualArray in project coprhd-controller by CoprHD.

the class PlacementTests method testVPlexLocalSRDFBasicPlacement.

@Test
public void testVPlexLocalSRDFBasicPlacement() {
    String[] vmax1FE = { "50:FE:FE:FE:FE:FE:FE:00", "50:FE:FE:FE:FE:FE:FE:01" };
    String[] vmax2FE = { "51:FE:FE:FE:FE:FE:FE:00", "51:FE:FE:FE:FE:FE:FE:01" };
    String[] vplexFE = { "FE:FE:FE:FE:FE:FE:FE:00", "FE:FE:FE:FE:FE:FE:FE:01" };
    String[] vplexBE = { "BE:BE:BE:BE:BE:BE:BE:00", "BE:BE:BE:BE:BE:BE:BE:01" };
    // Create 2 Virtual Arrays
    VirtualArray srcVarray = PlacementTestUtils.createVirtualArray(_dbClient, "srcVarray");
    VirtualArray tgtVarray = PlacementTestUtils.createVirtualArray(_dbClient, "tgtVarray");
    // Create 2 Networks
    StringSet connVA = new StringSet();
    connVA.add(srcVarray.getId().toString());
    Network network1 = PlacementTestUtils.createNetwork(_dbClient, vmax1FE, "VSANSite1", "FC+BROCADE+FE", connVA);
    connVA = new StringSet();
    connVA.add(tgtVarray.getId().toString());
    Network network2 = PlacementTestUtils.createNetwork(_dbClient, vmax2FE, "VSANSite2", "FC+CISCO+FE", connVA);
    // Create 2 storage systems
    StorageSystem[] storageSystems = PlacementTestUtils.createSRDFStorageSystems(_dbClient, "vmax1", network1, vmax1FE, srcVarray, "vmax2", network2, vmax2FE, tgtVarray);
    StorageSystem storageSystem1 = storageSystems[1];
    StorageSystem storageSystem2 = storageSystems[2];
    StoragePool[] storagePools = PlacementTestUtils.createStoragePoolsForTwo(_dbClient, storageSystem1, srcVarray, storageSystem2, tgtVarray);
    StorageSystem vplexSystem = PlacementTestUtils.createVPlexOneCluster(_dbClient, "vplex1", srcVarray, network1, network1, vplexFE, vplexBE);
    // Create a target virtual pool
    VirtualPool tgtVpool = new VirtualPool();
    tgtVpool.setId(URI.create("tgtVpool"));
    tgtVpool.setLabel("Target Vpool");
    tgtVpool.setSupportedProvisioningType(VirtualPool.ProvisioningType.Thin.name());
    tgtVpool.setDriveType(SupportedDriveTypes.FC.name());
    StringSet matchedPools2 = new StringSet();
    matchedPools2.add(storagePools[4].getId().toString());
    matchedPools2.add(storagePools[5].getId().toString());
    matchedPools2.add(storagePools[6].getId().toString());
    tgtVpool.setMatchedStoragePools(matchedPools2);
    tgtVpool.setUseMatchedPools(true);
    StringSet virtualArrays2 = new StringSet();
    virtualArrays2.add(tgtVarray.getId().toString());
    tgtVpool.setVirtualArrays(virtualArrays2);
    _dbClient.createObject(tgtVpool);
    // Make a remote copy protection setting
    VpoolRemoteCopyProtectionSettings settings = new VpoolRemoteCopyProtectionSettings();
    settings.setId(URI.create("remoteCopySettings"));
    settings.setCopyMode(VpoolRemoteCopyProtectionSettings.CopyModes.ASYNCHRONOUS.name());
    settings.setVirtualArray(tgtVarray.getId());
    settings.setVirtualPool(tgtVpool.getId());
    _dbClient.createObject(settings);
    // Create an VPLEX Local/SRDF source virtual pool
    VirtualPool srcVpool = new VirtualPool();
    srcVpool.setId(URI.create("srcVpool"));
    srcVpool.setLabel("Source Vpool");
    srcVpool.setSupportedProvisioningType(VirtualPool.ProvisioningType.Thin.name());
    srcVpool.setDriveType(SupportedDriveTypes.FC.name());
    StringSet matchedPools1 = new StringSet();
    matchedPools1.add(storagePools[1].getId().toString());
    matchedPools1.add(storagePools[2].getId().toString());
    matchedPools1.add(storagePools[3].getId().toString());
    srcVpool.setMatchedStoragePools(matchedPools1);
    srcVpool.setUseMatchedPools(true);
    StringSet virtualArrays1 = new StringSet();
    virtualArrays1.add(srcVarray.getId().toString());
    srcVpool.setVirtualArrays(virtualArrays1);
    StringMap remoteProtectionSettings = new StringMap();
    remoteProtectionSettings.put(tgtVarray.getId().toString(), settings.getId().toString());
    srcVpool.setProtectionRemoteCopySettings(remoteProtectionSettings);
    srcVpool.setHighAvailability(VirtualPool.HighAvailabilityType.vplex_local.name());
    _dbClient.createObject(srcVpool);
    // Create Tenant
    TenantOrg tenant = new TenantOrg();
    tenant.setId(URI.create("tenant"));
    _dbClient.createObject(tenant);
    // Create a project object
    Project project = new Project();
    project.setId(URI.create("project"));
    project.setLabel("RDG1");
    project.setTenantOrg(new NamedURI(tenant.getId(), project.getLabel()));
    _dbClient.createObject(project);
    // Create capabilities
    VirtualPoolCapabilityValuesWrapper capabilities = PlacementTestUtils.createCapabilities("2GB", 1, null);
    // Run single volume placement: Run 10 times to make sure pool3 never comes up for source and pool6 for target.
    for (int i = 0; i < 10; i++) {
        Map<VpoolUse, List<Recommendation>> recommendationsMap = PlacementTestUtils.invokePlacementForVpool(_dbClient, _coordinator, srcVarray, project, srcVpool, capabilities);
        List<Recommendation> recommendations = recommendationsMap.get(VpoolUse.ROOT);
        assertNotNull(recommendations);
        assertTrue(!recommendations.isEmpty());
        assertNotNull(recommendations.get(0));
        assert (recommendations.get(0) instanceof VPlexRecommendation);
        VPlexRecommendation vplexRecommendation = (VPlexRecommendation) recommendations.get(0);
        URI srcStoragePool = vplexRecommendation.getSourceStoragePool();
        assert (srcStoragePool.equals(storagePools[1].getId()) || srcStoragePool.equals(storagePools[2].getId()));
        assert (vplexRecommendation.getRecommendation() instanceof SRDFRecommendation);
        recommendations = recommendationsMap.get(VpoolUse.SRDF_COPY);
        assertTrue(!recommendations.isEmpty());
        assertNotNull(recommendations.get(0));
        assert (recommendations.get(0) instanceof SRDFCopyRecommendation);
        SRDFCopyRecommendation srdfCopyRecommendation = (SRDFCopyRecommendation) recommendations.get(0);
        URI tgtStoragePool = srdfCopyRecommendation.getSourceStoragePool();
        assert (tgtStoragePool.equals(storagePools[4].getId()) || tgtStoragePool.equals(storagePools[5].getId()));
    }
}

Example 95 with VirtualArray

use of com.emc.storageos.db.client.model.VirtualArray in project coprhd-controller by CoprHD.

the class PlacementTests method testPlacementRpVplexProtectHASite.

/**
 * RP VPLEX placement -- placement decision based on RP array visibility
 * Protect HA side of the VPLEX Metro volume.
 */
@Test
public void testPlacementRpVplexProtectHASite() {
    String[] vmax1FE = { "50:FE:FE:FE:FE:FE:FE:00", "50:FE:FE:FE:FE:FE:FE:01" };
    String[] vmax2FE = { "51:FE:FE:FE:FE:FE:FE:00", "51:FE:FE:FE:FE:FE:FE:01" };
    String[] vmax3FE = { "52:FE:FE:FE:FE:FE:FE:00", "52:FE:FE:FE:FE:FE:FE:01" };
    String[] rp1FE = { "53:FE:FE:FE:FE:FE:FE:00", "53:FE:FE:FE:FE:FE:FE:01" };
    String[] rp2FE = { "54:FE:FE:FE:FE:FE:FE:00", "54:FE:FE:FE:FE:FE:FE:01" };
    String[] vplex1FE = { "FE:FE:FE:FE:FE:FE:FE:00", "FE:FE:FE:FE:FE:FE:FE:01" };
    String[] vplex1BE = { "BE:BE:BE:BE:BE:BE:BE:00", "BE:BE:BE:BE:BE:BE:BE:01" };
    String[] vplex2FE = { "FE:FE:FE:FE:FE:FE:FE:02", "FE:FE:FE:FE:FE:FE:FE:03" };
    String[] vplex2BE = { "BE:BE:BE:BE:BE:BE:BE:02", "BE:BE:BE:BE:BE:BE:BE:03" };
    String[] vplex3FE = { "FE:FE:FE:FE:FE:FE:FE:04", "FE:FE:FE:FE:FE:FE:FE:05" };
    String[] vplex3BE = { "BE:BE:BE:BE:BE:BE:BE:04", "BE:BE:BE:BE:BE:BE:BE:05" };
    // Create 3 Virtual Arrays
    VirtualArray haVarray = PlacementTestUtils.createVirtualArray(_dbClient, "varray1");
    VirtualArray srcVarray = PlacementTestUtils.createVirtualArray(_dbClient, "varray2");
    VirtualArray tgtVarray = PlacementTestUtils.createVirtualArray(_dbClient, "varray3");
    // Create 1 Network
    StringSet connVA = new StringSet();
    connVA.add(haVarray.getId().toString());
    connVA.add(srcVarray.getId().toString());
    connVA.add(tgtVarray.getId().toString());
    Network network = PlacementTestUtils.createNetwork(_dbClient, vplex1FE, "VSAN", "FC+BROCADE", connVA);
    PlacementTestUtils.addEndpoints(_dbClient, network, vplex2FE);
    PlacementTestUtils.addEndpoints(_dbClient, network, vplex3FE);
    PlacementTestUtils.addEndpoints(_dbClient, network, vplex1BE);
    PlacementTestUtils.addEndpoints(_dbClient, network, vplex2BE);
    PlacementTestUtils.addEndpoints(_dbClient, network, vplex3BE);
    PlacementTestUtils.addEndpoints(_dbClient, network, rp1FE);
    PlacementTestUtils.addEndpoints(_dbClient, network, rp2FE);
    PlacementTestUtils.addEndpoints(_dbClient, network, vmax1FE);
    PlacementTestUtils.addEndpoints(_dbClient, network, vmax2FE);
    PlacementTestUtils.addEndpoints(_dbClient, network, vmax3FE);
    // Create 3 storage systems
    StorageSystem storageSystem1 = PlacementTestUtils.createStorageSystem(_dbClient, "vmax", "vmax1");
    StorageSystem storageSystem2 = PlacementTestUtils.createStorageSystem(_dbClient, "vmax", "vmax2");
    StorageSystem storageSystem3 = PlacementTestUtils.createStorageSystem(_dbClient, "vmax", "vmax3");
    // Create two front-end storage ports VMAX1
    List<StoragePort> vmax1Ports = new ArrayList<StoragePort>();
    for (int i = 0; i < vmax1FE.length; i++) {
        vmax1Ports.add(PlacementTestUtils.createStoragePort(_dbClient, storageSystem1, network, vmax1FE[i], haVarray, StoragePort.PortType.frontend.name(), "portGroupvmax1" + i, "C0+FC0" + i));
    }
    // Create two front-end storage ports VMAX2
    List<StoragePort> vmax2Ports = new ArrayList<StoragePort>();
    for (int i = 0; i < vmax2FE.length; i++) {
        vmax2Ports.add(PlacementTestUtils.createStoragePort(_dbClient, storageSystem2, network, vmax2FE[i], srcVarray, StoragePort.PortType.frontend.name(), "portGroupvmax2" + i, "D0+FC0" + i));
    }
    // Create two front-end storage ports VMAX3
    List<StoragePort> vmax3Ports = new ArrayList<StoragePort>();
    for (int i = 0; i < vmax3FE.length; i++) {
        vmax3Ports.add(PlacementTestUtils.createStoragePort(_dbClient, storageSystem3, network, vmax3FE[i], tgtVarray, StoragePort.PortType.frontend.name(), "portGroupvmax3" + i, "E0+FC0" + i));
    }
    // Create 2 VPLEX storage systems
    StorageSystem vplexStorageSystem1 = PlacementTestUtils.createStorageSystem(_dbClient, "vplex", "vplex1");
    StorageSystem vplexStorageSystem2 = PlacementTestUtils.createStorageSystem(_dbClient, "vplex", "vplex2");
    // Create two front-end storage ports VPLEX1
    List<StoragePort> fePorts1 = new ArrayList<StoragePort>();
    for (int i = 0; i < vplex1FE.length; i++) {
        fePorts1.add(PlacementTestUtils.createStoragePort(_dbClient, vplexStorageSystem1, network, vplex1FE[i], haVarray, StoragePort.PortType.frontend.name(), "portGroupFE1-" + (i + 1), "A0+FC0" + i));
    }
    // Create two back-end storage ports VPLEX1
    List<StoragePort> bePorts1 = new ArrayList<StoragePort>();
    for (int i = 0; i < vplex1BE.length; i++) {
        bePorts1.add(PlacementTestUtils.createStoragePort(_dbClient, vplexStorageSystem1, network, vplex1BE[i], haVarray, StoragePort.PortType.backend.name(), "portGroupBE1-" + (i + 1), "B0+FC0" + i));
    }
    // Create two front-end storage ports VPLEX2
    List<StoragePort> fePorts2 = new ArrayList<StoragePort>();
    for (int i = 0; i < vplex2FE.length; i++) {
        fePorts2.add(PlacementTestUtils.createStoragePort(_dbClient, vplexStorageSystem1, network, vplex2FE[i], srcVarray, StoragePort.PortType.frontend.name(), "portGroupFE2-" + (i + 1), "F0+FC0" + i));
    }
    // Create two back-end storage ports VPLEX2
    List<StoragePort> bePorts2 = new ArrayList<StoragePort>();
    for (int i = 0; i < vplex2BE.length; i++) {
        bePorts2.add(PlacementTestUtils.createStoragePort(_dbClient, vplexStorageSystem1, network, vplex2BE[i], srcVarray, StoragePort.PortType.backend.name(), "portGroupBE2-" + (i + 1), "G0+FC0" + i));
    }
    // Create two front-end storage ports VPLEX3
    List<StoragePort> fePorts3 = new ArrayList<StoragePort>();
    for (int i = 0; i < vplex3FE.length; i++) {
        fePorts3.add(PlacementTestUtils.createStoragePort(_dbClient, vplexStorageSystem2, network, vplex3FE[i], tgtVarray, StoragePort.PortType.frontend.name(), "portGroupFE3-" + (i + 1), "H0+FC0" + i));
    }
    // Create two back-end storage ports VPLEX3
    List<StoragePort> bePorts3 = new ArrayList<StoragePort>();
    for (int i = 0; i < vplex3BE.length; i++) {
        bePorts3.add(PlacementTestUtils.createStoragePort(_dbClient, vplexStorageSystem2, network, vplex3BE[i], tgtVarray, StoragePort.PortType.backend.name(), "portGroupBE3-" + (i + 1), "I0+FC0" + i));
    }
    // Create RP system
    AbstractChangeTrackingSet<String> wwnSite1 = new StringSet();
    for (int i = 0; i < rp1FE.length; i++) {
        wwnSite1.add(rp1FE[i]);
    }
    StringSetMap initiatorsSiteMap = new StringSetMap();
    initiatorsSiteMap.put("site1", wwnSite1);
    AbstractChangeTrackingSet<String> wwnSite2 = new StringSet();
    for (int i = 0; i < rp2FE.length; i++) {
        wwnSite2.add(rp2FE[i]);
    }
    initiatorsSiteMap.put("site2", wwnSite2);
    StringSet storSystems = new StringSet();
    storSystems.add(ProtectionSystem.generateAssociatedStorageSystem("site1", "vplex1cluster1"));
    storSystems.add(ProtectionSystem.generateAssociatedStorageSystem("site1", "vplex1cluster2"));
    storSystems.add(ProtectionSystem.generateAssociatedStorageSystem("site2", "vplex2cluster1"));
    storSystems.add(ProtectionSystem.generateAssociatedStorageSystem("site2", "vplex2cluster2"));
    StringSetMap rpVisibleSystems = new StringSetMap();
    StringSet storageIds = new StringSet();
    storageIds.add(vplexStorageSystem1.getId().toString());
    rpVisibleSystems.put("site1", storageIds);
    StringSet storageIds2 = new StringSet();
    storageIds2.add(vplexStorageSystem2.getId().toString());
    rpVisibleSystems.put("site2", storageIds2);
    StringMap siteVolCap = new StringMap();
    siteVolCap.put("site1", "3221225472");
    siteVolCap.put("site2", "3221225472");
    StringMap siteVolCnt = new StringMap();
    siteVolCnt.put("site1", "10");
    siteVolCnt.put("site2", "10");
    ProtectionSystem rpSystem = PlacementTestUtils.createProtectionSystem(_dbClient, "rp", "rp1", "site1", "site2", null, "IP", initiatorsSiteMap, storSystems, rpVisibleSystems, Long.valueOf("3221225472"), Long.valueOf("2"), siteVolCap, siteVolCnt);
    // RP Site Array objects
    RPSiteArray rpSiteArray1 = new RPSiteArray();
    rpSiteArray1.setId(URI.create("rsa1"));
    rpSiteArray1.setStorageSystem(URI.create("vplex1"));
    rpSiteArray1.setRpInternalSiteName("site1");
    rpSiteArray1.setRpProtectionSystem(rpSystem.getId());
    _dbClient.createObject(rpSiteArray1);
    RPSiteArray rpSiteArray2 = new RPSiteArray();
    rpSiteArray2.setId(URI.create("rsa2"));
    rpSiteArray2.setStorageSystem(URI.create("vplex2"));
    rpSiteArray2.setRpInternalSiteName("site2");
    rpSiteArray2.setRpProtectionSystem(rpSystem.getId());
    _dbClient.createObject(rpSiteArray2);
    // Create a storage pool for vmax1
    StoragePool pool1 = PlacementTestUtils.createStoragePool(_dbClient, haVarray, storageSystem1, "pool1", "Pool1", Long.valueOf(1024 * 1024 * 10), Long.valueOf(1024 * 1024 * 10), 300, 300, StoragePool.SupportedResourceTypes.THIN_ONLY.toString());
    // Create a storage pool for vmax1
    StoragePool pool2 = PlacementTestUtils.createStoragePool(_dbClient, haVarray, storageSystem1, "pool2", "Pool2", Long.valueOf(1024 * 1024 * 10), Long.valueOf(1024 * 1024 * 10), 300, 300, StoragePool.SupportedResourceTypes.THIN_ONLY.toString());
    // Create a storage pool for vmax1
    StoragePool pool3 = PlacementTestUtils.createStoragePool(_dbClient, haVarray, storageSystem1, "pool3", "Pool3", Long.valueOf(1024 * 1024 * 1), Long.valueOf(1024 * 1024 * 1), 100, 100, StoragePool.SupportedResourceTypes.THIN_ONLY.toString());
    // Create a storage pool for vmax2
    StoragePool pool4 = PlacementTestUtils.createStoragePool(_dbClient, srcVarray, storageSystem2, "pool4", "Pool4", Long.valueOf(SIZE_GB * 10), Long.valueOf(SIZE_GB * 10), 300, 300, StoragePool.SupportedResourceTypes.THIN_ONLY.toString());
    // Create a storage pool for vmax2
    StoragePool pool5 = PlacementTestUtils.createStoragePool(_dbClient, srcVarray, storageSystem2, "pool5", "Pool5", Long.valueOf(SIZE_GB * 10), Long.valueOf(SIZE_GB * 50), 300, 300, StoragePool.SupportedResourceTypes.THIN_ONLY.toString());
    // Create a storage pool for vmax2
    StoragePool pool6 = PlacementTestUtils.createStoragePool(_dbClient, srcVarray, storageSystem2, "pool6", "Pool6", Long.valueOf(1024 * 1024 * 1), Long.valueOf(1024 * 1024 * 1), 100, 100, StoragePool.SupportedResourceTypes.THIN_ONLY.toString());
    // Create a storage pool for vmax3
    StoragePool pool7 = PlacementTestUtils.createStoragePool(_dbClient, tgtVarray, storageSystem3, "pool7", "Pool7", Long.valueOf(1024 * 1024 * 10), Long.valueOf(1024 * 1024 * 10), 300, 300, StoragePool.SupportedResourceTypes.THIN_ONLY.toString());
    // Create a storage pool for vmax3
    StoragePool pool8 = PlacementTestUtils.createStoragePool(_dbClient, tgtVarray, storageSystem3, "pool8", "Pool8", Long.valueOf(1024 * 1024 * 10), Long.valueOf(1024 * 1024 * 10), 300, 300, StoragePool.SupportedResourceTypes.THIN_ONLY.toString());
    // Create a storage pool for vmax3
    StoragePool pool9 = PlacementTestUtils.createStoragePool(_dbClient, tgtVarray, storageSystem3, "pool9", "Pool9", Long.valueOf(1024 * 1024 * 1), Long.valueOf(1024 * 1024 * 1), 100, 100, StoragePool.SupportedResourceTypes.THIN_ONLY.toString());
    // Create HA virtual pool
    VirtualPool haVpool = new VirtualPool();
    haVpool.setId(URI.create(haVpoolUri));
    haVpool.setLabel("haVpool");
    haVpool.setType("block");
    haVpool.setSupportedProvisioningType(VirtualPool.ProvisioningType.Thin.name());
    haVpool.setDriveType(SupportedDriveTypes.FC.name());
    StringSet matchedPools = new StringSet();
    matchedPools.add(pool1.getId().toString());
    matchedPools.add(pool2.getId().toString());
    matchedPools.add(pool3.getId().toString());
    haVpool.setMatchedStoragePools(matchedPools);
    StringSet virtualArrays1 = new StringSet();
    virtualArrays1.add(haVarray.getId().toString());
    haVpool.setVirtualArrays(virtualArrays1);
    haVpool.setUseMatchedPools(true);
    _dbClient.createObject(haVpool);
    // Create RP target vpool
    VirtualPool rpTgtVpool = new VirtualPool();
    rpTgtVpool.setId(URI.create("rpTgtVpool"));
    rpTgtVpool.setLabel("rpTgtVpool");
    rpTgtVpool.setType("block");
    rpTgtVpool.setSupportedProvisioningType(VirtualPool.ProvisioningType.Thin.name());
    rpTgtVpool.setDriveType(SupportedDriveTypes.FC.name());
    rpTgtVpool.setHighAvailability(VirtualPool.HighAvailabilityType.vplex_local.name());
    matchedPools = new StringSet();
    matchedPools.add(pool7.getId().toString());
    matchedPools.add(pool8.getId().toString());
    matchedPools.add(pool9.getId().toString());
    rpTgtVpool.setMatchedStoragePools(matchedPools);
    rpTgtVpool.setUseMatchedPools(true);
    StringSet virtualArrays3 = new StringSet();
    virtualArrays3.add(tgtVarray.getId().toString());
    rpTgtVpool.setVirtualArrays(virtualArrays3);
    _dbClient.createObject(rpTgtVpool);
    // Create a RP VPLEX source virtual pool
    VirtualPool rpVplexSrcVpool = new VirtualPool();
    rpVplexSrcVpool.setId(URI.create(dummyVpoolUri));
    rpVplexSrcVpool.setLabel("rpVplexSrcVpool");
    rpVplexSrcVpool.setType("block");
    rpVplexSrcVpool.setSupportedProvisioningType(VirtualPool.ProvisioningType.Thin.name());
    rpVplexSrcVpool.setDriveType(SupportedDriveTypes.FC.name());
    rpVplexSrcVpool.setHighAvailability(VirtualPool.HighAvailabilityType.vplex_distributed.name());
    StringMap vavpMap = new StringMap();
    vavpMap.put(haVarray.getId().toString(), haVpool.getId().toString());
    rpVplexSrcVpool.setHaVarrayVpoolMap(vavpMap);
    VpoolProtectionVarraySettings protectionSettings = new VpoolProtectionVarraySettings();
    protectionSettings.setVirtualPool(rpTgtVpool.getId());
    protectionSettings.setId(URI.create("protectionSettings"));
    _dbClient.createObject(protectionSettings);
    List<VpoolProtectionVarraySettings> protectionSettingsList = new ArrayList<VpoolProtectionVarraySettings>();
    protectionSettingsList.add(protectionSettings);
    StringMap protectionVarray = new StringMap();
    protectionVarray.put(tgtVarray.getId().toString(), protectionSettingsList.get(0).getId().toString());
    rpVplexSrcVpool.setProtectionVarraySettings(protectionVarray);
    rpVplexSrcVpool.setRpCopyMode("SYNCHRONOUS");
    rpVplexSrcVpool.setRpRpoType("MINUTES");
    rpVplexSrcVpool.setRpRpoValue(Long.valueOf("5"));
    matchedPools = new StringSet();
    matchedPools.add(pool4.getId().toString());
    matchedPools.add(pool5.getId().toString());
    matchedPools.add(pool6.getId().toString());
    rpVplexSrcVpool.setMatchedStoragePools(matchedPools);
    rpVplexSrcVpool.setHaVarrayConnectedToRp(haVarray.getId().toString());
    rpVplexSrcVpool.setUseMatchedPools(true);
    StringSet virtualArrays2 = new StringSet();
    virtualArrays2.add(srcVarray.getId().toString());
    rpVplexSrcVpool.setVirtualArrays(virtualArrays2);
    _dbClient.createObject(rpVplexSrcVpool);
    // Create Tenant
    TenantOrg tenant = new TenantOrg();
    tenant.setId(URI.create("tenant"));
    _dbClient.createObject(tenant);
    // Create a project object
    Project project = new Project();
    project.setId(URI.create("project"));
    project.setLabel("project");
    project.setTenantOrg(new NamedURI(tenant.getId(), project.getLabel()));
    _dbClient.createObject(project);
    // Create block consistency group
    BlockConsistencyGroup cg = new BlockConsistencyGroup();
    cg.setProject(new NamedURI(project.getId(), project.getLabel()));
    cg.setId(URI.create("blockCG"));
    _dbClient.createObject(cg);
    // Create capabilities
    VirtualPoolCapabilityValuesWrapper capabilities = PlacementTestUtils.createCapabilities("2GB", 1, cg);
    // Run single volume placement: Run 10 times to make sure pool6 never comes up for source and pool9 for target.
    for (int i = 0; i < 10; i++) {
        List recommendations = PlacementTestUtils.invokePlacement(_dbClient, _coordinator, srcVarray, project, rpVplexSrcVpool, capabilities);
        assertNotNull(recommendations);
        assertTrue(!recommendations.isEmpty());
        assertNotNull(recommendations.get(0));
        RPProtectionRecommendation rec = (RPProtectionRecommendation) recommendations.get(0);
        assertNotNull(rec.getSourceRecommendations());
        assertTrue(!rec.getSourceRecommendations().isEmpty());
        assertNotNull(rec.getProtectionDevice());
        assertTrue("rp1".equals(rec.getProtectionDevice().toString()));
        for (RPRecommendation sourceRec : rec.getSourceRecommendations()) {
            assertNotNull(sourceRec.getInternalSiteName());
            assertNotNull(sourceRec.getVirtualArray());
            assertNotNull(sourceRec.getVirtualPool());
            assertNotNull(sourceRec.getVirtualVolumeRecommendation());
            assertNotNull(sourceRec.getHaRecommendation());
            assertNotNull(sourceRec.getTargetRecommendations());
            assertTrue(!sourceRec.getTargetRecommendations().isEmpty());
            assertTrue("site1".equals(sourceRec.getInternalSiteName()));
            assertTrue("vmax1".equals(sourceRec.getSourceStorageSystem().toString()));
            assertTrue(("pool1".equals(sourceRec.getSourceStoragePool().toString())) || ("pool2".equals(sourceRec.getSourceStoragePool().toString())));
            assertTrue("vplex1".equals(sourceRec.getVirtualVolumeRecommendation().getVPlexStorageSystem().toString()));
            assertNotNull(sourceRec.getHaRecommendation().getVirtualVolumeRecommendation());
            assertTrue("vplex1".equals(sourceRec.getHaRecommendation().getVirtualVolumeRecommendation().getVPlexStorageSystem().toString()));
            assertTrue("varray2".equals(sourceRec.getHaRecommendation().getVirtualArray().toString()));
            assertTrue(dummyVpoolUri.equals(sourceRec.getHaRecommendation().getVirtualPool().getId().toString()));
            assertTrue("vmax2".equals(sourceRec.getHaRecommendation().getSourceStorageSystem().toString()));
            assertTrue(("pool4".equals(sourceRec.getHaRecommendation().getSourceStoragePool().toString())) || ("pool5".equals(sourceRec.getHaRecommendation().getSourceStoragePool().toString())));
            assertNotNull(sourceRec.getTargetRecommendations());
            assertTrue(!sourceRec.getTargetRecommendations().isEmpty());
            for (RPRecommendation targetRec : sourceRec.getTargetRecommendations()) {
                assertNotNull(targetRec.getInternalSiteName());
                assertNotNull(targetRec.getVirtualArray());
                assertNotNull(targetRec.getVirtualPool());
                if (VirtualPool.vPoolSpecifiesHighAvailability(targetRec.getVirtualPool())) {
                    assertNotNull(targetRec.getVirtualVolumeRecommendation());
                }
                assertTrue("varray3".equals(targetRec.getVirtualArray().toString()));
                assertTrue(rpTgtVpool.getId().toString().equals(targetRec.getVirtualPool().getId().toString()));
                assertTrue("site2".equals(targetRec.getInternalSiteName()));
                assertTrue("vmax3".equals(targetRec.getSourceStorageSystem().toString()));
                assertTrue(("pool8".equals(targetRec.getSourceStoragePool().toString())) || ("pool7".equals(targetRec.getSourceStoragePool().toString())));
            }
        }
        // source journal
        assertNotNull(rec.getSourceJournalRecommendation());
        RPRecommendation sourceJournalRec = rec.getSourceJournalRecommendation();
        if (VirtualPool.vPoolSpecifiesHighAvailability(sourceJournalRec.getVirtualPool())) {
            assertNotNull(sourceJournalRec.getVirtualVolumeRecommendation());
        }
        assertTrue(("pool1".equals(sourceJournalRec.getSourceStoragePool().toString())) || ("pool2".equals(sourceJournalRec.getSourceStoragePool().toString())));
        // target journals
        assertNotNull(rec.getTargetJournalRecommendations());
        assertTrue(!rec.getTargetJournalRecommendations().isEmpty());
        for (RPRecommendation targetJournalRec : rec.getTargetJournalRecommendations()) {
            assertNotNull(targetJournalRec.getInternalSiteName());
            assertNotNull(targetJournalRec.getVirtualArray());
            assertNotNull(targetJournalRec.getVirtualPool());
            if (VirtualPool.vPoolSpecifiesHighAvailability(targetJournalRec.getVirtualPool())) {
                assertNotNull(targetJournalRec.getVirtualVolumeRecommendation());
            }
            assertTrue("vmax3".equals(targetJournalRec.getSourceStorageSystem().toString()));
            assertTrue(("pool8".equals(targetJournalRec.getSourceStoragePool().toString())) || ("pool7".equals(targetJournalRec.getSourceStoragePool().toString())));
        }
        _log.info("Recommendation : " + rec.toString(_dbClient));
    }
}