Search in sources :

Example 1 with VPlexConsistencyGroupInfo

use of com.emc.storageos.vplex.api.VPlexConsistencyGroupInfo in project coprhd-controller by CoprHD.

the class VPlexCommunicationInterface method discoverUnmanagedVolumes.

/**
 * Implementation for discovering unmanaged virtual volumes in a VPLEX storage system.
 *
 * @param accessProfile providing context for this discovery session
 * @param client a reference to the VPLEX API client
 * @param vvolMap map of virtual volume names to virtual volume info objects
 * @param volumeToExportMasksMap map of volumes to a set of associated UnManagedExportMasks
 * @param volumeToStorageViewMap map of volumes to a set of associated VPlexStorageViewInfos
 * @param recoverpointExportMasks recoverpoint export mask uris
 * @param tracker the performance report tracking object for this discovery process
 * @throws BaseCollectionException
 */
private void discoverUnmanagedVolumes(AccessProfile accessProfile, VPlexApiClient client, Map<String, VPlexVirtualVolumeInfo> allVirtualVolumes, Map<String, Set<UnManagedExportMask>> volumeToExportMasksMap, Map<String, Set<VPlexStorageViewInfo>> volumeToStorageViewMap, Set<String> recoverPointExportMasks, UnmanagedDiscoveryPerformanceTracker tracker) throws BaseCollectionException {
    String statusMessage = "Starting discovery of Unmanaged VPLEX Volumes.";
    s_logger.info(statusMessage + " Access Profile Details :  IpAddress : " + "PortNumber : {}, namespace : {}", accessProfile.getIpAddress() + accessProfile.getPortNumber(), accessProfile.getnamespace());
    URI vplexUri = accessProfile.getSystemId();
    StorageSystem vplex = _dbClient.queryObject(StorageSystem.class, vplexUri);
    if (null == vplex) {
        s_logger.error("No VPLEX Device was found in ViPR for URI: " + vplexUri);
        s_logger.error("Unmanaged VPLEX Volume discovery cannot continue.");
        return;
    }
    Set<URI> allUnmanagedVolumes = new HashSet<URI>();
    List<UnManagedVolume> newUnmanagedVolumes = new ArrayList<UnManagedVolume>();
    List<UnManagedVolume> knownUnmanagedVolumes = new ArrayList<UnManagedVolume>();
    List<UnManagedExportMask> unmanagedExportMasksToUpdate = new ArrayList<UnManagedExportMask>();
    Map<String, String> backendVolumeGuidToVvolGuidMap = new HashMap<String, String>();
    try {
        long timer = System.currentTimeMillis();
        Map<String, String> volumesToCgs = new HashMap<String, String>();
        List<VPlexConsistencyGroupInfo> cgs = client.getConsistencyGroups();
        s_logger.info("Found {} Consistency Groups.", cgs.size());
        for (VPlexConsistencyGroupInfo cg : cgs) {
            for (String volumeName : cg.getVirtualVolumes()) {
                volumesToCgs.put(volumeName, cg.getName());
            }
        }
        s_logger.info("Volume to Consistency Group Map is: " + volumesToCgs.toString());
        tracker.consistencyGroupFetch = System.currentTimeMillis() - timer;
        Map<String, String> clusterIdToNameMap = client.getClusterIdToNameMap();
        Map<String, String> varrayToClusterIdMap = new HashMap<String, String>();
        Map<String, String> distributedDevicePathToClusterMap = VPlexControllerUtils.getDistributedDevicePathToClusterMap(vplexUri, _dbClient);
        if (null != allVirtualVolumes) {
            // Pre-populate the virtual pools
            List<URI> allVpoolUris = _dbClient.queryByType(VirtualPool.class, true);
            Iterator<VirtualPool> vpoolIter = _dbClient.queryIterativeObjects(VirtualPool.class, allVpoolUris);
            List<VirtualPool> allVpools = new ArrayList<VirtualPool>();
            while (vpoolIter.hasNext()) {
                VirtualPool vpool = vpoolIter.next();
                // Only cache vplex virtual pools for vpool filtering
                if (VirtualPool.vPoolSpecifiesHighAvailability(vpool)) {
                    allVpools.add(vpool);
                }
            }
            for (String name : allVirtualVolumes.keySet()) {
                timer = System.currentTimeMillis();
                s_logger.info("Discovering Virtual Volume {}", name);
                // UnManagedVolume discover does a pretty expensive
                // iterative call into the VPLEX API to get extended details
                String discoveryKillSwitch = ControllerUtils.getPropertyValueFromCoordinator(_coordinator, VplexBackendIngestionContext.DISCOVERY_KILL_SWITCH);
                if ("stop".equals(discoveryKillSwitch)) {
                    s_logger.warn("discovery kill switch was set to stop, " + "so discontinuing unmanaged volume discovery");
                    return;
                }
                // on every volume in each cluster. First it gets all the
                // volume names/paths (the inexpensive "lite" call), then
                // iterates through them getting the details to populate the
                String discoveryFilter = ControllerUtils.getPropertyValueFromCoordinator(_coordinator, VplexBackendIngestionContext.DISCOVERY_FILTER);
                if ((discoveryFilter != null && !discoveryFilter.isEmpty()) && !(name.matches(discoveryFilter))) {
                    s_logger.warn("name {} doesn't match discovery filter {}", name, discoveryFilter);
                    continue;
                }
                // VPlexVirtualVolumeInfo objects with extended details
                VPlexVirtualVolumeInfo info = allVirtualVolumes.get(name);
                // to here null, it would mean a 404 happened earlier.
                if (null == info) {
                    continue;
                }
                Volume managedVolume = findVirtualVolumeManagedByVipr(info);
                UnManagedVolume unmanagedVolume = findUnmanagedVolumeKnownToVipr(info);
                if (null == managedVolume) {
                    s_logger.info("Virtual Volume {} is not managed by ViPR", name);
                    if (null != unmanagedVolume) {
                        // just refresh / update the existing unmanaged volume
                        s_logger.info("Unmanaged Volume {} is already known to ViPR", name);
                        updateUnmanagedVolume(info, vplex, unmanagedVolume, volumesToCgs, clusterIdToNameMap, varrayToClusterIdMap, distributedDevicePathToClusterMap, backendVolumeGuidToVvolGuidMap, volumeToStorageViewMap, allVpools);
                        knownUnmanagedVolumes.add(unmanagedVolume);
                    } else {
                        // set up new unmanaged vplex volume
                        s_logger.info("Unmanaged Volume {} is not known to ViPR", name);
                        unmanagedVolume = createUnmanagedVolume(info, vplex, volumesToCgs, clusterIdToNameMap, varrayToClusterIdMap, distributedDevicePathToClusterMap, backendVolumeGuidToVvolGuidMap, volumeToStorageViewMap, allVpools);
                        newUnmanagedVolumes.add(unmanagedVolume);
                    }
                    boolean nonRpExported = false;
                    Set<UnManagedExportMask> uems = volumeToExportMasksMap.get(unmanagedVolume.getNativeGuid());
                    if (uems != null) {
                        s_logger.info("{} UnManagedExportMasks found in the map for volume {}", uems.size(), unmanagedVolume.getNativeGuid());
                        for (UnManagedExportMask uem : uems) {
                            s_logger.info("   adding UnManagedExportMask {} to UnManagedVolume", uem.getMaskingViewPath());
                            unmanagedVolume.getUnmanagedExportMasks().add(uem.getId().toString());
                            uem.getUnmanagedVolumeUris().add(unmanagedVolume.getId().toString());
                            unmanagedExportMasksToUpdate.add(uem);
                            // add the known initiators, too
                            for (String initUri : uem.getKnownInitiatorUris()) {
                                s_logger.info("   adding known Initiator URI {} to UnManagedVolume", initUri);
                                unmanagedVolume.getInitiatorUris().add(initUri);
                                Initiator init = _dbClient.queryObject(Initiator.class, URI.create(initUri));
                                unmanagedVolume.getInitiatorNetworkIds().add(init.getInitiatorPort());
                            }
                            // log this info for debugging
                            for (String path : uem.getUnmanagedInitiatorNetworkIds()) {
                                s_logger.info("   UnManagedExportMask has this initiator unknown to ViPR: {}", path);
                            }
                            // volume if it is
                            if (!recoverPointExportMasks.isEmpty() && recoverPointExportMasks.contains(uem.getId().toString())) {
                                s_logger.info("unmanaged volume {} is an RP volume", unmanagedVolume.getLabel());
                                unmanagedVolume.putVolumeCharacterstics(SupportedVolumeCharacterstics.IS_RECOVERPOINT_ENABLED.toString(), TRUE);
                            } else {
                                // this volume is contained in at least one export mask that is non-RP
                                nonRpExported = true;
                            }
                        }
                    }
                    persistUnManagedExportMasks(null, unmanagedExportMasksToUpdate, false);
                    // this as a convenience to ingest features.
                    if (nonRpExported) {
                        s_logger.info("unmanaged volume {} is exported to something other than RP.  Marking IS_NONRP_EXPORTED.", unmanagedVolume.getLabel());
                        unmanagedVolume.putVolumeCharacterstics(SupportedVolumeCharacterstics.IS_NONRP_EXPORTED.toString(), TRUE);
                        unmanagedVolume.putVolumeCharacterstics(SupportedVolumeCharacterstics.IS_VOLUME_EXPORTED.toString(), TRUE);
                    } else {
                        s_logger.info("unmanaged volume {} is not exported OR not exported to something other than RP.  Not marking IS_NONRP_EXPORTED.", unmanagedVolume.getLabel());
                        unmanagedVolume.putVolumeCharacterstics(SupportedVolumeCharacterstics.IS_NONRP_EXPORTED.toString(), FALSE);
                        unmanagedVolume.putVolumeCharacterstics(SupportedVolumeCharacterstics.IS_VOLUME_EXPORTED.toString(), FALSE);
                    }
                    persistUnManagedVolumes(newUnmanagedVolumes, knownUnmanagedVolumes, false);
                } else {
                    s_logger.info("Virtual Volume {} is already managed by ViPR", managedVolume.forDisplay());
                    Long currentCapacity = info.getCapacityBytes();
                    if (currentCapacity != null && currentCapacity > managedVolume.getCapacity()) {
                        // update the managed volume's capacity if it changed. this could possibly happen
                        // if the volume were expanded and the final status was not processed successfully by ViPR due to timeout
                        s_logger.info("Virtual Volume {} capacity on VPLEX is different ({}) than in database ({}), updating...", managedVolume.forDisplay(), info.getCapacityBytes(), managedVolume.getCapacity());
                        managedVolume.setAllocatedCapacity(Long.parseLong(String.valueOf(0)));
                        managedVolume.setProvisionedCapacity(currentCapacity);
                        managedVolume.setCapacity(currentCapacity);
                        _dbClient.updateObject(managedVolume);
                    }
                }
                if (null != unmanagedVolume && !unmanagedVolume.getInactive()) {
                    allUnmanagedVolumes.add(unmanagedVolume.getId());
                }
                tracker.volumeTimeResults.put(name, System.currentTimeMillis() - timer);
                tracker.totalVolumesDiscovered++;
                s_logger.info("estimated discovery time remaining: " + tracker.getDiscoveryTimeRemaining());
            }
        } else {
            s_logger.warn("No virtual volumes were found on VPLEX.");
        }
        persistUnManagedVolumes(newUnmanagedVolumes, knownUnmanagedVolumes, true);
        persistUnManagedExportMasks(null, unmanagedExportMasksToUpdate, true);
        cleanUpOrphanedVolumes(vplex.getId(), allUnmanagedVolumes);
        // this has to happen at the very end so that the map is complete,
        // and by supplying the vplex id, we'll re-fetch all the volumes
        // now that everything has been persisted and orphans cleared out
        processBackendClones(vplex.getId(), backendVolumeGuidToVvolGuidMap);
    } catch (Exception ex) {
        s_logger.error("An error occurred during VPLEX unmanaged volume discovery", ex);
        String vplexLabel = vplexUri.toString();
        if (null != vplex) {
            vplexLabel = vplex.getLabel();
        }
        throw VPlexCollectionException.exceptions.vplexUnmanagedVolumeDiscoveryFailed(vplexLabel, ex.toString());
    } finally {
        if (null != vplex) {
            try {
                vplex.setLastDiscoveryStatusMessage(statusMessage);
                _dbClient.updateObject(vplex);
            } catch (Exception ex) {
                s_logger.error("Error while saving VPLEX discovery status message: {} - Exception: {}", statusMessage, ex.getLocalizedMessage());
            }
        }
    }
}
Also used : HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) ArrayList(java.util.ArrayList) URI(java.net.URI) VPlexVirtualVolumeInfo(com.emc.storageos.vplex.api.VPlexVirtualVolumeInfo) Initiator(com.emc.storageos.db.client.model.Initiator) StorageSystem(com.emc.storageos.db.client.model.StorageSystem) HashSet(java.util.HashSet) UnManagedExportMask(com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedExportMask) VirtualPool(com.emc.storageos.db.client.model.VirtualPool) URISyntaxException(java.net.URISyntaxException) DatabaseException(com.emc.storageos.db.exceptions.DatabaseException) BaseCollectionException(com.emc.storageos.plugins.BaseCollectionException) VPlexApiException(com.emc.storageos.vplex.api.VPlexApiException) VPlexCollectionException(com.emc.storageos.plugins.metering.vplex.VPlexCollectionException) IOException(java.io.IOException) UnManagedVolume(com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedVolume) Volume(com.emc.storageos.db.client.model.Volume) UnManagedVolume(com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedVolume) VPlexConsistencyGroupInfo(com.emc.storageos.vplex.api.VPlexConsistencyGroupInfo)

Example 2 with VPlexConsistencyGroupInfo

use of com.emc.storageos.vplex.api.VPlexConsistencyGroupInfo in project coprhd-controller by CoprHD.

the class RPVplexConsistencyGroupManager method getClusterConsistencyGroup.

/**
 * Maps a VPlex cluster/consistency group to its volumes.
 *
 * @param vplexVolume The virtual volume from which to obtain the VPlex cluster.
 * @param clusterConsistencyGroupVolumes The map to store cluster/cg/volume relationships.
 * @param cgName The VPlex consistency group name.
 * @throws Exception
 */
@Override
public ClusterConsistencyGroupWrapper getClusterConsistencyGroup(Volume vplexVolume, BlockConsistencyGroup cg) throws Exception {
    ClusterConsistencyGroupWrapper clusterConsistencyGroup = new ClusterConsistencyGroupWrapper();
    String vplexCgName = null;
    // Find the correct VPLEX cluster for this volume
    String vplexCluster = VPlexControllerUtils.getVPlexClusterName(dbClient, vplexVolume);
    // Determine if the volume is distributed
    boolean distributed = false;
    StringSet assocVolumes = vplexVolume.getAssociatedVolumes();
    // Associated volume for the consistency group cannot be null, indicates back-end volumes are not ingested.
    if (vplexVolume.getAssociatedVolumes() != null && !vplexVolume.getAssociatedVolumes().isEmpty()) {
        if (assocVolumes.size() > 1) {
            distributed = true;
        }
    } else {
        String reason = "Associated volume is empty";
        throw VPlexApiException.exceptions.emptyAssociatedVolumes(vplexVolume.getDeviceLabel(), vplexCluster, reason);
    }
    // Keep a reference to the VPLEX
    URI vplexURI = vplexVolume.getStorageController();
    log.info(String.format("Find correct VPLEX CG for VPLEX %s volume [%s](%s) at cluster [%s] on VPLEX (%s)", (distributed ? "distribitued" : "local"), vplexVolume.getLabel(), vplexVolume.getId(), vplexCluster, vplexURI));
    // line up the CG name from the ViPR CG to the VPLEX CGs.
    if (cg.created(vplexURI)) {
        log.info("CG already exists on VPLEX, but we need to figure out the correct one to use...");
        List<String> validVPlexCGsForCluster = new ArrayList<String>();
        // Extract all the CG names for the VPLEX
        // These names are in the format of: cluster-1:cgName or cluster-2:cgName
        StringSet cgNames = cg.getSystemConsistencyGroups().get(vplexURI.toString());
        // Iterate over the names to line up with the cluster
        Iterator<String> cgNamesIter = cgNames.iterator();
        while (cgNamesIter.hasNext()) {
            String clusterCgName = cgNamesIter.next();
            String cluster = BlockConsistencyGroupUtils.fetchClusterName(clusterCgName);
            String cgName = BlockConsistencyGroupUtils.fetchCgName(clusterCgName);
            // We will narrow this down afterwards for distributed.
            if (vplexCluster.equals(cluster) || distributed) {
                validVPlexCGsForCluster.add(cgName);
            }
        }
        // Get the API client.
        StorageSystem vplexSystem = getDataObject(StorageSystem.class, vplexURI, dbClient);
        VPlexApiClient client = getVPlexAPIClient(vplexApiFactory, vplexSystem, dbClient);
        log.info("Got VPLEX API client.");
        // This is not ideal but we need to call the VPlex client to fetch all consistency
        // groups so we can find the exact one we are looking for.
        List<VPlexConsistencyGroupInfo> vplexCGs = client.getConsistencyGroups();
        log.info("Iterating over returned VPLEX CGs to find the one we should use...");
        for (VPlexConsistencyGroupInfo vplexCG : vplexCGs) {
            boolean cgNameFound = validVPlexCGsForCluster.contains(vplexCG.getName());
            boolean volumeFound = vplexCG.getVirtualVolumes().contains(vplexVolume.getDeviceLabel());
            // either way it's valid.
            if (cgNameFound || volumeFound) {
                // Determine if the VPLEX CG is distributed.
                // NOTE: VPlexConsistencyGroupInfo.isDistributed() is not returning
                // the correct information. This is probably due to teh fact that
                // visibleClusters is not being returned in the response.
                // Instead we are checking getStorageAtClusters().size().
                boolean vplexCGIsDistributed = (vplexCG.getStorageAtClusters().size() > 1);
                if ((distributed && vplexCGIsDistributed) || (!distributed && !vplexCGIsDistributed)) {
                    log.info(String.format("Found correct VPLEX CG: %s", vplexCG.getName()));
                    vplexCgName = vplexCG.getName();
                    break;
                }
            }
        }
    }
    // The format for VPLEX local: cgName-vplexCluster
    if (vplexCgName == null) {
        // If we do not have a VPLEX CG name it's time to create one.
        if (distributed) {
            // Add '-dist' to the end of the CG name for distributed consistency groups.
            vplexCgName = cg.getLabel() + "-dist";
        } else {
            vplexCgName = cg.getLabel() + "-" + vplexCluster;
        }
        log.info(String.format("There was no existing VPLEX CG, created CG name: %s", vplexCgName));
    }
    clusterConsistencyGroup.setClusterName(vplexCluster);
    clusterConsistencyGroup.setCgName(vplexCgName);
    clusterConsistencyGroup.setDistributed(distributed);
    return clusterConsistencyGroup;
}
Also used : ClusterConsistencyGroupWrapper(com.emc.storageos.volumecontroller.impl.utils.ClusterConsistencyGroupWrapper) VPlexApiClient(com.emc.storageos.vplex.api.VPlexApiClient) VPlexConsistencyGroupInfo(com.emc.storageos.vplex.api.VPlexConsistencyGroupInfo) StringSet(com.emc.storageos.db.client.model.StringSet) ArrayList(java.util.ArrayList) URI(java.net.URI) StorageSystem(com.emc.storageos.db.client.model.StorageSystem)

Example 3 with VPlexConsistencyGroupInfo

use of com.emc.storageos.vplex.api.VPlexConsistencyGroupInfo in project coprhd-controller by CoprHD.

the class RPVplexConsistencyGroupManager method modifyCGSettings.

/**
 * @param client
 * @param cgName
 * @param clusterName
 * @param isDistributed
 */
private void modifyCGSettings(VPlexApiClient client, String cgName, String clusterName, boolean isDistributed) {
    log.info(String.format("VPlex consistency group %s already exists on cluster %s.", cgName, clusterName));
    // See if the CG is created but contains no volumes. This CG may have existed
    // previously for other volumes, but the volumes were deleted and removed from the CG.
    // The visibility and cluster info would have been set for those volumes and may not be
    // appropriate for these volumes.
    boolean cgContainsVolumes = false;
    // This is not ideal but we need to call the VPlex client to fetch all consistency
    // groups so we can find the one we are looking for. We need to see if there are
    // any associated virtual volumes. The BlockServiceApiImpl code associates the volumes
    // with the BlockConsistencyGroup even though these volumes are not yet part of
    // the VPlex consistency group. Might be worth changing this logic.
    List<VPlexConsistencyGroupInfo> consistencyGroupsInfo = client.getConsistencyGroups();
    for (VPlexConsistencyGroupInfo consistencyGroupInfo : consistencyGroupsInfo) {
        if (consistencyGroupInfo.getName().equals(cgName) && consistencyGroupInfo.getClusterName().equals(clusterName)) {
            if (consistencyGroupInfo.getVirtualVolumes() != null && !consistencyGroupInfo.getVirtualVolumes().isEmpty()) {
                cgContainsVolumes = true;
            }
            break;
        }
    }
    if (!cgContainsVolumes) {
        log.info("Updating VPLEX consistency group properties.");
        // Now we can update the consistency group properties.
        client.updateConsistencyGroupProperties(cgName, clusterName, isDistributed);
        log.info("Updated VPLEX consistency group properties.");
    }
}
Also used : VPlexConsistencyGroupInfo(com.emc.storageos.vplex.api.VPlexConsistencyGroupInfo)

Example 4 with VPlexConsistencyGroupInfo

use of com.emc.storageos.vplex.api.VPlexConsistencyGroupInfo in project coprhd-controller by CoprHD.

the class AbstractConsistencyGroupManager method getIsCGDistributed.

/**
 * Determine if the consistency group with the passed name on the passed cluster
 * is distributed.
 *
 * @param client A reference to a VPLEX client
 * @param cgName The consistency group name
 * @param cgCluster The consistency group cluster
 *
 * @return true if the consistency group is distributed, false otherwise.
 */
protected boolean getIsCGDistributed(VPlexApiClient client, String cgName, String cgCluster) {
    log.info("Determine if CG {} on cluster {} is distributed", cgName, cgCluster);
    boolean isDistributed = false;
    List<VPlexConsistencyGroupInfo> cgInfos = client.getConsistencyGroups();
    for (VPlexConsistencyGroupInfo cgInfo : cgInfos) {
        if ((cgInfo.getClusterName().equals(cgCluster)) && (cgInfo.getName().equals(cgName))) {
            log.info("CG is distributed");
            isDistributed = cgInfo.isDistributed();
        }
    }
    return isDistributed;
}
Also used : VPlexConsistencyGroupInfo(com.emc.storageos.vplex.api.VPlexConsistencyGroupInfo)

Aggregations

VPlexConsistencyGroupInfo (com.emc.storageos.vplex.api.VPlexConsistencyGroupInfo)4 StorageSystem (com.emc.storageos.db.client.model.StorageSystem)2 URI (java.net.URI)2 ArrayList (java.util.ArrayList)2 Initiator (com.emc.storageos.db.client.model.Initiator)1 StringSet (com.emc.storageos.db.client.model.StringSet)1 UnManagedExportMask (com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedExportMask)1 UnManagedVolume (com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedVolume)1 VirtualPool (com.emc.storageos.db.client.model.VirtualPool)1 Volume (com.emc.storageos.db.client.model.Volume)1 DatabaseException (com.emc.storageos.db.exceptions.DatabaseException)1 BaseCollectionException (com.emc.storageos.plugins.BaseCollectionException)1 VPlexCollectionException (com.emc.storageos.plugins.metering.vplex.VPlexCollectionException)1 ClusterConsistencyGroupWrapper (com.emc.storageos.volumecontroller.impl.utils.ClusterConsistencyGroupWrapper)1 VPlexApiClient (com.emc.storageos.vplex.api.VPlexApiClient)1 VPlexApiException (com.emc.storageos.vplex.api.VPlexApiException)1 VPlexVirtualVolumeInfo (com.emc.storageos.vplex.api.VPlexVirtualVolumeInfo)1 IOException (java.io.IOException)1 URISyntaxException (java.net.URISyntaxException)1 HashMap (java.util.HashMap)1