Examples with UnManagedExportMask com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedExportMask used on opensource projects

Example 11 with UnManagedExportMask

use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedExportMask in project coprhd-controller by CoprHD.

the class VolumeIngestionUtil method findUnManagedExportMasksForHost.

/**
 * Returns a List of the UnManagedExportMasks associated with a given Host URI.
 *
 * @param hostUri the Host URI to check
 * @param dbClient a reference to the database client
 * @return a List of the UnManagedExportMasks associated with a given Host URI
 */
public static List<UnManagedExportMask> findUnManagedExportMasksForHost(URI hostUri, DbClient dbClient) {
    _logger.info("finding unmanaged export masks for host " + hostUri);
    List<UnManagedExportMask> uems = new ArrayList<UnManagedExportMask>();
    List<Initiator> initiators = ComputeSystemHelper.queryInitiators(dbClient, hostUri);
    Set<URI> uemUris = new HashSet<URI>();
    URIQueryResultList results = new URIQueryResultList();
    for (Initiator initiator : initiators) {
        _logger.info("      looking at initiator " + initiator.getInitiatorPort());
        dbClient.queryByConstraint(AlternateIdConstraint.Factory.getUnManagedExportMaskKnownInitiatorConstraint(initiator.getInitiatorPort()), results);
        if (results.iterator() != null) {
            for (URI uri : results) {
                _logger.info("      found UnManagedExportMask " + uri);
                uemUris.add(uri);
            }
        }
    }
    for (URI uemUri : uemUris) {
        UnManagedExportMask uem = dbClient.queryObject(UnManagedExportMask.class, uemUri);
        if (uem == null || uem.getInactive() == true) {
            continue;
        }
        uems.add(uem);
        _logger.info("   maskName: " + uem.getMaskName() + " nativeGuid: " + uem.getNativeGuid());
    }
    if (uems.isEmpty()) {
        _logger.info("   did not find any unmanaged export masks for this host");
    }
    return uems;
}

Example 12 with UnManagedExportMask

use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedExportMask in project coprhd-controller by CoprHD.

the class ExternalDeviceUnManagedVolumeDiscoverer method updateUnManagedVolumesWithExportData.

/**
 * Update unmanaged volumes from export masks with export data.
 *
 * @param unManagedVolumeNativeIdToUriMap [IN] helper map
 * @param unManagedExportMasksToCreate [IN] list of masks with volumes
 * @param unManagedExportMasksToUpdate [IN] list of masks with volumes
 * @param dbClient reference to db client [IN]
 * @param partitionManager partition manager [IN]
 */
private void updateUnManagedVolumesWithExportData(Map<String, URI> unManagedVolumeNativeIdToUriMap, List<UnManagedExportMask> unManagedExportMasksToCreate, List<UnManagedExportMask> unManagedExportMasksToUpdate, DbClient dbClient, PartitionManager partitionManager) {
    // update unmanaged volumes with export data
    // helper map:
    Map<String, List<UnManagedExportMask>> volumeToMasksMap = new HashMap<>();
    // key --- volume uri, value: unmanaged masks for this volume.
    List<UnManagedVolume> unManagedVolumesToUpdate = new ArrayList<UnManagedVolume>();
    // build volume to mask map
    for (UnManagedExportMask mask : unManagedExportMasksToUpdate) {
        StringSet volumes = mask.getUnmanagedVolumeUris();
        for (String volumeUri : volumes) {
            List<UnManagedExportMask> volumeMasks = volumeToMasksMap.get(volumeUri);
            if (volumeMasks == null) {
                volumeMasks = new ArrayList<>();
                volumeToMasksMap.put(volumeUri, volumeMasks);
            }
            volumeMasks.add(mask);
        }
    }
    for (UnManagedExportMask mask : unManagedExportMasksToCreate) {
        StringSet volumes = mask.getUnmanagedVolumeUris();
        for (String volumeUri : volumes) {
            List<UnManagedExportMask> volumeMasks = volumeToMasksMap.get(volumeUri);
            if (volumeMasks == null) {
                volumeMasks = new ArrayList<>();
                volumeToMasksMap.put(volumeUri, volumeMasks);
            }
            volumeMasks.add(mask);
        }
    }
    for (URI volumeUri : unManagedVolumeNativeIdToUriMap.values()) {
        UnManagedVolume volume = dbClient.queryObject(UnManagedVolume.class, volumeUri);
        // Clean old export data
        volume.getInitiatorNetworkIds().clear();
        volume.getInitiatorUris().clear();
        volume.getUnmanagedExportMasks().clear();
        volume.getVolumeCharacterstics().put(UnManagedVolume.SupportedVolumeCharacterstics.IS_VOLUME_EXPORTED.toString(), FALSE);
        volume.getVolumeCharacterstics().put(UnManagedVolume.SupportedVolumeCharacterstics.IS_NONRP_EXPORTED.toString(), FALSE);
        List<UnManagedExportMask> volumeMasks = volumeToMasksMap.get(volumeUri.toString());
        if (volumeMasks != null && !volumeMasks.isEmpty()) {
            // update unmanaged volume with export data
            log.info("Updating volume {} with export data: {} .", volume.getNativeGuid(), volumeMasks);
            // set to  new data
            for (UnManagedExportMask mask : volumeMasks) {
                volume.getInitiatorNetworkIds().addAll(mask.getKnownInitiatorNetworkIds());
                volume.getInitiatorUris().addAll(mask.getKnownInitiatorUris());
                volume.getUnmanagedExportMasks().add(mask.getId().toString());
            }
            volume.getVolumeCharacterstics().put(UnManagedVolume.SupportedVolumeCharacterstics.IS_VOLUME_EXPORTED.toString(), TRUE);
            volume.getVolumeCharacterstics().put(UnManagedVolume.SupportedVolumeCharacterstics.IS_NONRP_EXPORTED.toString(), TRUE);
            unManagedVolumesToUpdate.add(volume);
        } else {
            log.info("Volume {} does not have export masks.", volume.getNativeGuid());
        }
    }
    if (!unManagedVolumesToUpdate.isEmpty()) {
        partitionManager.updateAndReIndexInBatches(unManagedVolumesToUpdate, Constants.DEFAULT_PARTITION_SIZE, dbClient, UNMANAGED_VOLUME);
        unManagedVolumesToUpdate.clear();
    }
}

Example 13 with UnManagedExportMask

use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedExportMask in project coprhd-controller by CoprHD.

the class ExternalDeviceUnManagedVolumeDiscoverer method determineUnManagedExportMasksForExportInfo.

/**
 * This method processes hostToVolumeExportInfoMap to find out which existing unmanaged masks has to be updated,
 * and which unmanaged masks have to be created new for this export info. It also identifies hosts with unsupported
 * export info data (exported host volumes are not seen through the same set of initiators and the same set of storage
 * ports --- which require more than one mask per host) and adds these hosts to invalidExportHosts set.
 *
 * @param storageSystem
 * @param hostToVolumeExportInfoMap [IN] map: key --- host FQDN, value --- list of volume export info instances
 * @param invalidExportHosts        [IN, OUT] set of invalid hosts, for which we skip export processing for a given array
 * @param dbClient                  reference to db client [IN]
 * @param masksToUpdateForVolumes   [OUT] map: key --- URI of existing unmanaged export mask, value --- export info to use
 *                                  to update the mask.
 * @param masksToCreateForVolumes   [OUT] list of export info instances for which we need to create new unmanaged masks.
 */
private void determineUnManagedExportMasksForExportInfo(com.emc.storageos.db.client.model.StorageSystem storageSystem, Map<String, List<HostExportInfo>> hostToVolumeExportInfoMap, Set<String> invalidExportHosts, DbClient dbClient, Map<URI, HostExportInfo> masksToUpdateForVolumes, List<HostExportInfo> masksToCreateForVolumes) {
    for (Map.Entry<String, List<HostExportInfo>> entry : hostToVolumeExportInfoMap.entrySet()) {
        String hostName = entry.getKey();
        log.info("Processing export info for host {} .", hostName);
        if (invalidExportHosts.contains(hostName)) {
            // skip and continue to the next host.
            log.info("Found host {} in invalid hosts list. We will not process this host export data.", hostName);
            continue;
        }
        List<HostExportInfo> volumeToHostExportInfoList = entry.getValue();
        log.info("Processing export info list {} .", volumeToHostExportInfoList);
        String maskName = getUnManagedMaskName(hostName, storageSystem.getNativeGuid());
        HostExportInfo hostExportInfo = verifyHostExports(volumeToHostExportInfoList);
        if (hostExportInfo == null) {
            // invalid, continue to the next host
            invalidExportHosts.add(hostName);
            log.info("Found export info for host {} invalid. We will not process this host export data.", hostName);
            // check existing UnManaged export mask for host/array: the mask could be discovered for volumes on previous
            // pages (all unmanaged masks from previous discovery have been deactivated at the begging).
            UnManagedExportMask unManagedMask = getUnManagedExportMask(maskName, dbClient, storageSystem.getId());
            if (unManagedMask != null) {
                log.info("Found existing unmanaged export mask for host {} and array {} --- {} . We will deactivate this mask.", hostName, storageSystem.getNativeId(), unManagedMask);
                removeInvalidMaskDataFromVolumes(unManagedMask, dbClient);
                unManagedMask.setInactive(true);
                dbClient.updateObject(unManagedMask);
            }
            continue;
        }
        log.info("The result export info for host {} and array {} : {} .", hostName, storageSystem.getNativeId(), hostExportInfo);
        // check existing UnManaged export mask for host/array: the mask could be discovered for volumes on previous
        // pages (all unmanaged masks from previous discovery have been deactivated at the begging).
        UnManagedExportMask unManagedMask = getUnManagedExportMask(maskName, dbClient, storageSystem.getId());
        boolean isValid = true;
        if (unManagedMask != null) {
            log.info("Found existing unmanaged export mask for host {} and array {} --- {} .", hostName, storageSystem.getNativeId(), unManagedMask);
            // check that existing host/array unManaged export mask has the same set of initiators and the same
            // set of ports as new discovered hostExportInfo
            StringSet storagePortsUris = unManagedMask.getKnownStoragePortUris();
            Set<String> storagePortsNativeIds = new HashSet<>();
            Set<String> initiatorsNativeIds = new HashSet<>();
            for (String portUriString : storagePortsUris) {
                URI portUri = URI.create(portUriString);
                com.emc.storageos.db.client.model.StoragePort port = dbClient.queryObject(com.emc.storageos.db.client.model.StoragePort.class, portUri);
                storagePortsNativeIds.add(port.getNativeId());
            }
            storagePortsNativeIds.addAll(unManagedMask.getUnmanagedStoragePortNetworkIds());
            initiatorsNativeIds.addAll(unManagedMask.getKnownInitiatorNetworkIds());
            initiatorsNativeIds.addAll(unManagedMask.getUnmanagedInitiatorNetworkIds());
            isValid = verifyHostExports(initiatorsNativeIds, storagePortsNativeIds, hostExportInfo);
            if (!isValid) {
                // Invalid, we deactivate existing unmanaged mask --- make sure we do not discover invalid export
                // masks. We also, remove this mask from "unmanagedExportMasks" set in its unmanaged storage volumes.
                log.info("The result export info for host {} and storage array {} does not comply with existing mask.", hostName, storageSystem.getNativeId());
                removeInvalidMaskDataFromVolumes(unManagedMask, dbClient);
                unManagedMask.setInactive(true);
                dbClient.updateObject(unManagedMask);
            }
        } else {
            // Check if export mask for host/array is already managed. If host/array mask is managed, check that hostExportInfo has the same
            // storage ports and the same host initiators as in the managed mask. If we have a match for ports/initiators between the mask and hostExportInfo, we will process this
            // host export info and create a new UnManagedExportMask for the host.
            log.info("There is no existing unmanaged export mask for host {} and array {} .", hostName, storageSystem.getNativeId());
            List<String> initiatorPorts = new ArrayList<>();
            for (Initiator initiator : hostExportInfo.getInitiators()) {
                initiatorPorts.add(initiator.getPort());
            }
            // We enforce single export mask for host/array for ingested masks, so if only one initiator port match, the mask is a match.
            Map<URI, ExportMask> uriToExportMask = ExportMaskUtils.getExportMasksWithInitiatorPorts(dbClient, initiatorPorts);
            // Look for export mask for the storage system under processing.
            for (ExportMask mask : uriToExportMask.values()) {
                if (URIUtil.identical(mask.getStorageDevice(), storageSystem.getId())) {
                    // found managed export mask for storage system and host initiator
                    // the mask is already managed.
                    log.info("Found managed export mask for host {} and array {} --- {}." + " We will process this host export data to see if we can add volumes to this mask.", hostName, storageSystem.getNativeId(), mask.getId());
                    // check that this managed mask has the same initiators and ports as in the hostExportInfo
                    StringSet storagePortsUris = mask.getStoragePorts();
                    StringSet initiatorsUris = mask.getInitiators();
                    List<com.emc.storageos.db.client.model.StoragePort> ports = dbClient.queryObjectField(com.emc.storageos.db.client.model.StoragePort.class, "nativeId", StringSetUtil.stringSetToUriList(storagePortsUris));
                    List<com.emc.storageos.db.client.model.Initiator> initiators = dbClient.queryObjectField(com.emc.storageos.db.client.model.Initiator.class, "iniport", StringSetUtil.stringSetToUriList(initiatorsUris));
                    Set<String> maskStoragePortsNativeIds = new HashSet<>();
                    Set<String> maskInitiatorPorts = new HashSet<>();
                    for (com.emc.storageos.db.client.model.StoragePort storagePort : ports) {
                        maskStoragePortsNativeIds.add(storagePort.getNativeId());
                    }
                    for (com.emc.storageos.db.client.model.Initiator initiator : initiators) {
                        maskInitiatorPorts.add(initiator.getInitiatorPort());
                    }
                    log.info("Managed ExportMask {} has the following storage ports {}", mask.getId(), maskStoragePortsNativeIds);
                    log.info("Managed ExportMask {} has the following initiator ports {}", mask.getId(), maskInitiatorPorts);
                    // check that hostExportInfo has the same ports and initiators as in the export mask
                    isValid = verifyHostExports(maskInitiatorPorts, maskStoragePortsNativeIds, hostExportInfo);
                    if (isValid) {
                        // we will create unmanaged mask for this hostExportInfo
                        // we rely on ingestion to add new volumes to the managed mask.
                        log.info("Managed export mask {} has the same initiators and ports as in hostExportInfo. We will create unmanaged mask for new volumes.", mask.getId());
                        break;
                    } else {
                        log.info("Managed export mask {} has different initiators or ports as those in hostExportInfo.", mask.getId());
                    }
                }
            }
        }
        if (!isValid) {
            // invalid, continue to the next host
            // add host to invalid hosts list, so we do not process any export volume
            // info for this host in the future (for volumes found on next pages).
            log.info("Found export info for host {} invalid. Export info: {}." + " We will not process this host export data.", hostName, hostExportInfo);
            invalidExportHosts.add(hostName);
            continue;
        }
        if (unManagedMask != null) {
            // we will update this mask with additional volumes.
            URI maskId = unManagedMask.getId();
            masksToUpdateForVolumes.put(maskId, hostExportInfo);
        } else {
            // we will create new unManaged mask for host/array.
            masksToCreateForVolumes.add(hostExportInfo);
        }
    }
}

Example 14 with UnManagedExportMask

use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedExportMask in project coprhd-controller by CoprHD.

the class VPlexCommunicationInterface method discover.

/**
 * Implementation for discovery of VPLEX storage systems.
 *
 * @param accessProfile providing context for this discovery session
 *
 * @throws BaseCollectionException
 */
@Override
public void discover(AccessProfile accessProfile) throws BaseCollectionException {
    s_logger.info("initiating discovery of VPLEX system {}", accessProfile.getProfileName());
    if ((null != accessProfile.getnamespace()) && (accessProfile.getnamespace().equals(StorageSystem.Discovery_Namespaces.UNMANAGED_VOLUMES.toString()))) {
        try {
            VPlexApiClient client = getVPlexAPIClient(accessProfile);
            long timer = System.currentTimeMillis();
            UnmanagedDiscoveryPerformanceTracker tracker = new UnmanagedDiscoveryPerformanceTracker();
            tracker.discoveryMode = ControllerUtils.getPropertyValueFromCoordinator(_coordinator, VplexBackendIngestionContext.DISCOVERY_MODE);
            // get all the detailed virtual volume info from the VPLEX API
            Map<String, VPlexVirtualVolumeInfo> vvolMap = client.getVirtualVolumes(true);
            tracker.virtualVolumeFetch = System.currentTimeMillis() - timer;
            tracker.totalVolumesFetched = vvolMap.size();
            // discover unmanaged storage views
            timer = System.currentTimeMillis();
            Map<String, Set<UnManagedExportMask>> volumeToExportMasksMap = new HashMap<String, Set<UnManagedExportMask>>();
            Map<String, Set<VPlexStorageViewInfo>> volumeToStorageViewMap = new HashMap<String, Set<VPlexStorageViewInfo>>();
            Set<String> recoverPointExportMasks = new HashSet<String>();
            discoverUnmanagedStorageViews(accessProfile, client, vvolMap, volumeToExportMasksMap, volumeToStorageViewMap, recoverPointExportMasks);
            tracker.storageViewFetch = System.currentTimeMillis() - timer;
            // discover unmanaged volumes
            timer = System.currentTimeMillis();
            discoverUnmanagedVolumes(accessProfile, client, vvolMap, volumeToExportMasksMap, volumeToStorageViewMap, recoverPointExportMasks, tracker);
            tracker.unmanagedVolumeProcessing = System.currentTimeMillis() - timer;
            // re-run vpool matching for all vpools so that backend volumes will
            // be updated after vvol discovery to match their parent's matched vpools
            timer = System.currentTimeMillis();
            List<URI> vpoolURIs = _dbClient.queryByType(VirtualPool.class, true);
            List<VirtualPool> vpoolList = _dbClient.queryObject(VirtualPool.class, vpoolURIs);
            Set<URI> srdfEnabledTargetVPools = SRDFUtils.fetchSRDFTargetVirtualPools(_dbClient);
            Set<URI> rpEnabledTargetVPools = RPHelper.fetchRPTargetVirtualPools(_dbClient);
            for (VirtualPool vpool : vpoolList) {
                ImplicitUnManagedObjectsMatcher.matchVirtualPoolsWithUnManagedVolumes(vpool, srdfEnabledTargetVPools, rpEnabledTargetVPools, _dbClient, true);
            }
            tracker.vpoolMatching = System.currentTimeMillis() - timer;
            s_logger.info(tracker.getPerformanceReport());
        } catch (URISyntaxException ex) {
            s_logger.error(ex.getLocalizedMessage());
            throw VPlexCollectionException.exceptions.vplexUnmanagedVolumeDiscoveryFailed(accessProfile.getSystemId().toString(), ex.getLocalizedMessage());
        }
    } else {
        discoverAll(accessProfile);
    }
}

Example 15 with UnManagedExportMask

use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedExportMask in project coprhd-controller by CoprHD.

the class VPlexCommunicationInterface method cleanUpOrphanedExportMasks.

/**
 * Cleans up any UnManagedExportMask objects that are present in the ViPR database,
 * but are no longer present on the VPLEX device.
 *
 * @param vplexUri device id of the VPLEX
 * @param allCurrentUnManagedExportMaskUris all the UnManagedExportMasks we found in this discovery run
 */
private void cleanUpOrphanedExportMasks(URI vplexUri, Set<URI> allCurrentUnManagedExportMaskUris) {
    URIQueryResultList result = new URIQueryResultList();
    _dbClient.queryByConstraint(ContainmentConstraint.Factory.getStorageSystemUnManagedExportMaskConstraint(vplexUri), result);
    Set<URI> allMasksInDatabase = new HashSet<URI>();
    Iterator<URI> it = result.iterator();
    while (it.hasNext()) {
        allMasksInDatabase.add(it.next());
    }
    SetView<URI> onlyAvailableinDB = Sets.difference(allMasksInDatabase, allCurrentUnManagedExportMaskUris);
    if (!onlyAvailableinDB.isEmpty()) {
        s_logger.info("these UnManagedExportMasks are orphaned and will be cleaned up:" + Joiner.on("\t").join(onlyAvailableinDB));
        List<UnManagedExportMask> unManagedExportMasksToBeDeleted = new ArrayList<UnManagedExportMask>();
        Iterator<UnManagedExportMask> unManagedExportMasks = _dbClient.queryIterativeObjects(UnManagedExportMask.class, new ArrayList<URI>(onlyAvailableinDB));
        while (unManagedExportMasks.hasNext()) {
            UnManagedExportMask uem = unManagedExportMasks.next();
            if (null == uem || uem.getInactive()) {
                continue;
            }
            s_logger.info("Setting UnManagedExportMask {} inactive", uem.getMaskingViewPath());
            uem.setStorageSystemUri(NullColumnValueGetter.getNullURI());
            uem.setInactive(true);
            unManagedExportMasksToBeDeleted.add(uem);
        }
        if (!unManagedExportMasksToBeDeleted.isEmpty()) {
            _partitionManager.updateAndReIndexInBatches(unManagedExportMasksToBeDeleted, BATCH_SIZE, _dbClient, UNMANAGED_EXPORT_MASK);
        }
    }
}