Examples with UnManagedConsistencyGroup com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedConsistencyGroup used on opensource projects

Example 11 with UnManagedConsistencyGroup

use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedConsistencyGroup in project coprhd-controller by CoprHD.

the class ExternalDeviceUnManagedVolumeDiscoverer method addObjectToUnManagedConsistencyGroup.

/**
 * Add storage object to unManaged consistency group.
 * Sets consistency group related attributes in the object and adds object to the list of unManaged
 * objects in the unManaged consistency group instance.
 *
 * @param storageSystem             storage system of the object [IN]
 * @param cgNativeId                native id of umanaged consistency group [IN]
 * @param unManagedVolume           unManaged object [IN/OUT] unmanaged obect (volume/snap/clone) with CG information
 * @param allCurrentUnManagedCgURIs set of unManaged CG uris found in the current discovery [OUT]
 * @param unManagedCGToUpdateMap    map of unManaged CG GUID to unManaged CG instance [IN/OUT]
 * @param driver                    storage driver [IN]
 * @param dbClient                  reference to db client [IN]
 * @throws Exception
 */
private void addObjectToUnManagedConsistencyGroup(com.emc.storageos.db.client.model.StorageSystem storageSystem, String cgNativeId, UnManagedVolume unManagedVolume, Set<URI> allCurrentUnManagedCgURIs, Map<String, UnManagedConsistencyGroup> unManagedCGToUpdateMap, BlockStorageDriver driver, DbClient dbClient) throws Exception {
    log.info("UnManaged storage object {} belongs to consistency group {} on the array", unManagedVolume.getLabel(), cgNativeId);
    // determine the native guid for the unManaged CG
    String unManagedCGNativeGuid = NativeGUIDGenerator.generateNativeGuidForCG(storageSystem.getNativeGuid(), cgNativeId);
    log.info("UnManaged consistency group has nativeGuid {} ", unManagedCGNativeGuid);
    // determine if the unManaged CG already exists in the unManagedCGToUpdateMap or in the database
    // if the the unManaged CG is not in either, create a new one
    UnManagedConsistencyGroup unManagedCG = null;
    if (unManagedCGToUpdateMap.containsKey(unManagedCGNativeGuid)) {
        unManagedCG = unManagedCGToUpdateMap.get(unManagedCGNativeGuid);
        log.info("UnManaged consistency group {} was previously added to the unManagedCGToUpdateMap", unManagedCG.getNativeGuid());
    } else {
        unManagedCG = DiscoveryUtils.checkUnManagedCGExistsInDB(dbClient, unManagedCGNativeGuid);
        if (null == unManagedCG) {
            // unManaged CG does not exist in the database, create it
            VolumeConsistencyGroup driverCG = driver.getStorageObject(storageSystem.getNativeId(), cgNativeId, VolumeConsistencyGroup.class);
            if (driverCG != null) {
                unManagedCG = createUnManagedCG(driverCG, storageSystem, dbClient);
                log.info("Created unManaged consistency group: {} with nativeGuid {}", unManagedCG.getId().toString(), unManagedCG.getNativeGuid());
            } else {
                String msg = String.format("Driver VolumeConsistencyGroup with native id %s does not exist on storage system %s", cgNativeId, storageSystem.getNativeId());
                log.error(msg);
                throw new Exception(msg);
            }
        } else {
            log.info("UnManaged consistency group {} was previously added to the database (by previous unManaged discovery).", unManagedCG.getNativeGuid());
            // clean out the list of unManaged objects if this unManaged cg was already
            // in the database and its first time being used in this discovery operation
            // the list should be re-populated by the current discovery operation
            log.info("Cleaning out unManaged object map from unManaged consistency group: {}", unManagedCG.getNativeGuid());
            unManagedCG.getUnManagedVolumesMap().clear();
        }
    }
    log.info("Adding unManaged storage object {} to unManaged consistency group {}", unManagedVolume.getLabel(), unManagedCG.getNativeGuid());
    // Update the unManagedVolume object with CG information
    unManagedVolume.getVolumeCharacterstics().put(UnManagedVolume.SupportedVolumeCharacterstics.IS_VOLUME_ADDED_TO_CONSISTENCYGROUP.toString(), Boolean.TRUE.toString());
    // set the uri of the unManaged CG in the unManaged volume object
    unManagedVolume.getVolumeInformation().remove(UnManagedVolume.SupportedVolumeInformation.UNMANAGED_CONSISTENCY_GROUP_URI.toString());
    unManagedVolume.getVolumeInformation().put(UnManagedVolume.SupportedVolumeInformation.UNMANAGED_CONSISTENCY_GROUP_URI.toString(), unManagedCG.getId().toString());
    // add the unManaged volume object to the unManaged CG
    unManagedCG.getUnManagedVolumesMap().put(unManagedVolume.getNativeGuid(), unManagedVolume.getId().toString());
    // add the unManaged CG to the map of unManaged CGs to be updated in the database once all volumes have been processed
    unManagedCGToUpdateMap.put(unManagedCGNativeGuid, unManagedCG);
    // add the unManaged CG to the current set of CGs being discovered on the array. This is for book keeping later.
    allCurrentUnManagedCgURIs.add(unManagedCG.getId());
}

Example 12 with UnManagedConsistencyGroup

use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedConsistencyGroup in project coprhd-controller by CoprHD.

the class ExternalDeviceUnManagedVolumeDiscoverer method createUnManagedCG.

/**
 * Create unManaged CG for a given driver CG.
 *
 * @param driverCG reference to native CG
 * @param storageSystem storage system
 * @param dbClient reference to db client [IN]
 * @return umanaged CG
 */
private UnManagedConsistencyGroup createUnManagedCG(VolumeConsistencyGroup driverCG, com.emc.storageos.db.client.model.StorageSystem storageSystem, DbClient dbClient) {
    UnManagedConsistencyGroup unManagedCG = new UnManagedConsistencyGroup();
    unManagedCG.setId(URIUtil.createId(UnManagedConsistencyGroup.class));
    unManagedCG.setLabel(driverCG.getDeviceLabel());
    unManagedCG.setName(driverCG.getDeviceLabel());
    String unManagedCGNativeGuid = NativeGUIDGenerator.generateNativeGuidForCG(storageSystem.getNativeGuid(), driverCG.getNativeId());
    unManagedCG.setNativeGuid(unManagedCGNativeGuid);
    unManagedCG.setNativeId(driverCG.getNativeId());
    unManagedCG.setStorageSystemUri(storageSystem.getId());
    dbClient.createObject(unManagedCG);
    return unManagedCG;
}

Example 13 with UnManagedConsistencyGroup

use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedConsistencyGroup in project coprhd-controller by CoprHD.

the class DiscoveryUtils method checkUnManagedCGExistsInDB.

/**
 * Determines if the UnManagedConsistencyGroup object exists in the database
 *
 * @param nativeGuid - native Guid for the unmanaged consistency group
 * @param dbClient - database client
 * @return unmanagedCG - null if it does not exist in the database, otherwise it returns the
 *         UnManagedConsistencyGroup object from the database
 * @throws IOException
 */
public static UnManagedConsistencyGroup checkUnManagedCGExistsInDB(DbClient dbClient, String nativeGuid) {
    UnManagedConsistencyGroup unmanagedCG = null;
    URIQueryResultList unManagedCGList = new URIQueryResultList();
    dbClient.queryByConstraint(AlternateIdConstraint.Factory.getCGInfoNativeIdConstraint(nativeGuid), unManagedCGList);
    if (unManagedCGList.iterator().hasNext()) {
        URI unManagedCGURI = unManagedCGList.iterator().next();
        unmanagedCG = dbClient.queryObject(UnManagedConsistencyGroup.class, unManagedCGURI);
    }
    return unmanagedCG;
}

Example 14 with UnManagedConsistencyGroup

use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedConsistencyGroup in project coprhd-controller by CoprHD.

the class VNXUnityUnManagedObjectDiscoverer method addObjectToUnManagedConsistencyGroup.

/**
 * Adds the passed in unmanaged volume to an unmanaged consistency group object
 *
 * @param apiClient
 *            - connection to Unity REST interface
 * @param unManagedVolume
 *            - unmanaged volume associated with a consistency group
 * @param cgNameToProcess
 *            - consistency group being processed
 * @param storageSystem
 *            - storage system the objects are on
 * @param dbClient
 *            - dbclient
 * @throws Exception
 */
private void addObjectToUnManagedConsistencyGroup(VNXeApiClient apiClient, UnManagedVolume unManagedVolume, String cgNameToProcess, StorageSystem storageSystem, DbClient dbClient) throws Exception {
    log.info("Unmanaged volume {} belongs to consistency group {} on the array", unManagedVolume.getLabel(), cgNameToProcess);
    // Update the unManagedVolume object with CG information
    unManagedVolume.getVolumeCharacterstics().put(SupportedVolumeCharacterstics.IS_VOLUME_ADDED_TO_CONSISTENCYGROUP.toString(), Boolean.TRUE.toString());
    String unManagedCGNativeGuid = NativeGUIDGenerator.generateNativeGuidForCG(storageSystem.getNativeGuid(), cgNameToProcess);
    // determine if the unmanaged CG already exists in the unManagedCGToUpdateMap or in the database
    // if the the unmanaged CG is not in either create a new one
    UnManagedConsistencyGroup unManagedCG = null;
    if (unManagedCGToUpdateMap.containsKey(unManagedCGNativeGuid)) {
        unManagedCG = unManagedCGToUpdateMap.get(unManagedCGNativeGuid);
        log.info("Unmanaged consistency group {} was previously added to the unManagedCGToUpdateMap", unManagedCG.getLabel());
    } else {
        unManagedCG = DiscoveryUtils.checkUnManagedCGExistsInDB(dbClient, unManagedCGNativeGuid);
        if (null == unManagedCG) {
            // unmanaged CG does not exist in the database, create it
            StorageResource res = apiClient.getStorageResource(cgNameToProcess);
            unManagedCG = createUnManagedCG(unManagedCGNativeGuid, res, storageSystem.getId(), dbClient);
            log.info("Created unmanaged consistency group: {}", unManagedCG.getId().toString());
        } else {
            log.info("Unmanaged consistency group {} was previously added to the database", unManagedCG.getLabel());
            // clean out the list of unmanaged volumes if this unmanaged cg was already
            // in the database and its first time being used in this discovery operation
            // the list should be re-populated by the current discovery operation
            log.info("Cleaning out unmanaged volume map from unmanaged consistency group: {}", unManagedCG.getLabel());
            unManagedCG.getUnManagedVolumesMap().clear();
        }
    }
    log.info("Adding unmanaged volume {} to unmanaged consistency group {}", unManagedVolume.getLabel(), unManagedCG.getLabel());
    // set the uri of the unmanaged CG in the unmanaged volume object
    unManagedVolume.getVolumeInformation().put(SupportedVolumeInformation.UNMANAGED_CONSISTENCY_GROUP_URI.toString(), unManagedCG.getId().toString());
    // add the unmanaged volume object to the unmanaged CG
    unManagedCG.getUnManagedVolumesMap().put(unManagedVolume.getNativeGuid(), unManagedVolume.getId().toString());
    // add the unmanaged CG to the map of unmanaged CGs to be updated in the database once all volumes have been
    // processed
    unManagedCGToUpdateMap.put(unManagedCGNativeGuid, unManagedCG);
    // add the unmanaged CG to the current set of CGs being discovered on the array. This is for book keeping later.
    allCurrentUnManagedCgURIs.add(unManagedCG.getId());
}

Example 15 with UnManagedConsistencyGroup

use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedConsistencyGroup in project coprhd-controller by CoprHD.

the class XtremIOUnManagedVolumeDiscoverer method discoverUnManagedObjects.

public void discoverUnManagedObjects(AccessProfile accessProfile, DbClient dbClient, PartitionManager partitionManager) throws Exception {
    log.info("Started discovery of UnManagedVolumes for system {}", accessProfile.getSystemId());
    StorageSystem storageSystem = dbClient.queryObject(StorageSystem.class, accessProfile.getSystemId());
    XtremIOClient xtremIOClient = XtremIOProvUtils.getXtremIOClient(dbClient, storageSystem, xtremioRestClientFactory);
    unManagedVolumesToCreate = new ArrayList<UnManagedVolume>();
    unManagedVolumesToUpdate = new ArrayList<UnManagedVolume>();
    unManagedCGToUpdateMap = new HashMap<String, UnManagedConsistencyGroup>();
    // get the storage pool associated with the xtremio system
    StoragePool storagePool = XtremIOProvUtils.getXtremIOStoragePool(storageSystem.getId(), dbClient);
    if (storagePool == null) {
        log.error("Skipping unmanaged volume discovery as the volume storage pool doesn't exist in ViPR");
        return;
    }
    Map<String, List<UnManagedVolume>> igUnmanagedVolumesMap = new HashMap<String, List<UnManagedVolume>>();
    Map<String, StringSet> igKnownVolumesMap = new HashMap<String, StringSet>();
    Map<String, Map<String, Integer>> volumeIGHLUMap = new HashMap<String, Map<String, Integer>>();
    String xioClusterName = xtremIOClient.getClusterDetails(storageSystem.getSerialNumber()).getName();
    // get the xtremio volume links and process them in batches
    List<XtremIOObjectInfo> volLinks = xtremIOClient.getXtremIOVolumeLinks(xioClusterName);
    // Get the volume details
    List<List<XtremIOObjectInfo>> volume_partitions = Lists.partition(volLinks, Constants.DEFAULT_PARTITION_SIZE);
    // Set containing cgs that cannot be ingested, for now that
    // means they contain volumes which belong to more than one cg
    unSupportedCG = new HashSet<String>();
    for (List<XtremIOObjectInfo> partition : volume_partitions) {
        List<XtremIOVolume> volumes = xtremIOClient.getXtremIOVolumesForLinks(partition, xioClusterName);
        for (XtremIOVolume volume : volumes) {
            try {
                // volumes later
                if (volume.getAncestoVolInfo() != null && !volume.getAncestoVolInfo().isEmpty()) {
                    log.debug("Skipping volume {} as it is a snap", volume.getVolInfo().get(0));
                    continue;
                }
                // check if cgs are null before trying to access, older versions of
                // the XtremIO REST client do not return cgs as part of volume response
                // flag indicating the volume is part of a cg
                boolean hasCGs = false;
                if (volume.getConsistencyGroups() != null && !volume.getConsistencyGroups().isEmpty()) {
                    hasCGs = true;
                    if (volume.getConsistencyGroups().size() > 1) {
                        // volumes that belong to multiple CGs are not supported in ViPR
                        log.warn("Skipping volume {} as it belongs to multiple CGs and this is not supported", volume.getVolInfo().get(0));
                        // add all the CGs that this volume belongs to to the list that are unsupported for ingestion
                        for (List<Object> cg : volume.getConsistencyGroups()) {
                            Object cgNameToProcess = cg.get(1);
                            unSupportedCG.add(cgNameToProcess.toString());
                            log.warn("Skipping CG {} as it contains volumes belonging to multiple CGs and this is not supported", cgNameToProcess.toString());
                        }
                        continue;
                    }
                }
                UnManagedVolume unManagedVolume = null;
                boolean isExported = !volume.getLunMaps().isEmpty();
                boolean hasSnaps = !volume.getSnaps().isEmpty();
                String managedVolumeNativeGuid = NativeGUIDGenerator.generateNativeGuidForVolumeOrBlockSnapShot(storageSystem.getNativeGuid(), volume.getVolInfo().get(0));
                Volume viprVolume = DiscoveryUtils.checkStorageVolumeExistsInDB(dbClient, managedVolumeNativeGuid);
                if (null != viprVolume) {
                    log.info("Skipping volume {} as it is already managed by ViPR", managedVolumeNativeGuid);
                    // export masks.
                    if (isExported) {
                        populateKnownVolsMap(volume, viprVolume, igKnownVolumesMap);
                    }
                    // retrieve snap info to be processed later
                    if (hasSnaps) {
                        StringSet vpoolUriSet = new StringSet();
                        vpoolUriSet.add(viprVolume.getVirtualPool().toString());
                        discoverVolumeSnaps(storageSystem, volume.getSnaps(), viprVolume.getNativeGuid(), vpoolUriSet, xtremIOClient, xioClusterName, dbClient, igUnmanagedVolumesMap, igKnownVolumesMap);
                    }
                    continue;
                }
                String unManagedVolumeNatvieGuid = NativeGUIDGenerator.generateNativeGuidForPreExistingVolume(storageSystem.getNativeGuid(), volume.getVolInfo().get(0));
                // retrieve snap info to be processed later
                unManagedVolume = DiscoveryUtils.checkUnManagedVolumeExistsInDB(dbClient, unManagedVolumeNatvieGuid);
                unManagedVolume = createUnManagedVolume(unManagedVolume, unManagedVolumeNatvieGuid, volume, igUnmanagedVolumesMap, storageSystem, storagePool, dbClient);
                // if the volume is associated with a CG, set up the unmanaged CG
                if (hasCGs) {
                    for (List<Object> cg : volume.getConsistencyGroups()) {
                        // retrieve what should be the first and only consistency group from the list
                        // volumes belonging to multiple cgs are not supported and were excluded above
                        Object cgNameToProcess = cg.get(1);
                        addObjectToUnManagedConsistencyGroup(xtremIOClient, unManagedVolume, cgNameToProcess.toString(), storageSystem, xioClusterName, dbClient);
                    }
                } else {
                    // Make sure the unManagedVolume object does not contain CG information from previous discovery
                    unManagedVolume.getVolumeCharacterstics().put(SupportedVolumeCharacterstics.IS_VOLUME_ADDED_TO_CONSISTENCYGROUP.toString(), Boolean.FALSE.toString());
                    // set the uri of the unmanaged CG in the unmanaged volume object to empty
                    unManagedVolume.getVolumeInformation().put(SupportedVolumeInformation.UNMANAGED_CONSISTENCY_GROUP_URI.toString(), "");
                }
                if (isExported) {
                    Map<String, Integer> igHLUMap = new HashMap<String, Integer>();
                    for (List<Object> lunMapEntries : volume.getLunMaps()) {
                        Double hlu = (Double) lunMapEntries.get(2);
                        log.info("Found HLU {}", hlu);
                        List<Object> igDetails = (List<Object>) lunMapEntries.get(0);
                        // key value IG-HLU
                        igHLUMap.put(igDetails.get(1).toString(), Integer.valueOf(hlu.intValue()));
                    }
                    if (!igHLUMap.isEmpty()) {
                        volumeIGHLUMap.put(unManagedVolumeNatvieGuid, igHLUMap);
                    }
                }
                boolean hasReplicas = false;
                if (hasSnaps) {
                    StringSet parentMatchedVPools = unManagedVolume.getSupportedVpoolUris();
                    StringSet discoveredSnaps = discoverVolumeSnaps(storageSystem, volume.getSnaps(), unManagedVolumeNatvieGuid, parentMatchedVPools, xtremIOClient, xioClusterName, dbClient, igUnmanagedVolumesMap, igKnownVolumesMap);
                    if (!discoveredSnaps.isEmpty()) {
                        hasReplicas = true;
                        // set the HAS_REPLICAS property
                        unManagedVolume.getVolumeCharacterstics().put(SupportedVolumeCharacterstics.HAS_REPLICAS.toString(), TRUE);
                        StringSetMap unManagedVolumeInformation = unManagedVolume.getVolumeInformation();
                        if (unManagedVolumeInformation.containsKey(SupportedVolumeInformation.SNAPSHOTS.toString())) {
                            log.debug("Snaps :" + Joiner.on("\t").join(discoveredSnaps));
                            if (null != discoveredSnaps && discoveredSnaps.isEmpty()) {
                                // replace with empty string set doesn't work, hence added explicit code to remove all
                                unManagedVolumeInformation.get(SupportedVolumeInformation.SNAPSHOTS.toString()).clear();
                            } else {
                                // replace with new StringSet
                                unManagedVolumeInformation.get(SupportedVolumeInformation.SNAPSHOTS.toString()).replace(discoveredSnaps);
                                log.info("Replaced snaps :" + Joiner.on("\t").join(unManagedVolumeInformation.get(SupportedVolumeInformation.SNAPSHOTS.toString())));
                            }
                        } else {
                            unManagedVolumeInformation.put(SupportedVolumeInformation.SNAPSHOTS.toString(), discoveredSnaps);
                        }
                    }
                }
                if (!hasReplicas) {
                    unManagedVolume.getVolumeCharacterstics().put(SupportedVolumeCharacterstics.HAS_REPLICAS.toString(), FALSE);
                    StringSet snapshots = unManagedVolume.getVolumeInformation().get(SupportedVolumeInformation.SNAPSHOTS.toString());
                    if (snapshots != null && !snapshots.isEmpty()) {
                        unManagedVolume.getVolumeInformation().get(SupportedVolumeInformation.SNAPSHOTS.toString()).clear();
                    }
                }
                allCurrentUnManagedVolumeUris.add(unManagedVolume.getId());
            } catch (Exception ex) {
                log.error("Error processing XIO volume {}", volume, ex);
            }
        }
    }
    if (!unManagedCGToUpdateMap.isEmpty()) {
        unManagedCGToUpdate = new ArrayList<UnManagedConsistencyGroup>(unManagedCGToUpdateMap.values());
        partitionManager.updateAndReIndexInBatches(unManagedCGToUpdate, unManagedCGToUpdate.size(), dbClient, UNMANAGED_CONSISTENCY_GROUP);
        unManagedCGToUpdate.clear();
    }
    if (!unManagedVolumesToCreate.isEmpty()) {
        partitionManager.insertInBatches(unManagedVolumesToCreate, Constants.DEFAULT_PARTITION_SIZE, dbClient, UNMANAGED_VOLUME);
        unManagedVolumesToCreate.clear();
    }
    if (!unManagedVolumesToUpdate.isEmpty()) {
        partitionManager.updateAndReIndexInBatches(unManagedVolumesToUpdate, Constants.DEFAULT_PARTITION_SIZE, dbClient, UNMANAGED_VOLUME);
        unManagedVolumesToUpdate.clear();
    }
    // Process those active unmanaged volume objects available in database but not in newly discovered items, to mark them inactive.
    DiscoveryUtils.markInActiveUnManagedVolumes(storageSystem, allCurrentUnManagedVolumeUris, dbClient, partitionManager);
    // Process those active unmanaged consistency group objects available in database but not in newly discovered items, to mark them
    // inactive.
    DiscoveryUtils.performUnManagedConsistencyGroupsBookKeeping(storageSystem, allCurrentUnManagedCgURIs, dbClient, partitionManager);
    // Next discover the unmanaged export masks
    discoverUnmanagedExportMasks(storageSystem.getId(), igUnmanagedVolumesMap, igKnownVolumesMap, xtremIOClient, xioClusterName, dbClient, partitionManager, volumeIGHLUMap);
}