Examples with UnManagedConsistencyGroup com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedConsistencyGroup used on opensource projects

Example 1 with UnManagedConsistencyGroup

use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedConsistencyGroup in project coprhd-controller by CoprHD.

the class BlockIngestOrchestrator method decorateCGInfoInVolumes.

/**
 * Decorates the BlockConsistencyGroup information in all other volumes ingested in the UnManagedConsistencyGroup
 * managed objects.
 *
 * For each unmanaged volume in unmanaged cg,
 * 1. We verify whether the BlockObject is available in the current createdBlockObjects in context or not.
 * If it is available, then set the CG properties
 * Else, verify in the current updatedBlockObjects in context.
 * 2. If the blockObject is available in updateBlockObjects, then update CG properties.
 * Else, blockObject might have ingested in previous requests, so, we should check from DB.
 * If blockObject is in DB, update CG properties else log a warning message.
 *
 * @param cg - cg object
 * @param blockObject - BlockObject to decorate
 * @param requestContext - current context of unmanagedVolume
 * @param unManagedVolume - current unmanagedVolume to ingest
 */
protected void decorateCGInfoInVolumes(BlockConsistencyGroup cg, BlockObject blockObject, IngestionRequestContext requestContext, UnManagedVolume unManagedVolume) {
    UnManagedConsistencyGroup umcg = requestContext.findUnManagedConsistencyGroup(cg.getLabel());
    Set<DataObject> blockObjectsToUpdate = new HashSet<DataObject>();
    if (null != umcg && null != umcg.getManagedVolumesMap() && !umcg.getManagedVolumesMap().isEmpty()) {
        for (Entry<String, String> managedVolumeEntry : umcg.getManagedVolumesMap().entrySet()) {
            BlockObject bo = requestContext.getRootIngestionRequestContext().findCreatedBlockObject(managedVolumeEntry.getKey());
            if (bo == null) {
                // Next look in the updated objects.
                bo = (BlockObject) requestContext.findInUpdatedObjects(URI.create(managedVolumeEntry.getKey()));
            }
            if (bo == null) {
                // Finally look in the DB itself. It may be from a previous ingestion operation.
                bo = BlockObject.fetch(_dbClient, URI.create(managedVolumeEntry.getValue()));
                // If blockObject is still not exists
                if (null == bo) {
                    _logger.warn("Volume {} is not yet ingested. Hence skipping", managedVolumeEntry.getKey());
                    continue;
                }
                blockObjectsToUpdate.add(bo);
            }
            bo.setConsistencyGroup(cg.getId());
            // Set the replication group instance only if it is not already populated during the block object's ingestion.
            if (bo.getReplicationGroupInstance() == null || bo.getReplicationGroupInstance().isEmpty()) {
                bo.setReplicationGroupInstance(cg.getLabel());
            }
        }
        if (!blockObjectsToUpdate.isEmpty()) {
            requestContext.getDataObjectsToBeUpdatedMap().put(unManagedVolume.getNativeGuid(), blockObjectsToUpdate);
        }
    }
    blockObject.setConsistencyGroup(cg.getId());
    blockObject.setReplicationGroupInstance(cg.getLabel());
    if (blockObject instanceof BlockSnapshot) {
        // Check if the unmanaged volume has SNAPSHOT_CONSISTENCY_GROUP_NAME property populated. If yes,
        // use that for replicationGroupInstance
        String snapsetName = PropertySetterUtil.extractValueFromStringSet(SupportedVolumeInformation.SNAPSHOT_CONSISTENCY_GROUP_NAME.toString(), unManagedVolume.getVolumeInformation());
        if (snapsetName != null && !snapsetName.isEmpty()) {
            blockObject.setReplicationGroupInstance(snapsetName);
        }
    }
}

Example 2 with UnManagedConsistencyGroup

use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedConsistencyGroup in project coprhd-controller by CoprHD.

the class BaseIngestionRequestContext method findUnManagedConsistencyGroup.

/*
     * (non-Javadoc)
     *
     * @see
     * com.emc.storageos.api.service.impl.resource.blockingestorchestration.context.IngestionRequestContext#findUnManagedConsistencyGroup(
     * com.emc.storageos.db.client.model.BlockConsistencyGroup)
     */
@Override
public UnManagedConsistencyGroup findUnManagedConsistencyGroup(String cgName) {
    VolumeIngestionContext currentVolumeContext = getVolumeContext();
    if (currentVolumeContext != null) {
        _logger.info("checking current volume ingestion context {}", currentVolumeContext.getUnmanagedVolume().forDisplay());
        List<UnManagedConsistencyGroup> umcgList = currentVolumeContext.getUmCGObjectsToUpdate();
        if (null != umcgList && !umcgList.isEmpty()) {
            for (UnManagedConsistencyGroup umcg : umcgList) {
                if (umcg.getLabel().equalsIgnoreCase(cgName)) {
                    return umcg;
                }
            }
        }
    }
    for (VolumeIngestionContext volumeContext : this.getProcessedUnManagedVolumeMap().values()) {
        _logger.info("checking already-ingested volume ingestion context {}", volumeContext.getUnmanagedVolume().forDisplay());
        List<UnManagedConsistencyGroup> umcgList = volumeContext.getUmCGObjectsToUpdate();
        if (null != umcgList && !umcgList.isEmpty()) {
            for (UnManagedConsistencyGroup umcg : umcgList) {
                if (umcg.getLabel().equalsIgnoreCase(cgName)) {
                    return umcg;
                }
            }
        }
    }
    return null;
}

Example 3 with UnManagedConsistencyGroup

use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedConsistencyGroup in project coprhd-controller by CoprHD.

the class VolumeIngestionUtil method getBlockObjectConsistencyGroup.

/**
 * Creates a BlockConsistencyGroup if it doesn't exist only when we are ingesting the last volume in unmanaged consistencygroup.
 *
 * In case if the volume is protected by RP or VPLEX, we should not create CG.
 *
 * @param unManagedVolume - UnManagedVolume object.
 * @param blockObj - Ingested BlockObject
 * @param context - current unManagedVolume Ingestion context.
 * @param dbClient - dbClient instance.
 * @return BlockConsistencyGroup
 */
public static BlockConsistencyGroup getBlockObjectConsistencyGroup(UnManagedVolume unManagedVolume, BlockObject blockObj, IngestionRequestContext context, DbClient dbClient) {
    UnManagedConsistencyGroup umcg = getUnManagedConsistencyGroup(unManagedVolume, dbClient);
    if (umcg != null) {
        // Check if the UnManagedConsistencyGroup is present in the volume context which should have the updated info
        UnManagedConsistencyGroup umcgInContext = context.findUnManagedConsistencyGroup(umcg.getLabel());
        if (umcgInContext != null) {
            umcg = umcgInContext;
        }
    }
    // CG ingestion support for such volumes.
    if (umcg == null || umcg.getUnManagedVolumesMap() == null) {
        _logger.info("There is no unmanaged consistency group associated with unmanaged volume {}, however " + "the volume has the IS_VOLUME_IN_CONSISTENCYGROUP flag set to true.  Ignoring CG operation" + " as there is not enough information to put this volume in a CG by itself.", unManagedVolume.getNativeGuid());
        return null;
    }
    List<UnManagedConsistencyGroup> umcgsToUpdate = context.getVolumeContext().getUmCGObjectsToUpdate();
    boolean isLastUmvToIngest = isLastUnManagedVolumeToIngest(umcg, unManagedVolume);
    boolean isVplexOrRPProtected = isRPOrVplexProtected(unManagedVolume);
    if (isVplexOrRPProtected || !isLastUmvToIngest) {
        _logger.info("Ignoring the CG creation as the volume is either isVplexRPProtected:{} or isLastUmvToIngest: {} exists to ingest.", isLastUmvToIngest, isVplexOrRPProtected);
        _logger.info("Remaining volumes in CG to ingest: {}", umcg.getUnManagedVolumesMap());
        // set ReplicationGroupInstance in the block object.
        blockObj.setReplicationGroupInstance(umcg.getLabel());
        if (blockObj instanceof BlockSnapshot) {
            // Check if the unmanaged volume has SNAPSHOT_CONSISTENCY_GROUP_NAME property populated. If yes,
            // use that for replicationGroupInstance
            String snapsetName = PropertySetterUtil.extractValueFromStringSet(SupportedVolumeInformation.SNAPSHOT_CONSISTENCY_GROUP_NAME.toString(), unManagedVolume.getVolumeInformation());
            if (snapsetName != null && !snapsetName.isEmpty()) {
                blockObj.setReplicationGroupInstance(snapsetName);
            }
        }
        updateVolumeInUnManagedConsistencyGroup(umcg, unManagedVolume, blockObj);
        umcgsToUpdate.add(umcg);
        return null;
    }
    // If the UMV is last volume, mark the UnManagedConsistencyGroup inactive to true.
    if (isLastUmvToIngest) {
        umcg.setInactive(true);
        umcgsToUpdate.add(umcg);
    }
    if (null == umcg || null == umcg.getLabel()) {
        _logger.warn("UnManaged volume {} CG doesn't have label. Hence exiting", unManagedVolume.getNativeGuid());
        return null;
    }
    String cgName = umcg.getLabel();
    StorageSystem storageSystem = dbClient.queryObject(StorageSystem.class, unManagedVolume.getStorageSystemUri());
    _logger.info("UnManagedVolume {} is added to consistency group {}", unManagedVolume.getLabel(), cgName);
    URI projectUri = context.getProject().getId();
    URI tenantUri = context.getTenant().getId();
    URI varrayUri = context.getVarray(unManagedVolume).getId();
    VirtualPool vpool = context.getVpool(unManagedVolume);
    if (!vpool.getMultivolumeConsistency()) {
        _logger.warn("The requested Virtual Pool {} does not have " + "the Multi-Volume Consistency flag set, and this volume " + "is part of a consistency group.", vpool.getLabel());
        throw IngestionException.exceptions.unmanagedVolumeVpoolConsistencyGroupMismatch(vpool.getLabel(), unManagedVolume.getLabel());
    } else {
        List<BlockConsistencyGroup> groups = CustomQueryUtility.queryActiveResourcesByConstraint(dbClient, BlockConsistencyGroup.class, PrefixConstraint.Factory.getFullMatchConstraint(BlockConsistencyGroup.class, "label", cgName));
        BlockConsistencyGroup potentialUnclaimedCg = null;
        if (!groups.isEmpty()) {
            for (BlockConsistencyGroup cg : groups) {
                if (validateCGProjectDetails(cg, storageSystem, projectUri, tenantUri, varrayUri, unManagedVolume.getLabel(), cgName, dbClient)) {
                    return cg;
                }
                URI storageControllerUri = cg.getStorageController();
                URI virtualArrayUri = cg.getVirtualArray();
                if (NullColumnValueGetter.isNullURI(storageControllerUri) && NullColumnValueGetter.isNullURI(virtualArrayUri)) {
                    potentialUnclaimedCg = cg;
                }
            }
        }
        // used it yet in creating a volume
        if (null != potentialUnclaimedCg) {
            potentialUnclaimedCg.addConsistencyGroupTypes(Types.LOCAL.name());
            potentialUnclaimedCg.setStorageController(storageSystem.getId());
            potentialUnclaimedCg.setVirtualArray(varrayUri);
            return potentialUnclaimedCg;
        }
        _logger.info(String.format("Did not find an existing CG named %s that is associated already with the requested device %s and Virtual Array %s. ViPR will create a new one.", cgName, storageSystem.getNativeGuid(), varrayUri));
        // create a new consistency group
        BlockConsistencyGroup cg = new BlockConsistencyGroup();
        cg.setId(URIUtil.createId(BlockConsistencyGroup.class));
        cg.setLabel(cgName);
        if (NullColumnValueGetter.isNotNullValue(umcg.getNativeId())) {
            cg.setNativeId(umcg.getNativeId());
        }
        cg.setProject(new NamedURI(projectUri, context.getProject().getLabel()));
        cg.setTenant(context.getProject().getTenantOrg());
        cg.addConsistencyGroupTypes(Types.LOCAL.name());
        cg.addSystemConsistencyGroup(storageSystem.getId().toString(), cgName);
        cg.setStorageController(storageSystem.getId());
        cg.setVirtualArray(varrayUri);
        cg.setArrayConsistency(false);
        return cg;
    }
}

Example 4 with UnManagedConsistencyGroup

use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedConsistencyGroup in project coprhd-controller by CoprHD.

the class VolumeIngestionUtil method getUnManagedConsistencyGroup.

/**
 * Return the UnManagedConsistencyGroup in which the unManagedVolume belongs to.
 *
 * @param unManagedVolume - UnManagedVolume object.
 * @param dbClient - dbClient instance.
 * @return the UnManagedConsistencyGroup in which the unManagedVolume belongs to
 */
public static UnManagedConsistencyGroup getUnManagedConsistencyGroup(UnManagedVolume unManagedVolume, DbClient dbClient) {
    UnManagedConsistencyGroup unManagedCG = null;
    String unManagedCGURI = PropertySetterUtil.extractValueFromStringSet(SupportedVolumeInformation.UNMANAGED_CONSISTENCY_GROUP_URI.toString(), unManagedVolume.getVolumeInformation());
    if (unManagedCGURI != null) {
        unManagedCG = dbClient.queryObject(UnManagedConsistencyGroup.class, URI.create(unManagedCGURI));
    }
    return unManagedCG;
}

Example 5 with UnManagedConsistencyGroup

use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedConsistencyGroup in project coprhd-controller by CoprHD.

the class ExternalDeviceUnManagedVolumeDiscoverer method discoverUnManagedBlockObjects.

/**
 * Discovers unmanaged block objects: volumes, snaps, clones, their CG information and their exports.
 * @param driver storage driver reference [IN]
 * @param storageSystem storage system [IN]
 * @param dbClient reference to db client [IN]
 * @param partitionManager partition manager [IN]
 */
public void discoverUnManagedBlockObjects(BlockStorageDriver driver, com.emc.storageos.db.client.model.StorageSystem storageSystem, DbClient dbClient, PartitionManager partitionManager) {
    Set<URI> allCurrentUnManagedVolumeUris = new HashSet<>();
    Set<URI> allCurrentUnManagedCgURIs = new HashSet<>();
    MutableInt lastPage = new MutableInt(0);
    MutableInt nextPage = new MutableInt(0);
    List<UnManagedVolume> unManagedVolumesToCreate = new ArrayList<>();
    List<UnManagedVolume> unManagedVolumesToUpdate = new ArrayList<>();
    List<UnManagedConsistencyGroup> unManagedCGToUpdate;
    Map<String, UnManagedConsistencyGroup> unManagedCGToUpdateMap = new HashMap<>();
    // We support only single export mask concept for host-array combination for external devices.
    // If we find that storage system has volumes which are exported to the same host through
    // different initiators or different array ports (we cannot create single UnManaged export
    // mask for the host and the array in this case), we won't discover exports to this
    // host on the array; we discover only volumes.
    // The result of this limitation is that it could happen that for some volumes we are able to
    // discover all their host exports;
    // for some volumes we will be able to discover their exports to subset of hosts;
    // for some volumes we may not be able to discover their exports to hosts.
    // This limits management scope for pre-existing exports initially, but this does not
    // not present a management issue for exports going forward, since driver implementation should handle export requests based
    // on provided initiators and volumes in the requests and the current state of device.
    // set of hosts for which we cannot build single export mask
    Set<String> invalidExportHosts = new HashSet<>();
    // for exported array volumes
    // get inter-process lock for exclusive discovery of unmanaged objects for a given system
    // lock is backed by curator's InterProcessMutex.
    InterProcessLock lock = null;
    String lockName = UNMANAGED_DISCOVERY_LOCK + storageSystem.getSystemType() + "-" + storageSystem.getNativeId();
    try {
        lock = coordinator.getLock(lockName);
        boolean lockAcquired = lock.acquire(UNMANAGED_DISCOVERY_LOCK_TIMEOUT, TimeUnit.SECONDS);
        if (lockAcquired) {
            log.info("Acquired lock {} for storage system {} .", lockName, storageSystem.getNativeId());
        } else {
            log.info("Failed to acquire lock {} for storage system {} .", lockName, storageSystem.getNativeId());
            return;
        }
    } catch (Exception ex) {
        // check that lock was not acquired. if lock was acquired for this thread, proceed.
        if (lock == null || !lock.isAcquiredInThisProcess()) {
            log.error("Error processing unmanaged discovery for storage system: {}. Failed to get lock {} for this operation.", storageSystem.getNativeId(), lockName, ex);
            return;
        }
    }
    log.info("Started discovery of UnManagedVolumes for system {}", storageSystem.getId());
    try {
        // We need to deactivate all old unManaged export masks for this array. Each export discovery starts a new.
        // Otherwise, we cannot distinguish between stale host masks and host mask discovered for volumes on the previous pages.
        DiscoveryUtils.markInActiveUnManagedExportMask(storageSystem.getId(), new HashSet<URI>(), dbClient, partitionManager);
        // prepare storage system
        StorageSystem driverStorageSystem = ExternalDeviceCommunicationInterface.initStorageSystem(storageSystem);
        do {
            Map<String, List<HostExportInfo>> hostToVolumeExportInfoMap = new HashMap<>();
            List<StorageVolume> driverVolumes = new ArrayList<>();
            Map<String, URI> unManagedVolumeNativeIdToUriMap = new HashMap<>();
            Map<String, URI> managedVolumeNativeIdToUriMap = new HashMap<>();
            log.info("Processing page {} ", nextPage);
            driver.getStorageVolumes(driverStorageSystem, driverVolumes, nextPage);
            log.info("Volume count on this page {} ", driverVolumes.size());
            for (StorageVolume driverVolume : driverVolumes) {
                UnManagedVolume unManagedVolume = null;
                try {
                    com.emc.storageos.db.client.model.StoragePool storagePool = getStoragePoolOfUnManagedVolume(storageSystem, driverVolume, dbClient);
                    if (null == storagePool) {
                        log.error("Skipping unManaged volume discovery as the volume {} storage pool doesn't exist in controller", driverVolume.getNativeId());
                        continue;
                    }
                    String managedVolumeNativeGuid = NativeGUIDGenerator.generateNativeGuidForVolumeOrBlockSnapShot(storageSystem.getNativeGuid(), driverVolume.getNativeId());
                    Volume systemVolume = DiscoveryUtils.checkStorageVolumeExistsInDB(dbClient, managedVolumeNativeGuid);
                    if (null != systemVolume) {
                        log.info("Skipping volume {} as it is already managed by the system. Id: {}", managedVolumeNativeGuid, systemVolume.getId());
                        // get export data for managed volume to process later --- we need to collect export data for
                        // managed volume
                        managedVolumeNativeIdToUriMap.put(driverVolume.getNativeId(), systemVolume.getId());
                        getVolumeExportInfo(driver, driverVolume, hostToVolumeExportInfoMap);
                        getExportInfoForManagedVolumeReplicas(managedVolumeNativeIdToUriMap, hostToVolumeExportInfoMap, dbClient, storageSystem, systemVolume, driverVolume, driver);
                        continue;
                    }
                    unManagedVolume = createUnManagedVolume(driverVolume, storageSystem, storagePool, unManagedVolumesToCreate, unManagedVolumesToUpdate, dbClient);
                    unManagedVolumeNativeIdToUriMap.put(driverVolume.getNativeId(), unManagedVolume.getId());
                    // if the volume is associated with a CG, set up the unManaged CG
                    if (driverVolume.getConsistencyGroup() != null && !driverVolume.getConsistencyGroup().isEmpty()) {
                        addObjectToUnManagedConsistencyGroup(storageSystem, driverVolume.getConsistencyGroup(), unManagedVolume, allCurrentUnManagedCgURIs, unManagedCGToUpdateMap, driver, dbClient);
                    } else {
                        // Make sure the unManagedVolume object does not contain CG information from previous discovery
                        unManagedVolume.getVolumeCharacterstics().put(UnManagedVolume.SupportedVolumeCharacterstics.IS_VOLUME_ADDED_TO_CONSISTENCYGROUP.toString(), Boolean.FALSE.toString());
                        // remove uri of the unManaged CG in the unManaged volume object
                        unManagedVolume.getVolumeInformation().remove(UnManagedVolume.SupportedVolumeInformation.UNMANAGED_CONSISTENCY_GROUP_URI.toString());
                    }
                    allCurrentUnManagedVolumeUris.add(unManagedVolume.getId());
                    getVolumeExportInfo(driver, driverVolume, hostToVolumeExportInfoMap);
                    Set<URI> unManagedSnaphotUris = processUnManagedSnapshots(driverVolume, unManagedVolume, storageSystem, storagePool, unManagedVolumesToCreate, unManagedVolumesToUpdate, allCurrentUnManagedCgURIs, unManagedCGToUpdateMap, unManagedVolumeNativeIdToUriMap, hostToVolumeExportInfoMap, driver, dbClient);
                    allCurrentUnManagedVolumeUris.addAll(unManagedSnaphotUris);
                    Set<URI> unManagedCloneUris = processUnManagedClones(driverVolume, unManagedVolume, storageSystem, storagePool, unManagedVolumesToCreate, unManagedVolumesToUpdate, allCurrentUnManagedCgURIs, unManagedCGToUpdateMap, unManagedVolumeNativeIdToUriMap, hostToVolumeExportInfoMap, driver, dbClient);
                    allCurrentUnManagedVolumeUris.addAll(unManagedCloneUris);
                } catch (Exception ex) {
                    log.error("Error processing {} volume {}", storageSystem.getNativeId(), driverVolume.getNativeId(), ex);
                }
            }
            if (!unManagedVolumesToCreate.isEmpty()) {
                log.info("Unmanaged volumes to create: {}", unManagedVolumesToCreate);
                partitionManager.insertInBatches(unManagedVolumesToCreate, Constants.DEFAULT_PARTITION_SIZE, dbClient, UNMANAGED_VOLUME);
                unManagedVolumesToCreate.clear();
            }
            if (!unManagedVolumesToUpdate.isEmpty()) {
                log.info("Unmanaged volumes to update: {}", unManagedVolumesToUpdate);
                partitionManager.updateAndReIndexInBatches(unManagedVolumesToUpdate, Constants.DEFAULT_PARTITION_SIZE, dbClient, UNMANAGED_VOLUME);
                unManagedVolumesToUpdate.clear();
            }
            // Process export data for volumes
            processExportData(driver, storageSystem, unManagedVolumeNativeIdToUriMap, managedVolumeNativeIdToUriMap, hostToVolumeExportInfoMap, invalidExportHosts, dbClient, partitionManager);
        } while (!nextPage.equals(lastPage));
        if (!unManagedCGToUpdateMap.isEmpty()) {
            unManagedCGToUpdate = new ArrayList<>(unManagedCGToUpdateMap.values());
            partitionManager.updateAndReIndexInBatches(unManagedCGToUpdate, unManagedCGToUpdate.size(), dbClient, UNMANAGED_CONSISTENCY_GROUP);
            unManagedCGToUpdate.clear();
        }
        log.info("Processed {} unmanged objects.", allCurrentUnManagedVolumeUris.size());
        // Process those active unManaged volume objects available in database but not in newly discovered items, to mark them inactive.
        DiscoveryUtils.markInActiveUnManagedVolumes(storageSystem, allCurrentUnManagedVolumeUris, dbClient, partitionManager);
        // Process those active unManaged consistency group objects available in database but not in newly discovered items, to mark them
        // inactive.
        DiscoveryUtils.performUnManagedConsistencyGroupsBookKeeping(storageSystem, allCurrentUnManagedCgURIs, dbClient, partitionManager);
    } catch (Exception ex) {
        log.error("Error processing unmanaged discovery for storage system: {}. Error on page: {}.", storageSystem.getNativeId(), nextPage.toString(), ex);
    } finally {
        // release lock
        try {
            lock.release();
            log.info("Released lock for storage system {}", storageSystem.getNativeId());
        } catch (Exception e) {
            log.error("Failed to release  Lock {} : {}", lockName, e.getMessage());
        }
    }
}