Examples with UnManagedVolume com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedVolume used on opensource projects

Example 41 with UnManagedVolume

use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedVolume in project coprhd-controller by CoprHD.

the class ExternalDeviceUnManagedVolumeDiscoverer method discoverUnManagedBlockObjects.

/**
 * Discovers unmanaged block objects: volumes, snaps, clones, their CG information and their exports.
 * @param driver storage driver reference [IN]
 * @param storageSystem storage system [IN]
 * @param dbClient reference to db client [IN]
 * @param partitionManager partition manager [IN]
 */
public void discoverUnManagedBlockObjects(BlockStorageDriver driver, com.emc.storageos.db.client.model.StorageSystem storageSystem, DbClient dbClient, PartitionManager partitionManager) {
    Set<URI> allCurrentUnManagedVolumeUris = new HashSet<>();
    Set<URI> allCurrentUnManagedCgURIs = new HashSet<>();
    MutableInt lastPage = new MutableInt(0);
    MutableInt nextPage = new MutableInt(0);
    List<UnManagedVolume> unManagedVolumesToCreate = new ArrayList<>();
    List<UnManagedVolume> unManagedVolumesToUpdate = new ArrayList<>();
    List<UnManagedConsistencyGroup> unManagedCGToUpdate;
    Map<String, UnManagedConsistencyGroup> unManagedCGToUpdateMap = new HashMap<>();
    // We support only single export mask concept for host-array combination for external devices.
    // If we find that storage system has volumes which are exported to the same host through
    // different initiators or different array ports (we cannot create single UnManaged export
    // mask for the host and the array in this case), we won't discover exports to this
    // host on the array; we discover only volumes.
    // The result of this limitation is that it could happen that for some volumes we are able to
    // discover all their host exports;
    // for some volumes we will be able to discover their exports to subset of hosts;
    // for some volumes we may not be able to discover their exports to hosts.
    // This limits management scope for pre-existing exports initially, but this does not
    // not present a management issue for exports going forward, since driver implementation should handle export requests based
    // on provided initiators and volumes in the requests and the current state of device.
    // set of hosts for which we cannot build single export mask
    Set<String> invalidExportHosts = new HashSet<>();
    // for exported array volumes
    // get inter-process lock for exclusive discovery of unmanaged objects for a given system
    // lock is backed by curator's InterProcessMutex.
    InterProcessLock lock = null;
    String lockName = UNMANAGED_DISCOVERY_LOCK + storageSystem.getSystemType() + "-" + storageSystem.getNativeId();
    try {
        lock = coordinator.getLock(lockName);
        boolean lockAcquired = lock.acquire(UNMANAGED_DISCOVERY_LOCK_TIMEOUT, TimeUnit.SECONDS);
        if (lockAcquired) {
            log.info("Acquired lock {} for storage system {} .", lockName, storageSystem.getNativeId());
        } else {
            log.info("Failed to acquire lock {} for storage system {} .", lockName, storageSystem.getNativeId());
            return;
        }
    } catch (Exception ex) {
        // check that lock was not acquired. if lock was acquired for this thread, proceed.
        if (lock == null || !lock.isAcquiredInThisProcess()) {
            log.error("Error processing unmanaged discovery for storage system: {}. Failed to get lock {} for this operation.", storageSystem.getNativeId(), lockName, ex);
            return;
        }
    }
    log.info("Started discovery of UnManagedVolumes for system {}", storageSystem.getId());
    try {
        // We need to deactivate all old unManaged export masks for this array. Each export discovery starts a new.
        // Otherwise, we cannot distinguish between stale host masks and host mask discovered for volumes on the previous pages.
        DiscoveryUtils.markInActiveUnManagedExportMask(storageSystem.getId(), new HashSet<URI>(), dbClient, partitionManager);
        // prepare storage system
        StorageSystem driverStorageSystem = ExternalDeviceCommunicationInterface.initStorageSystem(storageSystem);
        do {
            Map<String, List<HostExportInfo>> hostToVolumeExportInfoMap = new HashMap<>();
            List<StorageVolume> driverVolumes = new ArrayList<>();
            Map<String, URI> unManagedVolumeNativeIdToUriMap = new HashMap<>();
            Map<String, URI> managedVolumeNativeIdToUriMap = new HashMap<>();
            log.info("Processing page {} ", nextPage);
            driver.getStorageVolumes(driverStorageSystem, driverVolumes, nextPage);
            log.info("Volume count on this page {} ", driverVolumes.size());
            for (StorageVolume driverVolume : driverVolumes) {
                UnManagedVolume unManagedVolume = null;
                try {
                    com.emc.storageos.db.client.model.StoragePool storagePool = getStoragePoolOfUnManagedVolume(storageSystem, driverVolume, dbClient);
                    if (null == storagePool) {
                        log.error("Skipping unManaged volume discovery as the volume {} storage pool doesn't exist in controller", driverVolume.getNativeId());
                        continue;
                    }
                    String managedVolumeNativeGuid = NativeGUIDGenerator.generateNativeGuidForVolumeOrBlockSnapShot(storageSystem.getNativeGuid(), driverVolume.getNativeId());
                    Volume systemVolume = DiscoveryUtils.checkStorageVolumeExistsInDB(dbClient, managedVolumeNativeGuid);
                    if (null != systemVolume) {
                        log.info("Skipping volume {} as it is already managed by the system. Id: {}", managedVolumeNativeGuid, systemVolume.getId());
                        // get export data for managed volume to process later --- we need to collect export data for
                        // managed volume
                        managedVolumeNativeIdToUriMap.put(driverVolume.getNativeId(), systemVolume.getId());
                        getVolumeExportInfo(driver, driverVolume, hostToVolumeExportInfoMap);
                        getExportInfoForManagedVolumeReplicas(managedVolumeNativeIdToUriMap, hostToVolumeExportInfoMap, dbClient, storageSystem, systemVolume, driverVolume, driver);
                        continue;
                    }
                    unManagedVolume = createUnManagedVolume(driverVolume, storageSystem, storagePool, unManagedVolumesToCreate, unManagedVolumesToUpdate, dbClient);
                    unManagedVolumeNativeIdToUriMap.put(driverVolume.getNativeId(), unManagedVolume.getId());
                    // if the volume is associated with a CG, set up the unManaged CG
                    if (driverVolume.getConsistencyGroup() != null && !driverVolume.getConsistencyGroup().isEmpty()) {
                        addObjectToUnManagedConsistencyGroup(storageSystem, driverVolume.getConsistencyGroup(), unManagedVolume, allCurrentUnManagedCgURIs, unManagedCGToUpdateMap, driver, dbClient);
                    } else {
                        // Make sure the unManagedVolume object does not contain CG information from previous discovery
                        unManagedVolume.getVolumeCharacterstics().put(UnManagedVolume.SupportedVolumeCharacterstics.IS_VOLUME_ADDED_TO_CONSISTENCYGROUP.toString(), Boolean.FALSE.toString());
                        // remove uri of the unManaged CG in the unManaged volume object
                        unManagedVolume.getVolumeInformation().remove(UnManagedVolume.SupportedVolumeInformation.UNMANAGED_CONSISTENCY_GROUP_URI.toString());
                    }
                    allCurrentUnManagedVolumeUris.add(unManagedVolume.getId());
                    getVolumeExportInfo(driver, driverVolume, hostToVolumeExportInfoMap);
                    Set<URI> unManagedSnaphotUris = processUnManagedSnapshots(driverVolume, unManagedVolume, storageSystem, storagePool, unManagedVolumesToCreate, unManagedVolumesToUpdate, allCurrentUnManagedCgURIs, unManagedCGToUpdateMap, unManagedVolumeNativeIdToUriMap, hostToVolumeExportInfoMap, driver, dbClient);
                    allCurrentUnManagedVolumeUris.addAll(unManagedSnaphotUris);
                    Set<URI> unManagedCloneUris = processUnManagedClones(driverVolume, unManagedVolume, storageSystem, storagePool, unManagedVolumesToCreate, unManagedVolumesToUpdate, allCurrentUnManagedCgURIs, unManagedCGToUpdateMap, unManagedVolumeNativeIdToUriMap, hostToVolumeExportInfoMap, driver, dbClient);
                    allCurrentUnManagedVolumeUris.addAll(unManagedCloneUris);
                } catch (Exception ex) {
                    log.error("Error processing {} volume {}", storageSystem.getNativeId(), driverVolume.getNativeId(), ex);
                }
            }
            if (!unManagedVolumesToCreate.isEmpty()) {
                log.info("Unmanaged volumes to create: {}", unManagedVolumesToCreate);
                partitionManager.insertInBatches(unManagedVolumesToCreate, Constants.DEFAULT_PARTITION_SIZE, dbClient, UNMANAGED_VOLUME);
                unManagedVolumesToCreate.clear();
            }
            if (!unManagedVolumesToUpdate.isEmpty()) {
                log.info("Unmanaged volumes to update: {}", unManagedVolumesToUpdate);
                partitionManager.updateAndReIndexInBatches(unManagedVolumesToUpdate, Constants.DEFAULT_PARTITION_SIZE, dbClient, UNMANAGED_VOLUME);
                unManagedVolumesToUpdate.clear();
            }
            // Process export data for volumes
            processExportData(driver, storageSystem, unManagedVolumeNativeIdToUriMap, managedVolumeNativeIdToUriMap, hostToVolumeExportInfoMap, invalidExportHosts, dbClient, partitionManager);
        } while (!nextPage.equals(lastPage));
        if (!unManagedCGToUpdateMap.isEmpty()) {
            unManagedCGToUpdate = new ArrayList<>(unManagedCGToUpdateMap.values());
            partitionManager.updateAndReIndexInBatches(unManagedCGToUpdate, unManagedCGToUpdate.size(), dbClient, UNMANAGED_CONSISTENCY_GROUP);
            unManagedCGToUpdate.clear();
        }
        log.info("Processed {} unmanged objects.", allCurrentUnManagedVolumeUris.size());
        // Process those active unManaged volume objects available in database but not in newly discovered items, to mark them inactive.
        DiscoveryUtils.markInActiveUnManagedVolumes(storageSystem, allCurrentUnManagedVolumeUris, dbClient, partitionManager);
        // Process those active unManaged consistency group objects available in database but not in newly discovered items, to mark them
        // inactive.
        DiscoveryUtils.performUnManagedConsistencyGroupsBookKeeping(storageSystem, allCurrentUnManagedCgURIs, dbClient, partitionManager);
    } catch (Exception ex) {
        log.error("Error processing unmanaged discovery for storage system: {}. Error on page: {}.", storageSystem.getNativeId(), nextPage.toString(), ex);
    } finally {
        // release lock
        try {
            lock.release();
            log.info("Released lock for storage system {}", storageSystem.getNativeId());
        } catch (Exception e) {
            log.error("Failed to release  Lock {} : {}", lockName, e.getMessage());
        }
    }
}

Example 42 with UnManagedVolume

use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedVolume in project coprhd-controller by CoprHD.

the class ExternalDeviceUnManagedVolumeDiscoverer method createUnManagedVolume.

/**
 * Create or update unManaged volume for a given driver volume.
 *
 * @param driverVolume             storage system volume [IN]
 * @param storageSystem            storage system for unManaged volume [IN]
 * @param storagePool              storage pool for unManaged volume [IN]
 * @param unManagedVolumesToCreate list of new unManaged volumes [OUT]
 * @param unManagedVolumesToUpdate list of unManaged volumes to update [OUT]
 * @param dbClient reference to db client [IN]
 * @return unmanaged volume
 */
private UnManagedVolume createUnManagedVolume(StorageVolume driverVolume, com.emc.storageos.db.client.model.StorageSystem storageSystem, com.emc.storageos.db.client.model.StoragePool storagePool, List<UnManagedVolume> unManagedVolumesToCreate, List<UnManagedVolume> unManagedVolumesToUpdate, DbClient dbClient) {
    boolean newVolume = false;
    String unManagedVolumeNatvieGuid = NativeGUIDGenerator.generateNativeGuidForPreExistingVolume(storageSystem.getNativeGuid(), driverVolume.getNativeId());
    UnManagedVolume unManagedVolume = DiscoveryUtils.checkUnManagedVolumeExistsInDB(dbClient, unManagedVolumeNatvieGuid);
    if (null == unManagedVolume) {
        unManagedVolume = new UnManagedVolume();
        unManagedVolume.setId(URIUtil.createId(UnManagedVolume.class));
        unManagedVolume.setNativeGuid(unManagedVolumeNatvieGuid);
        unManagedVolume.setStorageSystemUri(storageSystem.getId());
        if (driverVolume.getWwn() == null) {
            unManagedVolume.setWwn(String.format("%s:%s", driverVolume.getStorageSystemId(), driverVolume.getNativeId()));
        } else {
            unManagedVolume.setWwn(driverVolume.getWwn());
        }
        newVolume = true;
    } else {
        // cleanup relationships from previous discoveries, will set them according to this discovery
        unManagedVolume.putVolumeCharacterstics(UnManagedVolume.SupportedVolumeCharacterstics.HAS_REPLICAS.toString(), FALSE);
        unManagedVolume.putVolumeInfo(UnManagedVolume.SupportedVolumeInformation.SNAPSHOTS.toString(), new StringSet());
        unManagedVolume.putVolumeInfo(UnManagedVolume.SupportedVolumeInformation.FULL_COPIES.toString(), new StringSet());
        // Clear old export mask information
        unManagedVolume.getUnmanagedExportMasks().clear();
    }
    unManagedVolume.setLabel(driverVolume.getDeviceLabel());
    Boolean isVolumeExported = false;
    unManagedVolume.putVolumeCharacterstics(UnManagedVolume.SupportedVolumeCharacterstics.IS_VOLUME_EXPORTED.toString(), isVolumeExported.toString());
    // Set these default to false.
    unManagedVolume.putVolumeCharacterstics(UnManagedVolume.SupportedVolumeCharacterstics.IS_NONRP_EXPORTED.toString(), FALSE);
    unManagedVolume.putVolumeCharacterstics(UnManagedVolume.SupportedVolumeCharacterstics.IS_RECOVERPOINT_ENABLED.toString(), FALSE);
    StringSet deviceLabel = new StringSet();
    deviceLabel.add(driverVolume.getDeviceLabel());
    unManagedVolume.putVolumeInfo(UnManagedVolume.SupportedVolumeInformation.DEVICE_LABEL.toString(), deviceLabel);
    StringSet accessState = new StringSet();
    accessState.add(driverVolume.getAccessStatus().toString());
    unManagedVolume.putVolumeInfo(UnManagedVolume.SupportedVolumeInformation.ACCESS.toString(), accessState);
    StringSet provCapacity = new StringSet();
    provCapacity.add(String.valueOf(driverVolume.getProvisionedCapacity()));
    unManagedVolume.putVolumeInfo(UnManagedVolume.SupportedVolumeInformation.PROVISIONED_CAPACITY.toString(), provCapacity);
    StringSet allocatedCapacity = new StringSet();
    allocatedCapacity.add(String.valueOf(driverVolume.getAllocatedCapacity()));
    unManagedVolume.putVolumeInfo(UnManagedVolume.SupportedVolumeInformation.ALLOCATED_CAPACITY.toString(), allocatedCapacity);
    StringSet systemTypes = new StringSet();
    systemTypes.add(storageSystem.getSystemType());
    unManagedVolume.putVolumeInfo(UnManagedVolume.SupportedVolumeInformation.SYSTEM_TYPE.toString(), systemTypes);
    StringSet nativeId = new StringSet();
    nativeId.add(driverVolume.getNativeId());
    unManagedVolume.putVolumeInfo(UnManagedVolume.SupportedVolumeInformation.NATIVE_ID.toString(), nativeId);
    unManagedVolume.putVolumeCharacterstics(UnManagedVolume.SupportedVolumeCharacterstics.IS_INGESTABLE.toString(), TRUE);
    unManagedVolume.putVolumeCharacterstics(UnManagedVolume.SupportedVolumeCharacterstics.IS_THINLY_PROVISIONED.toString(), driverVolume.getThinlyProvisioned().toString());
    unManagedVolume.setStoragePoolUri(storagePool.getId());
    StringSet pools = new StringSet();
    pools.add(storagePool.getId().toString());
    unManagedVolume.putVolumeInfo(UnManagedVolume.SupportedVolumeInformation.STORAGE_POOL.toString(), pools);
    StringSet driveTypes = storagePool.getSupportedDriveTypes();
    if (null != driveTypes) {
        unManagedVolume.putVolumeInfo(UnManagedVolume.SupportedVolumeInformation.DISK_TECHNOLOGY.toString(), driveTypes);
    }
    StringSet matchedVPools = DiscoveryUtils.getMatchedVirtualPoolsForPool(dbClient, storagePool.getId(), unManagedVolume.getVolumeCharacterstics().get(UnManagedVolume.SupportedVolumeCharacterstics.IS_THINLY_PROVISIONED.toString()), unManagedVolume);
    log.debug("Matched Pools : {}", Joiner.on("\t").join(matchedVPools));
    if (matchedVPools.isEmpty()) {
        // clear all existing supported vpools.
        unManagedVolume.getSupportedVpoolUris().clear();
    } else {
        // replace with new StringSet
        unManagedVolume.getSupportedVpoolUris().replace(matchedVPools);
        log.info("Replaced Pools : {}", Joiner.on("\t").join(unManagedVolume.getSupportedVpoolUris()));
    }
    if (newVolume) {
        unManagedVolumesToCreate.add(unManagedVolume);
    } else {
        unManagedVolumesToUpdate.add(unManagedVolume);
    }
    return unManagedVolume;
}

Example 43 with UnManagedVolume

use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedVolume in project coprhd-controller by CoprHD.

the class HDSVolumeDiscoverer method discoverUnManagedVolumes.

public void discoverUnManagedVolumes(AccessProfile accessProfile, DbClient dbClient, CoordinatorClient coordinator, PartitionManager partitionManager) throws Exception {
    log.info("Started discovery of UnManagedVolumes for system {}", accessProfile.getSystemId());
    HDSApiClient hdsApiClient = hdsApiFactory.getClient(HDSUtils.getHDSServerManagementServerInfo(accessProfile), accessProfile.getUserName(), accessProfile.getPassword());
    List<UnManagedVolume> newUnManagedVolumeList = new ArrayList<UnManagedVolume>();
    List<UnManagedVolume> updateUnManagedVolumeList = new ArrayList<UnManagedVolume>();
    Set<URI> allDiscoveredUnManagedVolumes = new HashSet<URI>();
    HDSApiVolumeManager volumeManager = hdsApiClient.getHDSApiVolumeManager();
    StorageSystem storageSystem = dbClient.queryObject(StorageSystem.class, accessProfile.getSystemId());
    String systemObjectId = HDSUtils.getSystemObjectID(storageSystem);
    List<LogicalUnit> luList = volumeManager.getAllLogicalUnits(systemObjectId);
    if (null != luList && !luList.isEmpty()) {
        log.info("Processing {} volumes received from HiCommand server.", luList.size());
        URIQueryResultList storagePoolURIs = new URIQueryResultList();
        dbClient.queryByConstraint(ContainmentConstraint.Factory.getStorageDeviceStoragePoolConstraint(storageSystem.getId()), storagePoolURIs);
        HashMap<String, StoragePool> pools = new HashMap<String, StoragePool>();
        Iterator<URI> poolsItr = storagePoolURIs.iterator();
        while (poolsItr.hasNext()) {
            URI storagePoolURI = poolsItr.next();
            StoragePool storagePool = dbClient.queryObject(StoragePool.class, storagePoolURI);
            pools.put(storagePool.getNativeGuid(), storagePool);
        }
        for (LogicalUnit logicalUnit : luList) {
            log.info("Processing LogicalUnit: {}", logicalUnit.getObjectID());
            UnManagedVolume unManagedVolume = null;
            String managedVolumeNativeGuid = NativeGUIDGenerator.generateNativeGuidForVolumeOrBlockSnapShot(storageSystem.getNativeGuid(), String.valueOf(logicalUnit.getDevNum()));
            if (null != DiscoveryUtils.checkStorageVolumeExistsInDB(dbClient, managedVolumeNativeGuid)) {
                log.info("Skipping volume {} as it is already managed by ViPR", managedVolumeNativeGuid);
            }
            String unManagedVolumeNativeGuid = NativeGUIDGenerator.generateNativeGuidForPreExistingVolume(storageSystem.getNativeGuid(), String.valueOf(logicalUnit.getDevNum()));
            unManagedVolume = DiscoveryUtils.checkUnManagedVolumeExistsInDB(dbClient, unManagedVolumeNativeGuid);
            boolean unManagedVolumeExists = (null != unManagedVolume) ? true : false;
            StoragePool storagePool = getStoragePoolOfUnManagedVolume(logicalUnit, storageSystem, pools, dbClient);
            if (null != storagePool) {
                if (!unManagedVolumeExists) {
                    unManagedVolume = createUnManagedVolume(unManagedVolumeNativeGuid, logicalUnit, storageSystem, storagePool, dbClient);
                    newUnManagedVolumeList.add(unManagedVolume);
                } else {
                    updateUnManagedVolumeInfo(logicalUnit, storageSystem, storagePool, unManagedVolume, dbClient);
                    updateUnManagedVolumeList.add(unManagedVolume);
                }
                allDiscoveredUnManagedVolumes.add(unManagedVolume.getId());
            } else {
                log.error("Skipping unmanaged volume discovery as the volume {} storage pool doesn't exist in ViPR", logicalUnit.getObjectID());
            }
            performUnManagedVolumesBookKeepting(newUnManagedVolumeList, updateUnManagedVolumeList, partitionManager, dbClient, Constants.DEFAULT_PARTITION_SIZE);
        }
        performUnManagedVolumesBookKeepting(newUnManagedVolumeList, updateUnManagedVolumeList, partitionManager, dbClient, 0);
        // Process those active unmanaged volume objects available in database but not in newly discovered items, to mark them inactive.
        DiscoveryUtils.markInActiveUnManagedVolumes(storageSystem, allDiscoveredUnManagedVolumes, dbClient, partitionManager);
    } else {
        log.info("No volumes retured by HiCommand Server for system {}", storageSystem.getId());
    }
}

Example 44 with UnManagedVolume

use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedVolume in project coprhd-controller by CoprHD.

the class HDSVolumeDiscoverer method createUnManagedVolume.

/**
 * Creates a new UnManagedVolume with the given arguments.
 *
 * @param unManagedVolumeNativeGuid
 * @param logicalUnit
 * @param system
 * @param pool
 * @param dbClient
 * @return
 */
private UnManagedVolume createUnManagedVolume(String unManagedVolumeNativeGuid, LogicalUnit logicalUnit, StorageSystem system, StoragePool pool, DbClient dbClient) {
    UnManagedVolume newUnManagedVolume = new UnManagedVolume();
    newUnManagedVolume.setId(URIUtil.createId(UnManagedVolume.class));
    newUnManagedVolume.setNativeGuid(unManagedVolumeNativeGuid);
    newUnManagedVolume.setStorageSystemUri(system.getId());
    newUnManagedVolume.setStoragePoolUri(pool.getId());
    updateUnManagedVolumeInfo(logicalUnit, system, pool, newUnManagedVolume, dbClient);
    return newUnManagedVolume;
}

Example 45 with UnManagedVolume

use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedVolume in project coprhd-controller by CoprHD.

the class VPlexCommunicationInterface method queryUnManagedVolumeByNativeGuid.

/**
 * Queries an UnManagedVolume in ViPR DB using volumeNativeID
 * @param volumeNativeGuid
 * @return an UnManagedVolume object if a match is found in the ViPR database
 */
private UnManagedVolume queryUnManagedVolumeByNativeGuid(String volumeNativeGuid) {
    s_logger.info("...checking ViPR's UnManagedVolume table for volume native guid {}", volumeNativeGuid);
    URIQueryResultList result = new URIQueryResultList();
    _dbClient.queryByConstraint(AlternateIdConstraint.Factory.getUnManagedVolumeByNativeGuidConstraint(volumeNativeGuid), result);
    if (result.iterator().hasNext()) {
        UnManagedVolume volume = _dbClient.queryObject(UnManagedVolume.class, result.iterator().next());
        // only return active volumes
        if (null != volume && !volume.getInactive()) {
            return volume;
        }
    }
    return null;
}