Examples with IsilonApi com.emc.storageos.isilon.restapi.IsilonApi used on opensource projects

Example 71 with IsilonApi

use of com.emc.storageos.isilon.restapi.IsilonApi in project coprhd-controller by CoprHD.

the class IsilonCommunicationInterface method discoverAllFileSystem.

private HashMap<String, Object> discoverAllFileSystem(StorageSystem storageSystem, String resumetoken, String umfsDiscoverPath) {
    URI storageSystemId = storageSystem.getId();
    try {
        _log.info("discoverAllFileSystem for storage system {} - start", storageSystemId);
        IsilonApi isilonApi = getIsilonDevice(storageSystem);
        List<IsilonAccessZone> accessZones = isilonApi.getAccessZones(null);
        List<String> tempAccessZonePath = new ArrayList<>();
        for (IsilonAccessZone accessZone : accessZones) {
            if (!accessZone.isSystem()) {
                tempAccessZonePath.add(accessZone.getPath() + "/");
            }
        }
        HashSet<String> fsPathSet = new HashSet<>();
        HashSet<String> fsQuotaPathSet = new HashSet<>();
        HashMap<String, IsilonSmartQuota> tempQuotaMap = new HashMap<>();
        IsilonApi.IsilonList<FileShare> isilonFSList = new IsilonApi.IsilonList<>();
        int accessZoneDiscPathLength = computeCustomConfigPathLengths(umfsDiscoverPath);
        IsilonApi.IsilonList<IsilonSmartQuota> quotas = isilonApi.listQuotas(resumetoken, umfsDiscoverPath);
        isilonFSList.setToken(quotas.getToken());
        for (IsilonSmartQuota quota : quotas.getList()) {
            if (quota.getType().compareTo("directory") != 0) {
                _log.debug("ignore quota path {} with quota id {}:", quota.getPath(), quota.getId() + " and quota type" + quota.getType());
                continue;
            }
            if ("/ifs/".equals(umfsDiscoverPath) && isQuotaUnderAccessZonePath(quota.getPath(), tempAccessZonePath)) {
                continue;
            }
            String fsNativeId = quota.getPath();
            if (isUnderUnmanagedDiscoveryPath(fsNativeId)) {
                int fsPathType = isQuotaOrFile(fsNativeId, accessZoneDiscPathLength);
                if (fsPathType == PATH_IS_FILE) {
                    tempQuotaMap.put(quota.getPath(), quota);
                    fsPathSet.add(fsNativeId);
                }
                if (fsPathType == PATH_IS_QUOTA) {
                    tempQuotaMap.put(quota.getPath(), quota);
                    fsQuotaPathSet.add(fsNativeId);
                }
            }
        }
        /*
             * Associate Quota directories with correct File paths
             */
        HashMap<String, Set<String>> fileQuotas = new HashMap<>();
        for (String filePath : fsPathSet) {
            HashSet<String> qdPaths = new HashSet<>();
            for (String qdPath : fsQuotaPathSet) {
                if (qdPath.startsWith(filePath + "/")) {
                    qdPaths.add(qdPath);
                }
            }
            if (!qdPaths.isEmpty()) {
                fsQuotaPathSet.removeAll(qdPaths);
                fileQuotas.put(filePath, qdPaths);
            }
        }
        HashMap<String, FileShare> fsWithQuotaMap = new HashMap<>();
        HashMap<String, UnManagedFileQuotaDirectory> qdMap = new HashMap<>();
        for (String fsNativeId : fsPathSet) {
            IsilonSmartQuota fileFsQuota = tempQuotaMap.get(fsNativeId);
            FileShare fs = extractFileShare(fsNativeId, fileFsQuota, storageSystem);
            _log.debug("quota id {} with capacity {}", fsNativeId + ":QUOTA:" + fileFsQuota.getId(), fs.getCapacity() + " used capacity " + fs.getUsedCapacity());
            fsWithQuotaMap.put(fsNativeId, fs);
            Set<String> fsQuotaIds = fileQuotas.get(fsNativeId);
            if (null != fsQuotaIds) {
                for (String quotaNativeId : fsQuotaIds) {
                    IsilonSmartQuota qdQuota = tempQuotaMap.get(quotaNativeId);
                    if (null != qdQuota) {
                        UnManagedFileQuotaDirectory qd = getUnManagedFileQuotaDirectory(fs.getNativeGuid(), qdQuota, storageSystem);
                        qdMap.put(quotaNativeId, qd);
                    }
                }
            }
        }
        List<FileShare> discoveredFS = new ArrayList<>();
        discoveredFS.addAll(fsWithQuotaMap.values());
        isilonFSList.addList(discoveredFS);
        List<UnManagedFileQuotaDirectory> discoverdQuotaDirectory = new ArrayList<>();
        discoverdQuotaDirectory.addAll(qdMap.values());
        HashMap<String, Object> discoveredFileDetails = new HashMap<>();
        discoveredFileDetails.put(UMFS_DETAILS, isilonFSList);
        discoveredFileDetails.put(UMFSQD_DETAILS, discoverdQuotaDirectory);
        discoveredFileDetails.put(UMFS_QD_MAP, fileQuotas);
        return discoveredFileDetails;
    } catch (IsilonException ie) {
        _log.error("discoverAllFileSystem failed. Storage system: {}", storageSystemId, ie);
        throw ie;
    } catch (Exception e) {
        _log.error("discoverAllFileSystem failed. Storage system: {}", storageSystemId, e);
        IsilonCollectionException ice = new IsilonCollectionException("discoverAllFileSystem failed. Storage system: " + storageSystemId);
        ice.initCause(e);
        throw ice;
    }
}

Example 72 with IsilonApi

use of com.emc.storageos.isilon.restapi.IsilonApi in project coprhd-controller by CoprHD.

the class IsilonCommunicationInterface method discoverPools.

private Map<String, List<StoragePool>> discoverPools(StorageSystem storageSystem, List<StoragePool> poolsToMatchWithVpool) throws IsilonCollectionException {
    // Discover storage pools
    Map<String, List<StoragePool>> storagePools = new HashMap<String, List<StoragePool>>();
    List<StoragePool> newPools = new ArrayList<StoragePool>();
    List<StoragePool> existingPools = new ArrayList<StoragePool>();
    URI storageSystemId = storageSystem.getId();
    try {
        _log.info("discoverPools for storage system {} - start", storageSystemId);
        IsilonApi isilonApi = getIsilonDevice(storageSystem);
        boolean isNfsV4Enabled = isilonApi.nfsv4Enabled(storageSystem.getFirmwareVersion());
        boolean syncLicenseValid = isValidLicense(isilonApi.getReplicationLicenseInfo(), storageSystem);
        boolean snapLicenseValid = isValidLicense(isilonApi.snapshotIQLicenseInfo(), storageSystem);
        // Set file replication type for Isilon storage system!!!
        if (syncLicenseValid) {
            StringSet supportReplicationTypes = new StringSet();
            supportReplicationTypes.add(SupportedFileReplicationTypes.REMOTE.name());
            supportReplicationTypes.add(SupportedFileReplicationTypes.LOCAL.name());
            storageSystem.setSupportedReplicationTypes(supportReplicationTypes);
        }
        _log.info("Isilon OneFS version: {}", storageSystem.getFirmwareVersion());
        List<? extends IsilonPool> isilonPools = null;
        if (VersionChecker.verifyVersionDetails(ONEFS_V7_2, storageSystem.getFirmwareVersion()) >= 0) {
            _log.info("Querying for Isilon storage pools...");
            isilonPools = isilonApi.getStoragePools();
        } else {
            _log.info("Querying for Isilon disk pools...");
            isilonPools = isilonApi.getDiskPools();
        }
        for (IsilonPool isilonPool : isilonPools) {
            // Check if this storage pool was already discovered
            StoragePool storagePool = null;
            String poolNativeGuid = NativeGUIDGenerator.generateNativeGuid(storageSystem, isilonPool.getNativeId(), NativeGUIDGenerator.POOL);
            @SuppressWarnings("deprecation") List<URI> poolURIs = _dbClient.queryByConstraint(AlternateIdConstraint.Factory.getStoragePoolByNativeGuidConstraint(poolNativeGuid));
            for (URI poolUri : poolURIs) {
                StoragePool pool = _dbClient.queryObject(StoragePool.class, poolUri);
                if (!pool.getInactive() && pool.getStorageDevice().equals(storageSystemId)) {
                    storagePool = pool;
                    break;
                }
            }
            if (storagePool == null) {
                // New storage pool
                storagePool = new StoragePool();
                storagePool.setId(URIUtil.createId(StoragePool.class));
                storagePool.setNativeGuid(poolNativeGuid);
                storagePool.setLabel(poolNativeGuid);
                storagePool.setPoolClassName(POOL_TYPE);
                storagePool.setPoolServiceType(PoolServiceType.file.toString());
                storagePool.setStorageDevice(storageSystemId);
                StringSet protocols = new StringSet();
                protocols.add("NFS");
                protocols.add("CIFS");
                storagePool.setProtocols(protocols);
                storagePool.setPoolName(isilonPool.getNativeId());
                storagePool.setNativeId(isilonPool.getNativeId());
                storagePool.setLabel(poolNativeGuid);
                storagePool.setSupportedResourceTypes(StoragePool.SupportedResourceTypes.THIN_AND_THICK.toString());
                storagePool.setOperationalStatus(StoragePool.PoolOperationalStatus.READY.toString());
                storagePool.setRegistrationStatus(RegistrationStatus.REGISTERED.toString());
                _log.info("Creating new storage pool using NativeGuid : {}", poolNativeGuid);
                newPools.add(storagePool);
            } else {
                existingPools.add(storagePool);
            }
            if (isNfsV4Enabled) {
                storagePool.getProtocols().add(NFSv4);
            } else {
                storagePool.getProtocols().remove(NFSv4);
            }
            // Add the Copy type ASYNC, if the Isilon is enabled with SyncIQ service!!
            StringSet copyTypesSupported = new StringSet();
            if (syncLicenseValid) {
                copyTypesSupported.add(CopyTypes.ASYNC.name());
                storagePool.setSupportedCopyTypes(copyTypesSupported);
            } else {
                if (storagePool.getSupportedCopyTypes() != null && storagePool.getSupportedCopyTypes().contains(CopyTypes.ASYNC.name())) {
                    storagePool.getSupportedCopyTypes().remove(CopyTypes.ASYNC.name());
                }
            }
            // Add the Copy type ScheduleSnapshot, if the Isilon is enabled with SnapshotIQ
            if (snapLicenseValid) {
                copyTypesSupported.add(CHECKPOINT_SCHEDULE);
                storagePool.setSupportedCopyTypes(copyTypesSupported);
            } else {
                if (storagePool.getSupportedCopyTypes() != null && storagePool.getSupportedCopyTypes().contains(CHECKPOINT_SCHEDULE)) {
                    storagePool.getSupportedCopyTypes().remove(CHECKPOINT_SCHEDULE);
                }
            }
            // scale capacity size
            storagePool.setFreeCapacity(isilonPool.getAvailableBytes() / BYTESCONVERTER);
            storagePool.setTotalCapacity(isilonPool.getTotalBytes() / BYTESCONVERTER);
            storagePool.setSubscribedCapacity(isilonPool.getUsedBytes() / BYTESCONVERTER);
            if (ImplicitPoolMatcher.checkPoolPropertiesChanged(storagePool.getCompatibilityStatus(), DiscoveredDataObject.CompatibilityStatus.COMPATIBLE.name()) || ImplicitPoolMatcher.checkPoolPropertiesChanged(storagePool.getDiscoveryStatus(), DiscoveryStatus.VISIBLE.name())) {
                poolsToMatchWithVpool.add(storagePool);
            }
            storagePool.setDiscoveryStatus(DiscoveryStatus.VISIBLE.name());
            storagePool.setCompatibilityStatus(DiscoveredDataObject.CompatibilityStatus.COMPATIBLE.name());
        }
        _log.info("discoverPools for storage system {} - complete", storageSystemId);
        storagePools.put(NEW, newPools);
        storagePools.put(EXISTING, existingPools);
        return storagePools;
    } catch (IsilonException ie) {
        _log.error("discoverPools failed. Storage system: {}", storageSystemId, ie);
        IsilonCollectionException ice = new IsilonCollectionException("discoverPools failed. Storage system: " + storageSystemId);
        ice.initCause(ie);
        throw ice;
    } catch (Exception e) {
        _log.error("discoverPools failed. Storage system: {}", storageSystemId, e);
        IsilonCollectionException ice = new IsilonCollectionException("discoverPools failed. Storage system: " + storageSystemId);
        ice.initCause(e);
        throw ice;
    }
}

Example 73 with IsilonApi

use of com.emc.storageos.isilon.restapi.IsilonApi in project coprhd-controller by CoprHD.

the class IsilonCommunicationInterface method discoverUmanagedFileSystems.

private void discoverUmanagedFileSystems(AccessProfile profile) throws BaseCollectionException {
    List<UnManagedFileSystem> newUnManagedFileSystems = new ArrayList<>();
    List<UnManagedFileSystem> existingUnManagedFileSystems = new ArrayList<>();
    List<UnManagedFileQuotaDirectory> newUnManagedFileQuotaDir = new ArrayList<>();
    List<UnManagedFileQuotaDirectory> existingUnManagedFileQuotaDir = new ArrayList<>();
    List<UnManagedCifsShareACL> newUnManagedCifsShareACLList = new ArrayList<>();
    List<UnManagedCifsShareACL> oldUnManagedCifsShareACLList = new ArrayList<>();
    List<UnManagedNFSShareACL> newUnManagedNfsShareACLList = new ArrayList<>();
    List<UnManagedNFSShareACL> oldUnManagedNfsShareACLList = new ArrayList<>();
    List<UnManagedFileExportRule> newUnManagedExportRules = new ArrayList<>();
    List<UnManagedFileExportRule> oldUnManagedExportRules = new ArrayList<>();
    _log.debug("Access Profile Details :  IpAddress : PortNumber : {}, namespace : {}", profile.getIpAddress() + profile.getPortNumber(), profile.getnamespace());
    URI storageSystemId = profile.getSystemId();
    StorageSystem storageSystem = _dbClient.queryObject(StorageSystem.class, storageSystemId);
    if (null == storageSystem) {
        return;
    }
    Set<URI> allDiscoveredUnManagedFileSystems = new HashSet<>();
    String detailedStatusMessage = "Discovery of Isilon Unmanaged FileSystem started";
    long unmanagedFsCount = 0;
    try {
        IsilonApi isilonApi = getIsilonDevice(storageSystem);
        URIQueryResultList storagePoolURIs = new URIQueryResultList();
        _dbClient.queryByConstraint(ContainmentConstraint.Factory.getStorageDeviceStoragePoolConstraint(storageSystem.getId()), storagePoolURIs);
        ArrayList<StoragePool> pools = new ArrayList<>();
        Iterator<URI> poolsItr = storagePoolURIs.iterator();
        while (poolsItr.hasNext()) {
            URI storagePoolURI = poolsItr.next();
            StoragePool storagePool = _dbClient.queryObject(StoragePool.class, storagePoolURI);
            if (storagePool != null && !storagePool.getInactive()) {
                pools.add(storagePool);
            }
        }
        StoragePool storagePool = null;
        if (pools != null && !pools.isEmpty()) {
            storagePool = pools.get(0);
        }
        StoragePort storagePort = getStoragePortPool(storageSystem);
        int totalIsilonFSDiscovered = 0;
        // get the associated storage port for vnas Server
        List<IsilonAccessZone> isilonAccessZones = isilonApi.getAccessZones(null);
        Map<String, NASServer> nasServers = validateAndGetNASServer(storageSystem, isilonAccessZones);
        // update the path from controller configuration
        updateDiscoveryPathForUnManagedFS(nasServers, storageSystem);
        // NFSv4 enabled on storage system!!!
        boolean isNfsV4Enabled = isilonApi.nfsv4Enabled(storageSystem.getFirmwareVersion());
        List<FileShare> discoveredFS = new ArrayList<>();
        String resumeToken = null;
        for (String umfsDiscoverPath : _discPathsForUnManaged) {
            IsilonAccessZone isilonAccessZone = getAccessZoneCorresDiscoveryPath(isilonAccessZones, umfsDiscoverPath);
            String isilonAccessZoneName;
            if (isilonAccessZone == null) {
                // System access zone
                isilonAccessZoneName = null;
            } else {
                isilonAccessZoneName = isilonAccessZone.getName();
            }
            // Get All SMB for this path access zone
            HashMap<String, HashSet<String>> zoneSMBShares = discoverAccessZoneSMBShares(storageSystem, isilonAccessZoneName);
            // Get all NFS Export for this path access zone
            HashMap<String, HashSet<Integer>> zoneNFSExports = discoverAccessZoneExports(storageSystem, isilonAccessZoneName);
            do {
                HashMap<String, Object> discoverdFileDetails = discoverAllFileSystem(storageSystem, resumeToken, umfsDiscoverPath);
                IsilonApi.IsilonList<FileShare> discoveredIsilonFS = (IsilonApi.IsilonList<FileShare>) discoverdFileDetails.get(UMFS_DETAILS);
                ArrayList<UnManagedFileQuotaDirectory> discoveredUmfsQd = (ArrayList<UnManagedFileQuotaDirectory>) discoverdFileDetails.get(UMFSQD_DETAILS);
                HashMap<String, HashSet<String>> umfsfileQuotaMap = (HashMap<String, HashSet<String>>) discoverdFileDetails.get(UMFS_QD_MAP);
                resumeToken = discoveredIsilonFS.getToken();
                discoveredFS = discoveredIsilonFS.getList();
                totalIsilonFSDiscovered += discoveredFS.size();
                for (FileShare fs : discoveredFS) {
                    if (!checkStorageFileSystemExistsInDB(fs.getNativeGuid())) {
                        // Create UnManaged FS
                        String fsPathName = fs.getPath();
                        UnManagedFileSystem unManagedFs = checkUnManagedFileSystemExistsInDB(fs.getNativeGuid());
                        if (unManagedFs != null) {
                            existingUnManagedFileSystems.add(unManagedFs);
                        }
                        // get the matched vNAS Server
                        NASServer nasServer = getMatchedNASServer(nasServers, fsPathName);
                        if (nasServer != null) {
                            // Get valid storage port from the NAS server!!!
                            _log.info("fs path {} and nas server details {}", fs.getPath(), nasServer.toString());
                            storagePort = getStoragePortFromNasServer(nasServer);
                            if (storagePort == null) {
                                _log.info("No valid storage port found for nas server {}", nasServer.toString());
                                continue;
                            }
                        } else {
                            _log.info("fs path {} and vnas server not found", fs.getPath());
                            // Skip further ingestion steps on this file share & move to next file share
                            continue;
                        }
                        unManagedFs = createUnManagedFileSystem(unManagedFs, fs.getNativeGuid(), storageSystem, storagePool, nasServer, fs);
                        unManagedFs.setHasNFSAcl(false);
                        newUnManagedFileSystems.add(unManagedFs);
                        /**
                         * Set and create the NFS ACLs only if the system is enabled with NFSv4!!!
                         */
                        HashMap<String, HashSet<Integer>> exportWithIdMap = getExportsIncludingSubDir(fs.getPath(), zoneNFSExports, umfsfileQuotaMap);
                        if (isNfsV4Enabled) {
                            Set<String> fsExportPaths = exportWithIdMap.keySet();
                            setUnmanagedNfsShareACL(unManagedFs, storageSystem, isilonApi, fsExportPaths, newUnManagedNfsShareACLList, oldUnManagedNfsShareACLList);
                        }
                        /**
                         * Set and Create Export Rules and export Map
                         */
                        if (!exportWithIdMap.keySet().isEmpty()) {
                            setUnManagedFSExportMap(unManagedFs, exportWithIdMap, storagePort, fs.getPath(), nasServer.getNasName(), isilonApi, storageSystem, newUnManagedExportRules, oldUnManagedExportRules);
                            _log.info("Number of exports discovered for file system {} is {}", unManagedFs.getId(), newUnManagedExportRules.size());
                            if (!newUnManagedExportRules.isEmpty()) {
                                unManagedFs.setHasExports(true);
                                _log.info("File System {} has Exports and their size is {}", unManagedFs.getId(), newUnManagedExportRules.size());
                            }
                        }
                        /**
                         * Create and set CIFS ACLS and SMB Share MAP
                         */
                        HashSet<String> shareIDs = getSharesIncludingSubDir(fs.getPath(), zoneSMBShares, umfsfileQuotaMap);
                        setUnmanagedCifsShareACL(unManagedFs, shareIDs, newUnManagedCifsShareACLList, storagePort, fs.getName(), nasServer.getNasName(), storageSystem, isilonApi, oldUnManagedCifsShareACLList);
                        _log.info("Number of shares ACLs discovered for file system {} is {}", unManagedFs.getId(), newUnManagedCifsShareACLList.size());
                        if (unManagedFs.getHasExports() || unManagedFs.getHasShares()) {
                            _log.info("FS {} is having exports/shares", fs.getPath());
                            unManagedFs.putFileSystemCharacterstics(UnManagedFileSystem.SupportedFileSystemCharacterstics.IS_FILESYSTEM_EXPORTED.toString(), TRUE);
                        } else {
                            _log.info("FS {} does not have export or share", fs.getPath());
                        }
                        /**
                         * Persist 200 objects and clear them to avoid memory issue
                         */
                        // save bunch of export rules in db
                        validateSizeLimitAndPersist(newUnManagedExportRules, oldUnManagedExportRules, Constants.DEFAULT_PARTITION_SIZE * 2);
                        // save bunch of ACLs in db
                        validateSizeLimitAndPersist(newUnManagedCifsShareACLList, oldUnManagedCifsShareACLList, Constants.DEFAULT_PARTITION_SIZE * 2);
                        // save bunch of NFS ACLs in db
                        validateSizeLimitAndPersist(newUnManagedNfsShareACLList, newUnManagedNfsShareACLList, Constants.DEFAULT_PARTITION_SIZE * 2);
                        allDiscoveredUnManagedFileSystems.add(unManagedFs.getId());
                        // save bunch of file system in db
                        validateListSizeLimitAndPersist(newUnManagedFileSystems, existingUnManagedFileSystems, Constants.DEFAULT_PARTITION_SIZE * 2);
                    }
                }
                for (UnManagedFileQuotaDirectory umfsQd : discoveredUmfsQd) {
                    if (!checkStorageQuotaDirectoryExistsInDB(umfsQd.getNativeGuid())) {
                        String fsUnManagedQdNativeGuid = NativeGUIDGenerator.generateNativeGuidForUnManagedQuotaDir(storageSystem.getSystemType(), storageSystem.getSerialNumber(), umfsQd.getNativeId(), "");
                        UnManagedFileQuotaDirectory unManagedFileQd = checkUnManagedFileSystemQuotaDirectoryExistsInDB(fsUnManagedQdNativeGuid);
                        boolean umfsQdExists = (unManagedFileQd == null) ? false : true;
                        if (umfsQdExists) {
                            umfsQd.setId(unManagedFileQd.getId());
                            existingUnManagedFileQuotaDir.add(umfsQd);
                        } else if (null != umfsQd) {
                            umfsQd.setId(URIUtil.createId(UnManagedFileQuotaDirectory.class));
                            newUnManagedFileQuotaDir.add(umfsQd);
                        }
                    }
                }
                // save bunch of QDs in db
                validateSizeLimitAndPersist(newUnManagedFileQuotaDir, existingUnManagedFileQuotaDir, Constants.DEFAULT_PARTITION_SIZE * 2);
            } while (resumeToken != null);
        }
        // Saving bunch of Unmanaged objects!!!
        if (!newUnManagedExportRules.isEmpty()) {
            _log.info("Saving Number of UnManagedFileExportRule(s) {}", newUnManagedExportRules.size());
            _dbClient.createObject(newUnManagedExportRules);
            newUnManagedExportRules.clear();
        }
        if (!oldUnManagedExportRules.isEmpty()) {
            _log.info("Saving Number of UnManagedFileExportRule(s) {}", newUnManagedExportRules.size());
            _dbClient.updateObject(newUnManagedExportRules);
            oldUnManagedExportRules.clear();
        }
        // save ACLs in db
        if (!newUnManagedCifsShareACLList.isEmpty()) {
            _log.info("Saving Number of UnManagedCifsShareACL(s) {}", newUnManagedCifsShareACLList.size());
            _dbClient.createObject(newUnManagedCifsShareACLList);
            newUnManagedCifsShareACLList.clear();
        }
        // save old acls
        if (!oldUnManagedCifsShareACLList.isEmpty()) {
            _log.info("Saving Number of UnManagedFileExportRule(s) {}", oldUnManagedCifsShareACLList.size());
            _dbClient.updateObject(oldUnManagedCifsShareACLList);
            oldUnManagedCifsShareACLList.clear();
        }
        // save NFS ACLs in db
        if (!newUnManagedNfsShareACLList.isEmpty()) {
            _log.info("Saving Number of UnManagedNfsShareACL(s) {}", newUnManagedNfsShareACLList.size());
            _dbClient.createObject(newUnManagedNfsShareACLList);
            newUnManagedNfsShareACLList.clear();
        }
        // save old acls
        if (!oldUnManagedNfsShareACLList.isEmpty()) {
            _log.info("Saving Number of NFS UnManagedFileExportRule(s) {}", oldUnManagedNfsShareACLList.size());
            _dbClient.updateObject(oldUnManagedNfsShareACLList);
            oldUnManagedNfsShareACLList.clear();
        }
        // save new QDs to DB
        if (!newUnManagedFileQuotaDir.isEmpty()) {
            _log.info("New unmanaged Isilon file systems QuotaDirecotry  count: {}", newUnManagedFileQuotaDir.size());
            _dbClient.createObject(newUnManagedFileQuotaDir);
        }
        // save old QDs
        if (!existingUnManagedFileQuotaDir.isEmpty()) {
            _log.info("Update unmanaged Isilon file systems QuotaDirectory count: {}", existingUnManagedFileQuotaDir.size());
            _dbClient.updateObject(existingUnManagedFileQuotaDir);
        }
        // save new FS
        if (!newUnManagedFileSystems.isEmpty()) {
            _dbClient.createObject(newUnManagedFileSystems);
        }
        // save old FS
        if (!existingUnManagedFileSystems.isEmpty()) {
            _dbClient.updateObject(existingUnManagedFileSystems);
        }
        _log.info("Discovered {} Isilon file systems.", totalIsilonFSDiscovered);
        // discovery succeeds
        detailedStatusMessage = String.format("Discovery completed successfully for Isilon: %s; new unmanaged file systems count: %s", storageSystemId.toString(), unmanagedFsCount);
        _log.info(detailedStatusMessage);
    } catch (IsilonException ex) {
        detailedStatusMessage = String.format("Discovery failed for Isilon %s because %s", storageSystemId.toString(), ex.getLocalizedMessage());
        _log.error(detailedStatusMessage, ex);
        throw ex;
    } catch (Exception e) {
        detailedStatusMessage = String.format("Discovery failed for Isilon %s because %s", storageSystemId.toString(), e.getLocalizedMessage());
        _log.error(detailedStatusMessage, e);
        throw new IsilonCollectionException(detailedStatusMessage);
    } finally {
        if (storageSystem != null) {
            try {
                // set detailed message
                storageSystem.setLastDiscoveryStatusMessage(detailedStatusMessage);
                _dbClient.updateObject(storageSystem);
            } catch (Exception ex) {
                _log.error("Error while persisting object to DB", ex);
            }
        }
    }
}

Example 74 with IsilonApi

use of com.emc.storageos.isilon.restapi.IsilonApi in project coprhd-controller by CoprHD.

the class IsilonCommunicationInterface method discoverPorts.

private HashMap<String, List<StoragePort>> discoverPorts(StorageSystem storageSystem) throws IsilonCollectionException {
    URI storageSystemId = storageSystem.getId();
    HashMap<String, List<StoragePort>> storagePorts = new HashMap<String, List<StoragePort>>();
    List<StoragePort> newStoragePorts = new ArrayList<StoragePort>();
    List<StoragePort> existingStoragePorts = new ArrayList<StoragePort>();
    // Discover storage ports
    try {
        _log.info("discoverPorts for storage system {} - start", storageSystemId);
        IsilonApi isilonApi = getIsilonDevice(storageSystem);
        List<IsilonStoragePort> isilonStoragePorts = new ArrayList<IsilonStoragePort>();
        try {
            _log.info("Trying to get latest smart connect version");
            IsilonSmartConnectInfoV2 connInfo = isilonApi.getSmartConnectInfoV2();
            if (connInfo == null || (connInfo != null && connInfo.getSmartZones() == null)) {
                throw new Exception("Failed new Interface, try old Interface");
            }
            if (connInfo != null) {
                isilonStoragePorts = connInfo.getPorts();
            }
        } catch (Exception e) {
            _log.info("Latest version failed so Trying to get old smart connect version");
            IsilonSmartConnectInfo connInfo = isilonApi.getSmartConnectInfo();
            if (connInfo != null) {
                isilonStoragePorts = connInfo.getPorts();
            }
        }
        if (isilonStoragePorts == null || isilonStoragePorts.isEmpty()) {
            // No ports defined throw an exception and fail the discovery
            IsilonCollectionException ice = new IsilonCollectionException("discoverPorts failed. No Smartzones defined");
            throw ice;
        }
        for (IsilonStoragePort isilonPort : isilonStoragePorts) {
            StoragePort storagePort = null;
            String portNativeGuid = NativeGUIDGenerator.generateNativeGuid(storageSystem, isilonPort.getIpAddress(), NativeGUIDGenerator.PORT);
            // Check if storage port was already discovered
            @SuppressWarnings("deprecation") List<URI> portURIs = _dbClient.queryByConstraint(AlternateIdConstraint.Factory.getStoragePortByNativeGuidConstraint(portNativeGuid));
            for (URI portUri : portURIs) {
                StoragePort port = _dbClient.queryObject(StoragePort.class, portUri);
                if (port.getStorageDevice().equals(storageSystemId) && !port.getInactive()) {
                    storagePort = port;
                    break;
                }
            }
            if (storagePort == null) {
                // Create Isilon storage port for Isilon cluster IP address (smart connect ip)
                storagePort = new StoragePort();
                storagePort.setId(URIUtil.createId(StoragePort.class));
                storagePort.setTransportType("IP");
                storagePort.setNativeGuid(portNativeGuid);
                storagePort.setLabel(portNativeGuid);
                storagePort.setStorageDevice(storageSystemId);
                storagePort.setPortNetworkId(isilonPort.getIpAddress().toLowerCase());
                storagePort.setPortName(isilonPort.getPortName());
                storagePort.setLabel(isilonPort.getPortName());
                storagePort.setPortSpeed(isilonPort.getPortSpeed());
                storagePort.setPortGroup(isilonPort.getPortName());
                storagePort.setRegistrationStatus(RegistrationStatus.REGISTERED.toString());
                _log.info("Creating new storage port using NativeGuid : {}", portNativeGuid);
                newStoragePorts.add(storagePort);
            } else {
                existingStoragePorts.add(storagePort);
            }
            storagePort.setDiscoveryStatus(DiscoveryStatus.VISIBLE.name());
            storagePort.setCompatibilityStatus(DiscoveredDataObject.CompatibilityStatus.COMPATIBLE.name());
        }
        _log.info("discoverPorts for storage system {} - complete", storageSystemId);
        storagePorts.put(NEW, newStoragePorts);
        storagePorts.put(EXISTING, existingStoragePorts);
        return storagePorts;
    } catch (Exception e) {
        _log.error("discoverPorts failed. Storage system: {}", storageSystemId, e);
        IsilonCollectionException ice = new IsilonCollectionException("discoverPorts failed. Storage system: " + storageSystemId);
        throw ice;
    }
}

Example 75 with IsilonApi

use of com.emc.storageos.isilon.restapi.IsilonApi in project coprhd-controller by CoprHD.

the class IsilonCommunicationInterface method collectStatisticsInformation.

@Override
public void collectStatisticsInformation(AccessProfile accessProfile) throws BaseCollectionException {
    URI storageSystemId = null;
    StorageSystem isilonCluster = null;
    long statsCount = 0;
    try {
        _log.info("Metering for {} using ip {}", accessProfile.getSystemId(), accessProfile.getIpAddress());
        IsilonApi api = getIsilonDevice(accessProfile);
        long latestSampleTime = accessProfile.getLastSampleTime();
        storageSystemId = accessProfile.getSystemId();
        isilonCluster = _dbClient.queryObject(StorageSystem.class, storageSystemId);
        String serialNumber = isilonCluster.getSerialNumber();
        String deviceType = isilonCluster.getSystemType();
        initializeKeyMap(accessProfile);
        boolean fsChanged = false;
        List<Stat> stats = new ArrayList<Stat>();
        List<FileShare> modifiedFileSystems = new ArrayList<FileShare>();
        ZeroRecordGenerator zeroRecordGenerator = new FileZeroRecordGenerator();
        CassandraInsertion statsColumnInjector = new FileDBInsertion();
        // get usage stats from quotas
        IsilonStatsRecorder recorder = new IsilonStatsRecorder(zeroRecordGenerator, statsColumnInjector);
        _keyMap.put(Constants._TimeCollected, System.currentTimeMillis());
        // compute static load processor code
        computeStaticLoadMetrics(storageSystemId);
        Map<String, String> fileSystemsMap = getStorageSystemFileShares(storageSystemId);
        if (fileSystemsMap.isEmpty()) {
            // No file shares for the storage system,
            // ignore stats collection for the system!!!
            _log.info("No file systems found for storage device {}. Hence metering stats collection ignored.", storageSystemId);
            return;
        }
        // Process IsilonQuotas page by page (MAX 1000) in a page...
        String resumeToken = null;
        do {
            IsilonApi.IsilonList<IsilonSmartQuota> quotas = api.listQuotas(resumeToken);
            resumeToken = quotas.getToken();
            for (IsilonSmartQuota quota : quotas.getList()) {
                String fsNativeId = quota.getPath();
                String fsNativeGuid = NativeGUIDGenerator.generateNativeGuid(deviceType, serialNumber, fsNativeId);
                String fsId = fileSystemsMap.get(fsNativeGuid);
                if (fsId == null || fsId.isEmpty()) {
                    // No file shares found for the quota
                    // ignore stats collection for the file system!!!
                    _log.debug("File System does not exists with nativeid {}. Hence ignoring stats collection.", fsNativeGuid);
                    continue;
                }
                Stat stat = recorder.addUsageStat(quota, _keyMap, fsId, api);
                fsChanged = false;
                if (null != stat) {
                    stats.add(stat);
                    // Persists the file system, only if change in used capacity.
                    FileShare fileSystem = _dbClient.queryObject(FileShare.class, stat.getResourceId());
                    if (fileSystem != null) {
                        if (!fileSystem.getInactive()) {
                            if (null != fileSystem.getUsedCapacity() && null != stat.getAllocatedCapacity() && !fileSystem.getUsedCapacity().equals(stat.getAllocatedCapacity())) {
                                fileSystem.setUsedCapacity(stat.getAllocatedCapacity());
                                fsChanged = true;
                            }
                            if (null != fileSystem.getSoftLimit() && null != fileSystem.getSoftLimitExceeded() && null != quota.getThresholds() && null != quota.getThresholds().getsoftExceeded() && !fileSystem.getSoftLimitExceeded().equals(quota.getThresholds().getsoftExceeded())) {
                                // softLimitExceeded
                                fileSystem.setSoftLimitExceeded(quota.getThresholds().getsoftExceeded());
                                fsChanged = true;
                            }
                            if (fsChanged) {
                                modifiedFileSystems.add(fileSystem);
                            }
                        }
                    }
                }
                // Each batch with MAX_RECORDS_SIZE - 100 records!!!
                if (modifiedFileSystems.size() >= MAX_RECORDS_SIZE) {
                    _dbClient.updateObject(modifiedFileSystems);
                    _log.info("Processed {} file systems stats ", modifiedFileSystems.size());
                    modifiedFileSystems.clear();
                }
                if (stats.size() >= MAX_RECORDS_SIZE) {
                    _log.info("Processed {} stats", stats.size());
                    persistStatsInDB(stats);
                }
            }
            statsCount = statsCount + quotas.size();
            _log.info("Processed {} file system stats for device {} ", quotas.size(), storageSystemId);
        } while (resumeToken != null);
        zeroRecordGenerator.identifyRecordstobeZeroed(_keyMap, stats, FileShare.class);
        // write the remaining records!!
        if (!modifiedFileSystems.isEmpty()) {
            _dbClient.updateObject(modifiedFileSystems);
            _log.info("Processed {} file systems stats ", modifiedFileSystems.size());
            modifiedFileSystems.clear();
        }
        if (!stats.isEmpty()) {
            _log.info("Processed {} stats", stats.size());
            persistStatsInDB(stats);
        }
        latestSampleTime = System.currentTimeMillis();
        accessProfile.setLastSampleTime(latestSampleTime);
        _log.info("Done metering device {}, processed {} file system stats ", storageSystemId, statsCount);
    } catch (Exception e) {
        if (isilonCluster != null) {
            cleanupDiscovery(isilonCluster);
        }
        _log.error("CollectStatisticsInformation failed. Storage system: " + storageSystemId, e);
        throw (new IsilonCollectionException(e.getMessage()));
    }
}