Examples with UnManagedFileExportRule com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedFileExportRule used on opensource projects

Example 21 with UnManagedFileExportRule

use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedFileExportRule in project coprhd-controller by CoprHD.

the class UnManagedFilesystemService method queryDBFSExports.

private List<UnManagedFileExportRule> queryDBFSExports(UnManagedFileSystem fs) {
    _logger.info("Querying all ExportRules Using FsId {}", fs.getId());
    try {
        ContainmentConstraint containmentConstraint = ContainmentConstraint.Factory.getUnManagedFileExportRulesConstraint(fs.getId());
        List<UnManagedFileExportRule> fileExportRules = CustomQueryUtility.queryActiveResourcesByConstraint(_dbClient, UnManagedFileExportRule.class, containmentConstraint);
        return fileExportRules;
    } catch (Exception e) {
        _logger.error("Error while querying {}", e);
    }
    return new ArrayList<UnManagedFileExportRule>();
}

Example 22 with UnManagedFileExportRule

use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedFileExportRule in project coprhd-controller by CoprHD.

the class UnManagedFilesystemService method ingestFileSystems.

/**
 * UnManaged file systems are file systems, which are present within ViPR
 * storage systems,but have not been ingested by ViPR which moves the unmanaged file systems under ViPR management.
 *
 * File system ingest provides flexibility in bringing unmanaged
 * file systems under ViPR management.
 * An unmanaged file system must be associated with a virtual pool, project,
 * and virtual array before it can be managed by ViPR.
 * List of supported virtual pools for each unmanaged file system is exposed using /vdc/unmanaged/filesystems/bulk.
 * Using an unsupported virtual pool results in an error
 *
 * Size of unmanaged file systems which can be ingested via a single API Call
 * is limited to 4000.
 *
 * @param param
 *            parameters required for unmanaged filesystem ingestion
 *
 * @prereq none
 * @brief Ingest unmanaged file systems
 * @throws InternalException
 */
@POST
@Consumes({ MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON })
@Produces({ MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON })
@Path("/ingest")
@CheckPermission(roles = { Role.SYSTEM_ADMIN, Role.RESTRICTED_SYSTEM_ADMIN })
public NamedFileSystemList ingestFileSystems(FileSystemIngest param) throws InternalException {
    if ((null == param.getUnManagedFileSystems()) || (param.getUnManagedFileSystems().toString().length() == 0) || (param.getUnManagedFileSystems().isEmpty()) || (param.getUnManagedFileSystems().get(0).toString().isEmpty())) {
        throw APIException.badRequests.invalidParameterUnManagedFsListEmpty();
    }
    if (null == param.getProject() || (param.getProject().toString().length() == 0)) {
        throw APIException.badRequests.invalidParameterProjectEmpty();
    }
    if (null == param.getVarray() || (param.getVarray().toString().length() == 0)) {
        throw APIException.badRequests.invalidParameterVirtualArrayEmpty();
    }
    if (null == param.getVpool() || (param.getVpool().toString().length() == 0)) {
        throw APIException.badRequests.invalidParameterVirtualPoolEmpty();
    }
    if (param.getUnManagedFileSystems().size() > getMaxBulkSize()) {
        throw APIException.badRequests.exceedingLimit("unmanaged filesystems", getMaxBulkSize());
    }
    _logger.info("Ingest called with Virtual Array {}", param.getVarray());
    _logger.info("Ingest called with Virtual Pool {}", param.getVpool());
    _logger.info("Ingest called with Project {}", param.getProject());
    _logger.info("Ingest called with UnManagedFileSystems {}", param.getUnManagedFileSystems());
    NamedFileSystemList filesystemList = new NamedFileSystemList();
    List<UnManagedFileSystem> unManagedFileSystems = new ArrayList<UnManagedFileSystem>();
    try {
        // Get and validate the project.
        Project project = _permissionsHelper.getObjectById(param.getProject(), Project.class);
        ArgValidator.checkUri(param.getProject());
        ArgValidator.checkEntity(project, param.getProject(), false);
        VirtualArray neighborhood = FileSystemIngestionUtil.getVirtualArrayForFileSystemCreateRequest(project, param.getVarray(), _permissionsHelper, _dbClient);
        // Get and validate the VirtualPool.
        VirtualPool cos = FileSystemIngestionUtil.getVirtualPoolForFileSystemCreateRequest(project, param.getVpool(), _permissionsHelper, _dbClient);
        if (null != cos.getVirtualArrays() && !cos.getVirtualArrays().isEmpty() && !cos.getVirtualArrays().contains(param.getVarray().toString())) {
            throw APIException.internalServerErrors.virtualPoolNotMatchingVArray(param.getVarray());
        }
        // check for Quotas
        long unManagedFileSystemsCapacity = FileSystemIngestionUtil.getTotalUnManagedFileSystemCapacity(_dbClient, param.getUnManagedFileSystems());
        _logger.info("Requested UnManagedFile System Capacity {}", unManagedFileSystemsCapacity);
        TenantOrg tenant = _dbClient.queryObject(TenantOrg.class, project.getTenantOrg().getURI());
        CapacityUtils.validateQuotasForProvisioning(_dbClient, cos, project, tenant, unManagedFileSystemsCapacity, "filesystem");
        FileSystemIngestionUtil.isIngestionRequestValidForUnManagedFileSystems(param.getUnManagedFileSystems(), cos, _dbClient);
        List<FileShare> filesystems = new ArrayList<FileShare>();
        Map<URI, FileShare> unManagedFSURIToFSMap = new HashMap<>();
        List<FileExportRule> fsExportRules = new ArrayList<FileExportRule>();
        List<CifsShareACL> fsCifsShareAcls = new ArrayList<CifsShareACL>();
        List<NFSShareACL> fsNfsShareAcls = new ArrayList<NFSShareACL>();
        List<UnManagedFileExportRule> inActiveUnManagedExportRules = new ArrayList<UnManagedFileExportRule>();
        List<UnManagedCifsShareACL> inActiveUnManagedShareCifs = new ArrayList<UnManagedCifsShareACL>();
        List<UnManagedNFSShareACL> inActiveUnManagedShareNfs = new ArrayList<UnManagedNFSShareACL>();
        // cifs share acl's
        List<CifsShareACL> cifsShareACLList = new ArrayList<CifsShareACL>();
        List<URI> full_pools = new ArrayList<URI>();
        List<URI> full_systems = new ArrayList<URI>();
        Calendar timeNow = Calendar.getInstance();
        for (URI unManagedFileSystemUri : param.getUnManagedFileSystems()) {
            long softLimit = 0;
            int softGrace = 0;
            long notificationLimit = 0;
            UnManagedFileSystem unManagedFileSystem = _dbClient.queryObject(UnManagedFileSystem.class, unManagedFileSystemUri);
            if (null == unManagedFileSystem || null == unManagedFileSystem.getFileSystemCharacterstics() || null == unManagedFileSystem.getFileSystemInformation()) {
                _logger.warn("UnManaged FileSystem {} partially discovered, hence not enough information available to validate neither virtualPool nor other criterias.Skipping Ingestion..", unManagedFileSystemUri);
                continue;
            }
            if (unManagedFileSystem.getInactive()) {
                _logger.warn("UnManaged FileSystem {} is inactive.Skipping Ingestion..", unManagedFileSystemUri);
                continue;
            }
            if (!FileSystemIngestionUtil.checkVirtualPoolValidForUnManagedFileSystem(_dbClient, cos, unManagedFileSystemUri)) {
                continue;
            }
            StringSetMap unManagedFileSystemInformation = unManagedFileSystem.getFileSystemInformation();
            String fsNativeGuid = unManagedFileSystem.getNativeGuid().replace(FileSystemIngestionUtil.UNMANAGEDFILESYSTEM, FileSystemIngestionUtil.FILESYSTEM);
            String deviceLabel = PropertySetterUtil.extractValueFromStringSet(SupportedFileSystemInformation.DEVICE_LABEL.toString(), unManagedFileSystemInformation);
            String fsName = PropertySetterUtil.extractValueFromStringSet(SupportedFileSystemInformation.NAME.toString(), unManagedFileSystemInformation);
            URI storagePoolUri = unManagedFileSystem.getStoragePoolUri();
            String storagePortUri = PropertySetterUtil.extractValueFromStringSet(SupportedFileSystemInformation.STORAGE_PORT.toString(), unManagedFileSystemInformation);
            String capacity = PropertySetterUtil.extractValueFromStringSet(SupportedFileSystemInformation.PROVISIONED_CAPACITY.toString(), unManagedFileSystemInformation);
            String usedCapacity = PropertySetterUtil.extractValueFromStringSet(SupportedFileSystemInformation.ALLOCATED_CAPACITY.toString(), unManagedFileSystemInformation);
            String nasUri = PropertySetterUtil.extractValueFromStringSet(SupportedFileSystemInformation.NAS.toString(), unManagedFileSystemInformation);
            String path = PropertySetterUtil.extractValueFromStringSet(SupportedFileSystemInformation.PATH.toString(), unManagedFileSystemInformation);
            String mountPath = PropertySetterUtil.extractValueFromStringSet(SupportedFileSystemInformation.MOUNT_PATH.toString(), unManagedFileSystemInformation);
            String nativeId = PropertySetterUtil.extractValueFromStringSet(SupportedFileSystemInformation.NATIVE_ID.toString(), unManagedFileSystemInformation);
            String systemType = PropertySetterUtil.extractValueFromStringSet(SupportedFileSystemInformation.SYSTEM_TYPE.toString(), unManagedFileSystemInformation);
            String softLt = PropertySetterUtil.extractValueFromStringSet(SupportedFileSystemInformation.SOFT_LIMIT.toString(), unManagedFileSystemInformation);
            String softGr = PropertySetterUtil.extractValueFromStringSet(SupportedFileSystemInformation.SOFT_GRACE.toString(), unManagedFileSystemInformation);
            String notificationLt = PropertySetterUtil.extractValueFromStringSet(SupportedFileSystemInformation.NOTIFICATION_LIMIT.toString(), unManagedFileSystemInformation);
            if (null != softLt && !softLt.isEmpty()) {
                softLimit = Long.valueOf(softLt);
            }
            if (null != softGr && !softGr.isEmpty()) {
                softGrace = Integer.valueOf(softGr);
            }
            if (null != notificationLt && !notificationLt.isEmpty()) {
                notificationLimit = Long.valueOf(notificationLt);
            }
            Long lcapcity = Long.valueOf(capacity);
            Long lusedCapacity = Long.valueOf(usedCapacity);
            // pool uri cannot be null
            StoragePool pool = _dbClient.queryObject(StoragePool.class, storagePoolUri);
            StoragePort port = null;
            if (storagePortUri != null) {
                port = _dbClient.queryObject(StoragePort.class, URI.create(storagePortUri));
            }
            StorageHADomain dataMover = null;
            if (port != null && port.getStorageHADomain() != null) {
                dataMover = _dbClient.queryObject(StorageHADomain.class, port.getStorageHADomain());
            }
            if (dataMover != null) {
                _logger.info("Data Mover to Use {} {} {}", new Object[] { dataMover.getAdapterName(), dataMover.getName(), dataMover.getLabel() });
            }
            // Check for same name File Share in this project
            if (FileSystemIngestionUtil.checkForDuplicateFSName(_dbClient, project.getId(), deviceLabel, filesystems)) {
                _logger.info("File System with name: {}  already exists in the given project: {} so, ignoring it..", deviceLabel, project.getLabel());
                continue;
            }
            // check ingestion is valid for given project
            if (!isIngestUmfsValidForProject(project, _dbClient, nasUri)) {
                _logger.info("UnManaged FileSystem path {} is mounted on vNAS URI {} which is invalid for project.", path, nasUri);
                continue;
            }
            // Check for same named File Share in this project
            if (FileSystemIngestionUtil.checkForDuplicateFSName(_dbClient, project.getId(), deviceLabel, filesystems)) {
                _logger.info("File System with name: {}  already exists in given project: {} so, ignoring it..", deviceLabel, project.getLabel());
                continue;
            }
            // if not don't ingest
            if (null != pool) {
                StringSet taggedVirtualArrays = pool.getTaggedVirtualArrays();
                if ((null == taggedVirtualArrays) || (!taggedVirtualArrays.contains(neighborhood.getId().toString()))) {
                    _logger.warn("UnManaged FileSystem {} storagepool doesn't related to the Virtual Array {}. Skipping Ingestion..", unManagedFileSystemUri, neighborhood.getId().toString());
                    continue;
                }
            } else {
                _logger.warn("UnManaged FileSystem {} doesn't contain a storagepool. Skipping Ingestiong", unManagedFileSystemUri);
                continue;
            }
            if (full_pools.contains(storagePoolUri)) {
                // skip this fileshare
                continue;
            }
            if (pool.getIsResourceLimitSet()) {
                if (pool.getMaxResources() <= StoragePoolService.getNumResources(pool, _dbClient)) {
                    // reached limit for this pool
                    full_pools.add(storagePoolUri);
                    continue;
                }
            }
            FileShare filesystem = new FileShare();
            filesystem.setId(URIUtil.createId(FileShare.class));
            filesystem.setNativeGuid(fsNativeGuid);
            filesystem.setCapacity(lcapcity);
            filesystem.setUsedCapacity(lusedCapacity);
            filesystem.setPath(path);
            filesystem.setMountPath(mountPath);
            filesystem.setVirtualPool(param.getVpool());
            filesystem.setVirtualArray(param.getVarray());
            filesystem.setSoftLimit(softLimit);
            filesystem.setSoftGracePeriod(softGrace);
            filesystem.setNotificationLimit(notificationLimit);
            if (nasUri != null) {
                filesystem.setVirtualNAS(URI.create(nasUri));
                if (!doesNASServerSupportVPoolProtocols(nasUri, cos.getProtocols())) {
                    _logger.warn("UnManaged FileSystem NAS server {} doesn't support vpool protocols. Skipping Ingestion...", nasUri);
                    continue;
                }
            }
            if (nativeId != null) {
                filesystem.setNativeId(nativeId);
            }
            URI storageSystemUri = unManagedFileSystem.getStorageSystemUri();
            StorageSystem system = _dbClient.queryObject(StorageSystem.class, storageSystemUri);
            if (full_systems.contains(storageSystemUri)) {
                // skip this fileshare
                continue;
            }
            if (system.getIsResourceLimitSet()) {
                if (system.getMaxResources() <= StorageSystemService.getNumResources(system, _dbClient)) {
                    // reached limit for this system
                    full_systems.add(storageSystemUri);
                    continue;
                }
            }
            filesystem.setStorageDevice(storageSystemUri);
            filesystem.setCreationTime(timeNow);
            filesystem.setPool(storagePoolUri);
            filesystem.setProtocol(new StringSet());
            StringSet fsSupportedProtocols = new StringSet();
            for (StorageProtocol.File fileProtocol : StorageProtocol.File.values()) {
                fsSupportedProtocols.add(fileProtocol.name());
            }
            // fs support protocol which is present in StoragePool and VirtualPool both
            fsSupportedProtocols.retainAll(pool.getProtocols());
            fsSupportedProtocols.retainAll(cos.getProtocols());
            filesystem.getProtocol().addAll(fsSupportedProtocols);
            filesystem.setLabel(null == deviceLabel ? "" : deviceLabel);
            filesystem.setName(null == fsName ? "" : fsName);
            filesystem.setTenant(new NamedURI(project.getTenantOrg().getURI(), filesystem.getLabel()));
            filesystem.setProject(new NamedURI(param.getProject(), filesystem.getLabel()));
            _logger.info("Un Managed File System {} has exports? : {}", unManagedFileSystem.getId(), unManagedFileSystem.getHasExports());
            StoragePort sPort = null;
            if (port != null && neighborhood != null) {
                if (StorageSystem.Type.isilon.toString().equals(system.getSystemType())) {
                    sPort = getIsilonStoragePort(port, nasUri, neighborhood.getId());
                } else {
                    sPort = compareAndSelectPortURIForUMFS(system, port, neighborhood);
                }
            }
            /*
                 * If UMFS storage port is not part of the vArray then skip ingestion
                 */
            if (sPort == null) {
                _logger.warn("Storage port of UMFS {} doesn't belong to a matching NetWork. So skipping ingestion", unManagedFileSystemUri);
                continue;
            }
            _logger.info("Storage Port Found {}", sPort);
            filesystem.setPortName(sPort.getPortName());
            filesystem.setStoragePort(sPort.getId());
            if (unManagedFileSystem.getHasExports()) {
                filesystem.setFsExports(PropertySetterUtil.convertUnManagedExportMapToManaged(unManagedFileSystem.getFsUnManagedExportMap(), sPort, dataMover));
                _logger.info("Export map for {} = {}", fsName, filesystem.getFsExports());
                // Process Exports
                // Step 1 : Query them and Retrive associated Exports
                List<UnManagedFileExportRule> exports = queryDBFSExports(unManagedFileSystem);
                _logger.info("Number of Exports Found : {} for UnManaged Fs path : {}", exports.size(), unManagedFileSystem.getMountPath());
                if (exports != null && !exports.isEmpty()) {
                    for (UnManagedFileExportRule rule : exports) {
                        // Step 2 : Convert them to File Export Rule
                        // Step 3 : Keep them as a list to store in db, down the line at a shot
                        // Important to relate the exports to a
                        rule.setFileSystemId(filesystem.getId());
                        // FileSystem.
                        createRule(rule, fsExportRules);
                        // Step 4: Update the UnManaged Exports : Set Inactive as true
                        rule.setInactive(true);
                        // Step 5 : Keep this list as updated.
                        inActiveUnManagedExportRules.add(rule);
                    }
                }
            }
            if (unManagedFileSystem.getHasShares()) {
                filesystem.setSMBFileShares(PropertySetterUtil.convertUnManagedSMBMapToManaged(unManagedFileSystem.getUnManagedSmbShareMap(), sPort, dataMover));
                _logger.info("Share map for {} = {}", fsName, filesystem.getSMBFileShares());
                // Process Exports
                // Step 1 : Query them and Retrive associated Exports
                List<UnManagedCifsShareACL> cifsACLs = queryDBCifsShares(unManagedFileSystem);
                _logger.info("Number of Cifs ACL Found : {} for UnManaged Fs path : {}", cifsACLs.size(), unManagedFileSystem.getMountPath());
                if (cifsACLs != null && !cifsACLs.isEmpty()) {
                    for (UnManagedCifsShareACL umCifsAcl : cifsACLs) {
                        // Step 2 : Convert them to Cifs Share ACL
                        // Step 3 : Keep them as a list to store in db, down the line at a shot
                        // Important to relate the shares to a
                        umCifsAcl.setFileSystemId(filesystem.getId());
                        // FileSystem.
                        createACL(umCifsAcl, fsCifsShareAcls, filesystem);
                        // Step 4: Update the UnManaged Share ACL : Set Inactive as true
                        umCifsAcl.setInactive(true);
                        // Step 5 : Keep this list as updated.
                        inActiveUnManagedShareCifs.add(umCifsAcl);
                    }
                }
            }
            if (unManagedFileSystem.getHasNFSAcl()) {
                List<UnManagedNFSShareACL> nfsACLs = queryDBNfsShares(unManagedFileSystem);
                if (nfsACLs != null && !nfsACLs.isEmpty()) {
                    for (UnManagedNFSShareACL umNfsAcl : nfsACLs) {
                        // Step 2 : Convert them to nfs Share ACL
                        // Step 3 : Keep them as a list to store in db, down the line at a shot
                        // Important to relate the shares to a
                        umNfsAcl.setFileSystemId(filesystem.getId());
                        // FileSystem.
                        if (umNfsAcl.getPermissions().isEmpty()) {
                            continue;
                        }
                        createNFSACL(umNfsAcl, fsNfsShareAcls, filesystem);
                        // Step 4: Update the UnManaged Share ACL : Set Inactive as true
                        umNfsAcl.setInactive(true);
                        // Step 5 : Keep this list as updated.
                        inActiveUnManagedShareNfs.add(umNfsAcl);
                    }
                }
            }
            // Set quota
            if (null != unManagedFileSystem.getExtensions() && null != unManagedFileSystem.getExtensions().get(QUOTA)) {
                if (null == filesystem.getExtensions()) {
                    filesystem.setExtensions(new StringMap());
                }
                filesystem.getExtensions().put(QUOTA, unManagedFileSystem.getExtensions().get(QUOTA));
            }
            filesystems.add(PropertySetterUtil.addFileSystemDetails(unManagedFileSystemInformation, filesystem));
            // Process Export Rules for the validated FS.
            filesystemList.getFilesystems().add(toNamedRelatedResource(ResourceTypeEnum.FILE, filesystem.getId(), filesystem.getNativeGuid()));
            unManagedFileSystem.setInactive(true);
            unManagedFileSystems.add(unManagedFileSystem);
            unManagedFSURIToFSMap.put(unManagedFileSystemUri, filesystem);
        }
        int i = 0;
        // Test
        for (FileShare fs : filesystems) {
            ++i;
            _logger.info("{} --> Saving FS to DB {}", i, fs);
            _logger.info(" --> Fs  Storage Pool {} and Virtual Pool {}", fs.getPool(), fs.getVirtualPool());
        }
        _dbClient.createObject(filesystems);
        for (URI unManagedFSURI : param.getUnManagedFileSystems()) {
            FileShare fs = unManagedFSURIToFSMap.get(unManagedFSURI);
            if (fs != null) {
                _logger.debug("ingesting quota directories for filesystem {}", fs.getId());
                ingestFileQuotaDirectories(fs);
            }
        }
        i = 0;
        // Test
        for (FileExportRule rule : fsExportRules) {
            ++i;
            _logger.info("{} --> Saving Export rule to DB {}", i, rule);
        }
        // Step 6.1 : Update the same in DB & Add new export rules
        _dbClient.createObject(fsExportRules);
        // Step 6.2 : Update Cifs Acls in DB & Add new ACLs
        i = 0;
        for (CifsShareACL acl : fsCifsShareAcls) {
            ++i;
            _logger.info("{} --> Saving New Cifs ACL to DB {}", i, acl);
        }
        if (fsCifsShareAcls != null && !fsCifsShareAcls.isEmpty()) {
            _dbClient.createObject(fsCifsShareAcls);
        }
        // Step 7.1 : Update the same in DB & clean ingested UnManagedCifsACLs
        i = 0;
        for (UnManagedCifsShareACL acl : inActiveUnManagedShareCifs) {
            ++i;
            _logger.info("{} Updating UnManagedACL DB as InActive TRUE {}", acl);
        }
        _dbClient.updateObject(inActiveUnManagedShareCifs);
        // Step 7.2 : Update the same in DB & clean Unmanaged ExportRule
        i = 0;
        for (UnManagedFileExportRule rule : inActiveUnManagedExportRules) {
            ++i;
            _logger.info("{} Updating DB as InActive TRUE {}", rule);
        }
        _dbClient.updateObject(inActiveUnManagedExportRules);
        _dbClient.updateObject(unManagedFileSystems);
        // Step 8.1 : Update NFS Acls in DB & Add new ACLs
        if (fsNfsShareAcls != null && !fsNfsShareAcls.isEmpty()) {
            _logger.info("Saving {} NFS ACLs to DB", fsNfsShareAcls.size());
            _dbClient.createObject(fsNfsShareAcls);
        }
        // UnManagedNFSShareACLs
        if (inActiveUnManagedShareNfs != null && !inActiveUnManagedShareNfs.isEmpty()) {
            _logger.info("Saving {} UnManagedNFS ACLs to DB", inActiveUnManagedShareNfs.size());
            _dbClient.updateObject(inActiveUnManagedShareNfs);
        }
        // record the events after they have been created
        for (FileShare filesystem : filesystems) {
            recordFileSystemOperation(_dbClient, OperationTypeEnum.INGEST_FILE_SYSTEM, Status.ready, filesystem.getId());
        }
    } catch (InternalException e) {
        throw e;
    } catch (Exception e) {
        _logger.error("Unexpected exception:", e);
        throw APIException.internalServerErrors.genericApisvcError(e.getMessage(), e);
    }
    return filesystemList;
}

Example 23 with UnManagedFileExportRule

use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedFileExportRule in project coprhd-controller by CoprHD.

the class IsilonCommunicationInterface method discoverUmanagedFileSystems.

private void discoverUmanagedFileSystems(AccessProfile profile) throws BaseCollectionException {
    List<UnManagedFileSystem> newUnManagedFileSystems = new ArrayList<>();
    List<UnManagedFileSystem> existingUnManagedFileSystems = new ArrayList<>();
    List<UnManagedFileQuotaDirectory> newUnManagedFileQuotaDir = new ArrayList<>();
    List<UnManagedFileQuotaDirectory> existingUnManagedFileQuotaDir = new ArrayList<>();
    List<UnManagedCifsShareACL> newUnManagedCifsShareACLList = new ArrayList<>();
    List<UnManagedCifsShareACL> oldUnManagedCifsShareACLList = new ArrayList<>();
    List<UnManagedNFSShareACL> newUnManagedNfsShareACLList = new ArrayList<>();
    List<UnManagedNFSShareACL> oldUnManagedNfsShareACLList = new ArrayList<>();
    List<UnManagedFileExportRule> newUnManagedExportRules = new ArrayList<>();
    List<UnManagedFileExportRule> oldUnManagedExportRules = new ArrayList<>();
    _log.debug("Access Profile Details :  IpAddress : PortNumber : {}, namespace : {}", profile.getIpAddress() + profile.getPortNumber(), profile.getnamespace());
    URI storageSystemId = profile.getSystemId();
    StorageSystem storageSystem = _dbClient.queryObject(StorageSystem.class, storageSystemId);
    if (null == storageSystem) {
        return;
    }
    Set<URI> allDiscoveredUnManagedFileSystems = new HashSet<>();
    String detailedStatusMessage = "Discovery of Isilon Unmanaged FileSystem started";
    long unmanagedFsCount = 0;
    try {
        IsilonApi isilonApi = getIsilonDevice(storageSystem);
        URIQueryResultList storagePoolURIs = new URIQueryResultList();
        _dbClient.queryByConstraint(ContainmentConstraint.Factory.getStorageDeviceStoragePoolConstraint(storageSystem.getId()), storagePoolURIs);
        ArrayList<StoragePool> pools = new ArrayList<>();
        Iterator<URI> poolsItr = storagePoolURIs.iterator();
        while (poolsItr.hasNext()) {
            URI storagePoolURI = poolsItr.next();
            StoragePool storagePool = _dbClient.queryObject(StoragePool.class, storagePoolURI);
            if (storagePool != null && !storagePool.getInactive()) {
                pools.add(storagePool);
            }
        }
        StoragePool storagePool = null;
        if (pools != null && !pools.isEmpty()) {
            storagePool = pools.get(0);
        }
        StoragePort storagePort = getStoragePortPool(storageSystem);
        int totalIsilonFSDiscovered = 0;
        // get the associated storage port for vnas Server
        List<IsilonAccessZone> isilonAccessZones = isilonApi.getAccessZones(null);
        Map<String, NASServer> nasServers = validateAndGetNASServer(storageSystem, isilonAccessZones);
        // update the path from controller configuration
        updateDiscoveryPathForUnManagedFS(nasServers, storageSystem);
        // NFSv4 enabled on storage system!!!
        boolean isNfsV4Enabled = isilonApi.nfsv4Enabled(storageSystem.getFirmwareVersion());
        List<FileShare> discoveredFS = new ArrayList<>();
        String resumeToken = null;
        for (String umfsDiscoverPath : _discPathsForUnManaged) {
            IsilonAccessZone isilonAccessZone = getAccessZoneCorresDiscoveryPath(isilonAccessZones, umfsDiscoverPath);
            String isilonAccessZoneName;
            if (isilonAccessZone == null) {
                // System access zone
                isilonAccessZoneName = null;
            } else {
                isilonAccessZoneName = isilonAccessZone.getName();
            }
            // Get All SMB for this path access zone
            HashMap<String, HashSet<String>> zoneSMBShares = discoverAccessZoneSMBShares(storageSystem, isilonAccessZoneName);
            // Get all NFS Export for this path access zone
            HashMap<String, HashSet<Integer>> zoneNFSExports = discoverAccessZoneExports(storageSystem, isilonAccessZoneName);
            do {
                HashMap<String, Object> discoverdFileDetails = discoverAllFileSystem(storageSystem, resumeToken, umfsDiscoverPath);
                IsilonApi.IsilonList<FileShare> discoveredIsilonFS = (IsilonApi.IsilonList<FileShare>) discoverdFileDetails.get(UMFS_DETAILS);
                ArrayList<UnManagedFileQuotaDirectory> discoveredUmfsQd = (ArrayList<UnManagedFileQuotaDirectory>) discoverdFileDetails.get(UMFSQD_DETAILS);
                HashMap<String, HashSet<String>> umfsfileQuotaMap = (HashMap<String, HashSet<String>>) discoverdFileDetails.get(UMFS_QD_MAP);
                resumeToken = discoveredIsilonFS.getToken();
                discoveredFS = discoveredIsilonFS.getList();
                totalIsilonFSDiscovered += discoveredFS.size();
                for (FileShare fs : discoveredFS) {
                    if (!checkStorageFileSystemExistsInDB(fs.getNativeGuid())) {
                        // Create UnManaged FS
                        String fsPathName = fs.getPath();
                        UnManagedFileSystem unManagedFs = checkUnManagedFileSystemExistsInDB(fs.getNativeGuid());
                        if (unManagedFs != null) {
                            existingUnManagedFileSystems.add(unManagedFs);
                        }
                        // get the matched vNAS Server
                        NASServer nasServer = getMatchedNASServer(nasServers, fsPathName);
                        if (nasServer != null) {
                            // Get valid storage port from the NAS server!!!
                            _log.info("fs path {} and nas server details {}", fs.getPath(), nasServer.toString());
                            storagePort = getStoragePortFromNasServer(nasServer);
                            if (storagePort == null) {
                                _log.info("No valid storage port found for nas server {}", nasServer.toString());
                                continue;
                            }
                        } else {
                            _log.info("fs path {} and vnas server not found", fs.getPath());
                            // Skip further ingestion steps on this file share & move to next file share
                            continue;
                        }
                        unManagedFs = createUnManagedFileSystem(unManagedFs, fs.getNativeGuid(), storageSystem, storagePool, nasServer, fs);
                        unManagedFs.setHasNFSAcl(false);
                        newUnManagedFileSystems.add(unManagedFs);
                        /**
                         * Set and create the NFS ACLs only if the system is enabled with NFSv4!!!
                         */
                        HashMap<String, HashSet<Integer>> exportWithIdMap = getExportsIncludingSubDir(fs.getPath(), zoneNFSExports, umfsfileQuotaMap);
                        if (isNfsV4Enabled) {
                            Set<String> fsExportPaths = exportWithIdMap.keySet();
                            setUnmanagedNfsShareACL(unManagedFs, storageSystem, isilonApi, fsExportPaths, newUnManagedNfsShareACLList, oldUnManagedNfsShareACLList);
                        }
                        /**
                         * Set and Create Export Rules and export Map
                         */
                        if (!exportWithIdMap.keySet().isEmpty()) {
                            setUnManagedFSExportMap(unManagedFs, exportWithIdMap, storagePort, fs.getPath(), nasServer.getNasName(), isilonApi, storageSystem, newUnManagedExportRules, oldUnManagedExportRules);
                            _log.info("Number of exports discovered for file system {} is {}", unManagedFs.getId(), newUnManagedExportRules.size());
                            if (!newUnManagedExportRules.isEmpty()) {
                                unManagedFs.setHasExports(true);
                                _log.info("File System {} has Exports and their size is {}", unManagedFs.getId(), newUnManagedExportRules.size());
                            }
                        }
                        /**
                         * Create and set CIFS ACLS and SMB Share MAP
                         */
                        HashSet<String> shareIDs = getSharesIncludingSubDir(fs.getPath(), zoneSMBShares, umfsfileQuotaMap);
                        setUnmanagedCifsShareACL(unManagedFs, shareIDs, newUnManagedCifsShareACLList, storagePort, fs.getName(), nasServer.getNasName(), storageSystem, isilonApi, oldUnManagedCifsShareACLList);
                        _log.info("Number of shares ACLs discovered for file system {} is {}", unManagedFs.getId(), newUnManagedCifsShareACLList.size());
                        if (unManagedFs.getHasExports() || unManagedFs.getHasShares()) {
                            _log.info("FS {} is having exports/shares", fs.getPath());
                            unManagedFs.putFileSystemCharacterstics(UnManagedFileSystem.SupportedFileSystemCharacterstics.IS_FILESYSTEM_EXPORTED.toString(), TRUE);
                        } else {
                            _log.info("FS {} does not have export or share", fs.getPath());
                        }
                        /**
                         * Persist 200 objects and clear them to avoid memory issue
                         */
                        // save bunch of export rules in db
                        validateSizeLimitAndPersist(newUnManagedExportRules, oldUnManagedExportRules, Constants.DEFAULT_PARTITION_SIZE * 2);
                        // save bunch of ACLs in db
                        validateSizeLimitAndPersist(newUnManagedCifsShareACLList, oldUnManagedCifsShareACLList, Constants.DEFAULT_PARTITION_SIZE * 2);
                        // save bunch of NFS ACLs in db
                        validateSizeLimitAndPersist(newUnManagedNfsShareACLList, newUnManagedNfsShareACLList, Constants.DEFAULT_PARTITION_SIZE * 2);
                        allDiscoveredUnManagedFileSystems.add(unManagedFs.getId());
                        // save bunch of file system in db
                        validateListSizeLimitAndPersist(newUnManagedFileSystems, existingUnManagedFileSystems, Constants.DEFAULT_PARTITION_SIZE * 2);
                    }
                }
                for (UnManagedFileQuotaDirectory umfsQd : discoveredUmfsQd) {
                    if (!checkStorageQuotaDirectoryExistsInDB(umfsQd.getNativeGuid())) {
                        String fsUnManagedQdNativeGuid = NativeGUIDGenerator.generateNativeGuidForUnManagedQuotaDir(storageSystem.getSystemType(), storageSystem.getSerialNumber(), umfsQd.getNativeId(), "");
                        UnManagedFileQuotaDirectory unManagedFileQd = checkUnManagedFileSystemQuotaDirectoryExistsInDB(fsUnManagedQdNativeGuid);
                        boolean umfsQdExists = (unManagedFileQd == null) ? false : true;
                        if (umfsQdExists) {
                            umfsQd.setId(unManagedFileQd.getId());
                            existingUnManagedFileQuotaDir.add(umfsQd);
                        } else if (null != umfsQd) {
                            umfsQd.setId(URIUtil.createId(UnManagedFileQuotaDirectory.class));
                            newUnManagedFileQuotaDir.add(umfsQd);
                        }
                    }
                }
                // save bunch of QDs in db
                validateSizeLimitAndPersist(newUnManagedFileQuotaDir, existingUnManagedFileQuotaDir, Constants.DEFAULT_PARTITION_SIZE * 2);
            } while (resumeToken != null);
        }
        // Saving bunch of Unmanaged objects!!!
        if (!newUnManagedExportRules.isEmpty()) {
            _log.info("Saving Number of UnManagedFileExportRule(s) {}", newUnManagedExportRules.size());
            _dbClient.createObject(newUnManagedExportRules);
            newUnManagedExportRules.clear();
        }
        if (!oldUnManagedExportRules.isEmpty()) {
            _log.info("Saving Number of UnManagedFileExportRule(s) {}", newUnManagedExportRules.size());
            _dbClient.updateObject(newUnManagedExportRules);
            oldUnManagedExportRules.clear();
        }
        // save ACLs in db
        if (!newUnManagedCifsShareACLList.isEmpty()) {
            _log.info("Saving Number of UnManagedCifsShareACL(s) {}", newUnManagedCifsShareACLList.size());
            _dbClient.createObject(newUnManagedCifsShareACLList);
            newUnManagedCifsShareACLList.clear();
        }
        // save old acls
        if (!oldUnManagedCifsShareACLList.isEmpty()) {
            _log.info("Saving Number of UnManagedFileExportRule(s) {}", oldUnManagedCifsShareACLList.size());
            _dbClient.updateObject(oldUnManagedCifsShareACLList);
            oldUnManagedCifsShareACLList.clear();
        }
        // save NFS ACLs in db
        if (!newUnManagedNfsShareACLList.isEmpty()) {
            _log.info("Saving Number of UnManagedNfsShareACL(s) {}", newUnManagedNfsShareACLList.size());
            _dbClient.createObject(newUnManagedNfsShareACLList);
            newUnManagedNfsShareACLList.clear();
        }
        // save old acls
        if (!oldUnManagedNfsShareACLList.isEmpty()) {
            _log.info("Saving Number of NFS UnManagedFileExportRule(s) {}", oldUnManagedNfsShareACLList.size());
            _dbClient.updateObject(oldUnManagedNfsShareACLList);
            oldUnManagedNfsShareACLList.clear();
        }
        // save new QDs to DB
        if (!newUnManagedFileQuotaDir.isEmpty()) {
            _log.info("New unmanaged Isilon file systems QuotaDirecotry  count: {}", newUnManagedFileQuotaDir.size());
            _dbClient.createObject(newUnManagedFileQuotaDir);
        }
        // save old QDs
        if (!existingUnManagedFileQuotaDir.isEmpty()) {
            _log.info("Update unmanaged Isilon file systems QuotaDirectory count: {}", existingUnManagedFileQuotaDir.size());
            _dbClient.updateObject(existingUnManagedFileQuotaDir);
        }
        // save new FS
        if (!newUnManagedFileSystems.isEmpty()) {
            _dbClient.createObject(newUnManagedFileSystems);
        }
        // save old FS
        if (!existingUnManagedFileSystems.isEmpty()) {
            _dbClient.updateObject(existingUnManagedFileSystems);
        }
        _log.info("Discovered {} Isilon file systems.", totalIsilonFSDiscovered);
        // discovery succeeds
        detailedStatusMessage = String.format("Discovery completed successfully for Isilon: %s; new unmanaged file systems count: %s", storageSystemId.toString(), unmanagedFsCount);
        _log.info(detailedStatusMessage);
    } catch (IsilonException ex) {
        detailedStatusMessage = String.format("Discovery failed for Isilon %s because %s", storageSystemId.toString(), ex.getLocalizedMessage());
        _log.error(detailedStatusMessage, ex);
        throw ex;
    } catch (Exception e) {
        detailedStatusMessage = String.format("Discovery failed for Isilon %s because %s", storageSystemId.toString(), e.getLocalizedMessage());
        _log.error(detailedStatusMessage, e);
        throw new IsilonCollectionException(detailedStatusMessage);
    } finally {
        if (storageSystem != null) {
            try {
                // set detailed message
                storageSystem.setLastDiscoveryStatusMessage(detailedStatusMessage);
                _dbClient.updateObject(storageSystem);
            } catch (Exception ex) {
                _log.error("Error while persisting object to DB", ex);
            }
        }
    }
}

Example 24 with UnManagedFileExportRule

use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedFileExportRule in project coprhd-controller by CoprHD.

the class IsilonCommunicationInterface method getUnManagedFSExportRules.

/**
 * Generate Export Map for UnManagedFileSystem
 * Ignore exports with multiple exports for the same path
 * Ignore exports that have multiple security flavors
 * Ignore exports with multiple paths
 * Ignore exports not found on the array
 * Ignore exports which have the same internal export key ( <sec, perm, root-mapping>)
 *
 * @param umfs
 * @param isilonExportIds
 * @param storagePort
 * @param fsPath
 * @param isilonApi
 * @return boolean
 */
private List<UnManagedFileExportRule> getUnManagedFSExportRules(UnManagedFileSystem umfs, StoragePort storagePort, HashSet<Integer> isilonExportIds, String fsPath, String zoneName, IsilonApi isilonApi) {
    List<UnManagedFileExportRule> expRules = new ArrayList<UnManagedFileExportRule>();
    ArrayList<IsilonExport> isilonExports = new ArrayList<IsilonExport>();
    if (isilonExportIds != null && isilonExportIds.size() > 1) {
        _log.info("Ignoring file system {}, Multiple export rulues found {} ", fsPath, isilonExportIds.size());
    }
    for (Integer expId : isilonExportIds) {
        IsilonExport exp = getIsilonExport(isilonApi, expId, zoneName);
        if (exp == null) {
            _log.info("Ignoring file system {}, export {} not found", fsPath, expId);
        } else if (exp.getSecurityFlavors().size() > 1) {
            _log.info("Ignoring file system {}, multiple security flavors {} found", fsPath, exp.getSecurityFlavors().toString());
        } else if (exp.getPaths().size() > 1) {
            _log.info("Ignoring file system {}, multiple paths {} found", fsPath, exp.getPaths().toString());
        } else {
            isilonExports.add(exp);
        }
    }
    for (IsilonExport exp : isilonExports) {
        String securityFlavor = exp.getSecurityFlavors().get(0);
        // Isilon Maps sys to unix and we do this conversion during export from ViPR
        if (securityFlavor.equalsIgnoreCase(UNIXSECURITY)) {
            securityFlavor = SYSSECURITY;
        }
        String path = exp.getPaths().get(0);
        // Get User
        String rootUserMapping = "";
        String mapAllUserMapping = "";
        if (exp.getMap_root() != null && exp.getMap_root().getUser() != null) {
            rootUserMapping = exp.getMap_root().getUser();
        } else if (exp.getMap_all() != null && exp.getMap_all().getUser() != null) {
            mapAllUserMapping = exp.getMap_all().getUser();
        }
        String resolvedUser = (rootUserMapping != null && (!rootUserMapping.isEmpty())) ? rootUserMapping : mapAllUserMapping;
        UnManagedFileExportRule expRule = new UnManagedFileExportRule();
        expRule.setExportPath(path);
        expRule.setSecFlavor(securityFlavor);
        expRule.setAnon(resolvedUser);
        expRule.setDeviceExportId(exp.getId().toString());
        expRule.setFileSystemId(umfs.getId());
        expRule.setMountPoint(storagePort.getPortNetworkId() + ":" + fsPath);
        if (exp != null && exp.getReadOnlyClients() != null && !exp.getReadOnlyClients().isEmpty()) {
            expRule.setReadOnlyHosts(new StringSet(exp.getReadOnlyClients()));
        }
        if (exp != null && exp.getReadWriteClients() != null && !exp.getReadWriteClients().isEmpty()) {
            expRule.setReadWriteHosts(new StringSet(exp.getReadWriteClients()));
        }
        if (exp != null && exp.getRootClients() != null && !exp.getRootClients().isEmpty()) {
            expRule.setRootHosts(new StringSet(exp.getRootClients()));
        }
        if (exp.getReadOnlyClients() != null && exp.getReadWriteClients() != null && exp.getRootClients() != null) {
            // Check Clients size
            if (exp.getReadOnlyClients().isEmpty() && exp.getReadWriteClients().isEmpty() && exp.getRootClients().isEmpty()) {
                if (exp.getReadOnly()) {
                    // This is a read only export for all hosts
                    expRule.setReadOnlyHosts(new StringSet(exp.getClients()));
                } else {
                    // Not read Only case
                    if (exp.getMap_all() != null && exp.getMap_all().getUser() != null && exp.getMap_all().getUser().equalsIgnoreCase(ROOT)) {
                        // All hosts with root permission
                        expRule.setRootHosts(new StringSet(exp.getClients()));
                    } else if (exp.getMap_all() != null) {
                        // All hosts with RW permission
                        expRule.setReadWriteHosts(new StringSet(exp.getClients()));
                    }
                }
            }
        }
        expRules.add(expRule);
    }
    return expRules;
}

Example 25 with UnManagedFileExportRule

use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedFileExportRule in project coprhd-controller by CoprHD.

the class IsilonCommunicationInterface method setUnManagedFSExportMap.

/**
 * Generate Export Map for UnManagedFileSystem
 * Ignore exports with multiple exports for the same path
 * Ignore exports that have multiple security flavors
 * Ignore exports with multiple paths
 * Ignore exports not found on the array
 * Ignore exports which have the same internal export key ( <sec, perm, root-mapping>)
 *
 * @param umfs
 * @param expIdMap
 * @param storagePort
 * @param fsPath
 * @param isilonApi
 * @return boolean
 */
private void setUnManagedFSExportMap(UnManagedFileSystem umfs, HashMap<String, HashSet<Integer>> expIdMap, StoragePort storagePort, String fsPath, String zoneName, IsilonApi isilonApi, StorageSystem storageSystem, List<UnManagedFileExportRule> newUnManagedExportRules, List<UnManagedFileExportRule> oldUnManagedExportRules) {
    boolean validExports = false;
    List<UnManagedFileExportRule> exportRules = new ArrayList<>();
    for (Entry<String, HashSet<Integer>> entry : expIdMap.entrySet()) {
        HashSet<Integer> isilonExportIds = entry.getValue();
        _log.info("getting exports for the path {} with id {}", entry.getKey(), entry.getValue());
        List<UnManagedFileExportRule> exportRulesForPath = new ArrayList<>();
        if (isilonExportIds != null && !isilonExportIds.isEmpty()) {
            validExports = getUnManagedFSExportMap(umfs, isilonExportIds, storagePort, fsPath, zoneName, isilonApi, exportRulesForPath);
        }
        if (!validExports) {
            // Clear the export rule list for this path,
            // if there are some invalid export rule at this path
            exportRulesForPath.clear();
        }
        if (!exportRulesForPath.isEmpty()) {
            exportRules.addAll(exportRulesForPath);
            exportRulesForPath.clear();
        }
    }
    UnManagedFileExportRule existingRule = null;
    for (UnManagedFileExportRule dbExportRule : exportRules) {
        _log.info("Un Managed File Export Rule : {}", dbExportRule);
        String fsExportRulenativeId = dbExportRule.getFsExportIndex();
        _log.info("Native Id using to build Native Guid {}", fsExportRulenativeId);
        String fsUnManagedFileExportRuleNativeGuid = NativeGUIDGenerator.generateNativeGuidForPreExistingFileExportRule(storageSystem, fsExportRulenativeId);
        _log.info("Native GUID {}", fsUnManagedFileExportRuleNativeGuid);
        dbExportRule.setNativeGuid(fsUnManagedFileExportRuleNativeGuid);
        dbExportRule.setId(URIUtil.createId(UnManagedFileExportRule.class));
        existingRule = checkUnManagedFsExportRuleExistsInDB(_dbClient, dbExportRule.getNativeGuid());
        if (null != existingRule) {
            existingRule.setInactive(true);
            oldUnManagedExportRules.add(existingRule);
        }
        newUnManagedExportRules.add(dbExportRule);
    }
}