Examples with NamedFileSystemList com.emc.storageos.model.file.NamedFileSystemList used on opensource projects

Example 1 with NamedFileSystemList

use of com.emc.storageos.model.file.NamedFileSystemList in project coprhd-controller by CoprHD.

the class UnManagedFilesystemService method ingestFileSystems.

/**
 * UnManaged file systems are file systems, which are present within ViPR
 * storage systems,but have not been ingested by ViPR which moves the unmanaged file systems under ViPR management.
 *
 * File system ingest provides flexibility in bringing unmanaged
 * file systems under ViPR management.
 * An unmanaged file system must be associated with a virtual pool, project,
 * and virtual array before it can be managed by ViPR.
 * List of supported virtual pools for each unmanaged file system is exposed using /vdc/unmanaged/filesystems/bulk.
 * Using an unsupported virtual pool results in an error
 *
 * Size of unmanaged file systems which can be ingested via a single API Call
 * is limited to 4000.
 *
 * @param param
 *            parameters required for unmanaged filesystem ingestion
 *
 * @prereq none
 * @brief Ingest unmanaged file systems
 * @throws InternalException
 */
@POST
@Consumes({ MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON })
@Produces({ MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON })
@Path("/ingest")
@CheckPermission(roles = { Role.SYSTEM_ADMIN, Role.RESTRICTED_SYSTEM_ADMIN })
public NamedFileSystemList ingestFileSystems(FileSystemIngest param) throws InternalException {
    if ((null == param.getUnManagedFileSystems()) || (param.getUnManagedFileSystems().toString().length() == 0) || (param.getUnManagedFileSystems().isEmpty()) || (param.getUnManagedFileSystems().get(0).toString().isEmpty())) {
        throw APIException.badRequests.invalidParameterUnManagedFsListEmpty();
    }
    if (null == param.getProject() || (param.getProject().toString().length() == 0)) {
        throw APIException.badRequests.invalidParameterProjectEmpty();
    }
    if (null == param.getVarray() || (param.getVarray().toString().length() == 0)) {
        throw APIException.badRequests.invalidParameterVirtualArrayEmpty();
    }
    if (null == param.getVpool() || (param.getVpool().toString().length() == 0)) {
        throw APIException.badRequests.invalidParameterVirtualPoolEmpty();
    }
    if (param.getUnManagedFileSystems().size() > getMaxBulkSize()) {
        throw APIException.badRequests.exceedingLimit("unmanaged filesystems", getMaxBulkSize());
    }
    _logger.info("Ingest called with Virtual Array {}", param.getVarray());
    _logger.info("Ingest called with Virtual Pool {}", param.getVpool());
    _logger.info("Ingest called with Project {}", param.getProject());
    _logger.info("Ingest called with UnManagedFileSystems {}", param.getUnManagedFileSystems());
    NamedFileSystemList filesystemList = new NamedFileSystemList();
    List<UnManagedFileSystem> unManagedFileSystems = new ArrayList<UnManagedFileSystem>();
    try {
        // Get and validate the project.
        Project project = _permissionsHelper.getObjectById(param.getProject(), Project.class);
        ArgValidator.checkUri(param.getProject());
        ArgValidator.checkEntity(project, param.getProject(), false);
        VirtualArray neighborhood = FileSystemIngestionUtil.getVirtualArrayForFileSystemCreateRequest(project, param.getVarray(), _permissionsHelper, _dbClient);
        // Get and validate the VirtualPool.
        VirtualPool cos = FileSystemIngestionUtil.getVirtualPoolForFileSystemCreateRequest(project, param.getVpool(), _permissionsHelper, _dbClient);
        if (null != cos.getVirtualArrays() && !cos.getVirtualArrays().isEmpty() && !cos.getVirtualArrays().contains(param.getVarray().toString())) {
            throw APIException.internalServerErrors.virtualPoolNotMatchingVArray(param.getVarray());
        }
        // check for Quotas
        long unManagedFileSystemsCapacity = FileSystemIngestionUtil.getTotalUnManagedFileSystemCapacity(_dbClient, param.getUnManagedFileSystems());
        _logger.info("Requested UnManagedFile System Capacity {}", unManagedFileSystemsCapacity);
        TenantOrg tenant = _dbClient.queryObject(TenantOrg.class, project.getTenantOrg().getURI());
        CapacityUtils.validateQuotasForProvisioning(_dbClient, cos, project, tenant, unManagedFileSystemsCapacity, "filesystem");
        FileSystemIngestionUtil.isIngestionRequestValidForUnManagedFileSystems(param.getUnManagedFileSystems(), cos, _dbClient);
        List<FileShare> filesystems = new ArrayList<FileShare>();
        Map<URI, FileShare> unManagedFSURIToFSMap = new HashMap<>();
        List<FileExportRule> fsExportRules = new ArrayList<FileExportRule>();
        List<CifsShareACL> fsCifsShareAcls = new ArrayList<CifsShareACL>();
        List<NFSShareACL> fsNfsShareAcls = new ArrayList<NFSShareACL>();
        List<UnManagedFileExportRule> inActiveUnManagedExportRules = new ArrayList<UnManagedFileExportRule>();
        List<UnManagedCifsShareACL> inActiveUnManagedShareCifs = new ArrayList<UnManagedCifsShareACL>();
        List<UnManagedNFSShareACL> inActiveUnManagedShareNfs = new ArrayList<UnManagedNFSShareACL>();
        // cifs share acl's
        List<CifsShareACL> cifsShareACLList = new ArrayList<CifsShareACL>();
        List<URI> full_pools = new ArrayList<URI>();
        List<URI> full_systems = new ArrayList<URI>();
        Calendar timeNow = Calendar.getInstance();
        for (URI unManagedFileSystemUri : param.getUnManagedFileSystems()) {
            long softLimit = 0;
            int softGrace = 0;
            long notificationLimit = 0;
            UnManagedFileSystem unManagedFileSystem = _dbClient.queryObject(UnManagedFileSystem.class, unManagedFileSystemUri);
            if (null == unManagedFileSystem || null == unManagedFileSystem.getFileSystemCharacterstics() || null == unManagedFileSystem.getFileSystemInformation()) {
                _logger.warn("UnManaged FileSystem {} partially discovered, hence not enough information available to validate neither virtualPool nor other criterias.Skipping Ingestion..", unManagedFileSystemUri);
                continue;
            }
            if (unManagedFileSystem.getInactive()) {
                _logger.warn("UnManaged FileSystem {} is inactive.Skipping Ingestion..", unManagedFileSystemUri);
                continue;
            }
            if (!FileSystemIngestionUtil.checkVirtualPoolValidForUnManagedFileSystem(_dbClient, cos, unManagedFileSystemUri)) {
                continue;
            }
            StringSetMap unManagedFileSystemInformation = unManagedFileSystem.getFileSystemInformation();
            String fsNativeGuid = unManagedFileSystem.getNativeGuid().replace(FileSystemIngestionUtil.UNMANAGEDFILESYSTEM, FileSystemIngestionUtil.FILESYSTEM);
            String deviceLabel = PropertySetterUtil.extractValueFromStringSet(SupportedFileSystemInformation.DEVICE_LABEL.toString(), unManagedFileSystemInformation);
            String fsName = PropertySetterUtil.extractValueFromStringSet(SupportedFileSystemInformation.NAME.toString(), unManagedFileSystemInformation);
            URI storagePoolUri = unManagedFileSystem.getStoragePoolUri();
            String storagePortUri = PropertySetterUtil.extractValueFromStringSet(SupportedFileSystemInformation.STORAGE_PORT.toString(), unManagedFileSystemInformation);
            String capacity = PropertySetterUtil.extractValueFromStringSet(SupportedFileSystemInformation.PROVISIONED_CAPACITY.toString(), unManagedFileSystemInformation);
            String usedCapacity = PropertySetterUtil.extractValueFromStringSet(SupportedFileSystemInformation.ALLOCATED_CAPACITY.toString(), unManagedFileSystemInformation);
            String nasUri = PropertySetterUtil.extractValueFromStringSet(SupportedFileSystemInformation.NAS.toString(), unManagedFileSystemInformation);
            String path = PropertySetterUtil.extractValueFromStringSet(SupportedFileSystemInformation.PATH.toString(), unManagedFileSystemInformation);
            String mountPath = PropertySetterUtil.extractValueFromStringSet(SupportedFileSystemInformation.MOUNT_PATH.toString(), unManagedFileSystemInformation);
            String nativeId = PropertySetterUtil.extractValueFromStringSet(SupportedFileSystemInformation.NATIVE_ID.toString(), unManagedFileSystemInformation);
            String systemType = PropertySetterUtil.extractValueFromStringSet(SupportedFileSystemInformation.SYSTEM_TYPE.toString(), unManagedFileSystemInformation);
            String softLt = PropertySetterUtil.extractValueFromStringSet(SupportedFileSystemInformation.SOFT_LIMIT.toString(), unManagedFileSystemInformation);
            String softGr = PropertySetterUtil.extractValueFromStringSet(SupportedFileSystemInformation.SOFT_GRACE.toString(), unManagedFileSystemInformation);
            String notificationLt = PropertySetterUtil.extractValueFromStringSet(SupportedFileSystemInformation.NOTIFICATION_LIMIT.toString(), unManagedFileSystemInformation);
            if (null != softLt && !softLt.isEmpty()) {
                softLimit = Long.valueOf(softLt);
            }
            if (null != softGr && !softGr.isEmpty()) {
                softGrace = Integer.valueOf(softGr);
            }
            if (null != notificationLt && !notificationLt.isEmpty()) {
                notificationLimit = Long.valueOf(notificationLt);
            }
            Long lcapcity = Long.valueOf(capacity);
            Long lusedCapacity = Long.valueOf(usedCapacity);
            // pool uri cannot be null
            StoragePool pool = _dbClient.queryObject(StoragePool.class, storagePoolUri);
            StoragePort port = null;
            if (storagePortUri != null) {
                port = _dbClient.queryObject(StoragePort.class, URI.create(storagePortUri));
            }
            StorageHADomain dataMover = null;
            if (port != null && port.getStorageHADomain() != null) {
                dataMover = _dbClient.queryObject(StorageHADomain.class, port.getStorageHADomain());
            }
            if (dataMover != null) {
                _logger.info("Data Mover to Use {} {} {}", new Object[] { dataMover.getAdapterName(), dataMover.getName(), dataMover.getLabel() });
            }
            // Check for same name File Share in this project
            if (FileSystemIngestionUtil.checkForDuplicateFSName(_dbClient, project.getId(), deviceLabel, filesystems)) {
                _logger.info("File System with name: {}  already exists in the given project: {} so, ignoring it..", deviceLabel, project.getLabel());
                continue;
            }
            // check ingestion is valid for given project
            if (!isIngestUmfsValidForProject(project, _dbClient, nasUri)) {
                _logger.info("UnManaged FileSystem path {} is mounted on vNAS URI {} which is invalid for project.", path, nasUri);
                continue;
            }
            // Check for same named File Share in this project
            if (FileSystemIngestionUtil.checkForDuplicateFSName(_dbClient, project.getId(), deviceLabel, filesystems)) {
                _logger.info("File System with name: {}  already exists in given project: {} so, ignoring it..", deviceLabel, project.getLabel());
                continue;
            }
            // if not don't ingest
            if (null != pool) {
                StringSet taggedVirtualArrays = pool.getTaggedVirtualArrays();
                if ((null == taggedVirtualArrays) || (!taggedVirtualArrays.contains(neighborhood.getId().toString()))) {
                    _logger.warn("UnManaged FileSystem {} storagepool doesn't related to the Virtual Array {}. Skipping Ingestion..", unManagedFileSystemUri, neighborhood.getId().toString());
                    continue;
                }
            } else {
                _logger.warn("UnManaged FileSystem {} doesn't contain a storagepool. Skipping Ingestiong", unManagedFileSystemUri);
                continue;
            }
            if (full_pools.contains(storagePoolUri)) {
                // skip this fileshare
                continue;
            }
            if (pool.getIsResourceLimitSet()) {
                if (pool.getMaxResources() <= StoragePoolService.getNumResources(pool, _dbClient)) {
                    // reached limit for this pool
                    full_pools.add(storagePoolUri);
                    continue;
                }
            }
            FileShare filesystem = new FileShare();
            filesystem.setId(URIUtil.createId(FileShare.class));
            filesystem.setNativeGuid(fsNativeGuid);
            filesystem.setCapacity(lcapcity);
            filesystem.setUsedCapacity(lusedCapacity);
            filesystem.setPath(path);
            filesystem.setMountPath(mountPath);
            filesystem.setVirtualPool(param.getVpool());
            filesystem.setVirtualArray(param.getVarray());
            filesystem.setSoftLimit(softLimit);
            filesystem.setSoftGracePeriod(softGrace);
            filesystem.setNotificationLimit(notificationLimit);
            if (nasUri != null) {
                filesystem.setVirtualNAS(URI.create(nasUri));
                if (!doesNASServerSupportVPoolProtocols(nasUri, cos.getProtocols())) {
                    _logger.warn("UnManaged FileSystem NAS server {} doesn't support vpool protocols. Skipping Ingestion...", nasUri);
                    continue;
                }
            }
            if (nativeId != null) {
                filesystem.setNativeId(nativeId);
            }
            URI storageSystemUri = unManagedFileSystem.getStorageSystemUri();
            StorageSystem system = _dbClient.queryObject(StorageSystem.class, storageSystemUri);
            if (full_systems.contains(storageSystemUri)) {
                // skip this fileshare
                continue;
            }
            if (system.getIsResourceLimitSet()) {
                if (system.getMaxResources() <= StorageSystemService.getNumResources(system, _dbClient)) {
                    // reached limit for this system
                    full_systems.add(storageSystemUri);
                    continue;
                }
            }
            filesystem.setStorageDevice(storageSystemUri);
            filesystem.setCreationTime(timeNow);
            filesystem.setPool(storagePoolUri);
            filesystem.setProtocol(new StringSet());
            StringSet fsSupportedProtocols = new StringSet();
            for (StorageProtocol.File fileProtocol : StorageProtocol.File.values()) {
                fsSupportedProtocols.add(fileProtocol.name());
            }
            // fs support protocol which is present in StoragePool and VirtualPool both
            fsSupportedProtocols.retainAll(pool.getProtocols());
            fsSupportedProtocols.retainAll(cos.getProtocols());
            filesystem.getProtocol().addAll(fsSupportedProtocols);
            filesystem.setLabel(null == deviceLabel ? "" : deviceLabel);
            filesystem.setName(null == fsName ? "" : fsName);
            filesystem.setTenant(new NamedURI(project.getTenantOrg().getURI(), filesystem.getLabel()));
            filesystem.setProject(new NamedURI(param.getProject(), filesystem.getLabel()));
            _logger.info("Un Managed File System {} has exports? : {}", unManagedFileSystem.getId(), unManagedFileSystem.getHasExports());
            StoragePort sPort = null;
            if (port != null && neighborhood != null) {
                if (StorageSystem.Type.isilon.toString().equals(system.getSystemType())) {
                    sPort = getIsilonStoragePort(port, nasUri, neighborhood.getId());
                } else {
                    sPort = compareAndSelectPortURIForUMFS(system, port, neighborhood);
                }
            }
            /*
                 * If UMFS storage port is not part of the vArray then skip ingestion
                 */
            if (sPort == null) {
                _logger.warn("Storage port of UMFS {} doesn't belong to a matching NetWork. So skipping ingestion", unManagedFileSystemUri);
                continue;
            }
            _logger.info("Storage Port Found {}", sPort);
            filesystem.setPortName(sPort.getPortName());
            filesystem.setStoragePort(sPort.getId());
            if (unManagedFileSystem.getHasExports()) {
                filesystem.setFsExports(PropertySetterUtil.convertUnManagedExportMapToManaged(unManagedFileSystem.getFsUnManagedExportMap(), sPort, dataMover));
                _logger.info("Export map for {} = {}", fsName, filesystem.getFsExports());
                // Process Exports
                // Step 1 : Query them and Retrive associated Exports
                List<UnManagedFileExportRule> exports = queryDBFSExports(unManagedFileSystem);
                _logger.info("Number of Exports Found : {} for UnManaged Fs path : {}", exports.size(), unManagedFileSystem.getMountPath());
                if (exports != null && !exports.isEmpty()) {
                    for (UnManagedFileExportRule rule : exports) {
                        // Step 2 : Convert them to File Export Rule
                        // Step 3 : Keep them as a list to store in db, down the line at a shot
                        // Important to relate the exports to a
                        rule.setFileSystemId(filesystem.getId());
                        // FileSystem.
                        createRule(rule, fsExportRules);
                        // Step 4: Update the UnManaged Exports : Set Inactive as true
                        rule.setInactive(true);
                        // Step 5 : Keep this list as updated.
                        inActiveUnManagedExportRules.add(rule);
                    }
                }
            }
            if (unManagedFileSystem.getHasShares()) {
                filesystem.setSMBFileShares(PropertySetterUtil.convertUnManagedSMBMapToManaged(unManagedFileSystem.getUnManagedSmbShareMap(), sPort, dataMover));
                _logger.info("Share map for {} = {}", fsName, filesystem.getSMBFileShares());
                // Process Exports
                // Step 1 : Query them and Retrive associated Exports
                List<UnManagedCifsShareACL> cifsACLs = queryDBCifsShares(unManagedFileSystem);
                _logger.info("Number of Cifs ACL Found : {} for UnManaged Fs path : {}", cifsACLs.size(), unManagedFileSystem.getMountPath());
                if (cifsACLs != null && !cifsACLs.isEmpty()) {
                    for (UnManagedCifsShareACL umCifsAcl : cifsACLs) {
                        // Step 2 : Convert them to Cifs Share ACL
                        // Step 3 : Keep them as a list to store in db, down the line at a shot
                        // Important to relate the shares to a
                        umCifsAcl.setFileSystemId(filesystem.getId());
                        // FileSystem.
                        createACL(umCifsAcl, fsCifsShareAcls, filesystem);
                        // Step 4: Update the UnManaged Share ACL : Set Inactive as true
                        umCifsAcl.setInactive(true);
                        // Step 5 : Keep this list as updated.
                        inActiveUnManagedShareCifs.add(umCifsAcl);
                    }
                }
            }
            if (unManagedFileSystem.getHasNFSAcl()) {
                List<UnManagedNFSShareACL> nfsACLs = queryDBNfsShares(unManagedFileSystem);
                if (nfsACLs != null && !nfsACLs.isEmpty()) {
                    for (UnManagedNFSShareACL umNfsAcl : nfsACLs) {
                        // Step 2 : Convert them to nfs Share ACL
                        // Step 3 : Keep them as a list to store in db, down the line at a shot
                        // Important to relate the shares to a
                        umNfsAcl.setFileSystemId(filesystem.getId());
                        // FileSystem.
                        if (umNfsAcl.getPermissions().isEmpty()) {
                            continue;
                        }
                        createNFSACL(umNfsAcl, fsNfsShareAcls, filesystem);
                        // Step 4: Update the UnManaged Share ACL : Set Inactive as true
                        umNfsAcl.setInactive(true);
                        // Step 5 : Keep this list as updated.
                        inActiveUnManagedShareNfs.add(umNfsAcl);
                    }
                }
            }
            // Set quota
            if (null != unManagedFileSystem.getExtensions() && null != unManagedFileSystem.getExtensions().get(QUOTA)) {
                if (null == filesystem.getExtensions()) {
                    filesystem.setExtensions(new StringMap());
                }
                filesystem.getExtensions().put(QUOTA, unManagedFileSystem.getExtensions().get(QUOTA));
            }
            filesystems.add(PropertySetterUtil.addFileSystemDetails(unManagedFileSystemInformation, filesystem));
            // Process Export Rules for the validated FS.
            filesystemList.getFilesystems().add(toNamedRelatedResource(ResourceTypeEnum.FILE, filesystem.getId(), filesystem.getNativeGuid()));
            unManagedFileSystem.setInactive(true);
            unManagedFileSystems.add(unManagedFileSystem);
            unManagedFSURIToFSMap.put(unManagedFileSystemUri, filesystem);
        }
        int i = 0;
        // Test
        for (FileShare fs : filesystems) {
            ++i;
            _logger.info("{} --> Saving FS to DB {}", i, fs);
            _logger.info(" --> Fs  Storage Pool {} and Virtual Pool {}", fs.getPool(), fs.getVirtualPool());
        }
        _dbClient.createObject(filesystems);
        for (URI unManagedFSURI : param.getUnManagedFileSystems()) {
            FileShare fs = unManagedFSURIToFSMap.get(unManagedFSURI);
            if (fs != null) {
                _logger.debug("ingesting quota directories for filesystem {}", fs.getId());
                ingestFileQuotaDirectories(fs);
            }
        }
        i = 0;
        // Test
        for (FileExportRule rule : fsExportRules) {
            ++i;
            _logger.info("{} --> Saving Export rule to DB {}", i, rule);
        }
        // Step 6.1 : Update the same in DB & Add new export rules
        _dbClient.createObject(fsExportRules);
        // Step 6.2 : Update Cifs Acls in DB & Add new ACLs
        i = 0;
        for (CifsShareACL acl : fsCifsShareAcls) {
            ++i;
            _logger.info("{} --> Saving New Cifs ACL to DB {}", i, acl);
        }
        if (fsCifsShareAcls != null && !fsCifsShareAcls.isEmpty()) {
            _dbClient.createObject(fsCifsShareAcls);
        }
        // Step 7.1 : Update the same in DB & clean ingested UnManagedCifsACLs
        i = 0;
        for (UnManagedCifsShareACL acl : inActiveUnManagedShareCifs) {
            ++i;
            _logger.info("{} Updating UnManagedACL DB as InActive TRUE {}", acl);
        }
        _dbClient.updateObject(inActiveUnManagedShareCifs);
        // Step 7.2 : Update the same in DB & clean Unmanaged ExportRule
        i = 0;
        for (UnManagedFileExportRule rule : inActiveUnManagedExportRules) {
            ++i;
            _logger.info("{} Updating DB as InActive TRUE {}", rule);
        }
        _dbClient.updateObject(inActiveUnManagedExportRules);
        _dbClient.updateObject(unManagedFileSystems);
        // Step 8.1 : Update NFS Acls in DB & Add new ACLs
        if (fsNfsShareAcls != null && !fsNfsShareAcls.isEmpty()) {
            _logger.info("Saving {} NFS ACLs to DB", fsNfsShareAcls.size());
            _dbClient.createObject(fsNfsShareAcls);
        }
        // UnManagedNFSShareACLs
        if (inActiveUnManagedShareNfs != null && !inActiveUnManagedShareNfs.isEmpty()) {
            _logger.info("Saving {} UnManagedNFS ACLs to DB", inActiveUnManagedShareNfs.size());
            _dbClient.updateObject(inActiveUnManagedShareNfs);
        }
        // record the events after they have been created
        for (FileShare filesystem : filesystems) {
            recordFileSystemOperation(_dbClient, OperationTypeEnum.INGEST_FILE_SYSTEM, Status.ready, filesystem.getId());
        }
    } catch (InternalException e) {
        throw e;
    } catch (Exception e) {
        _logger.error("Unexpected exception:", e);
        throw APIException.internalServerErrors.genericApisvcError(e.getMessage(), e);
    }
    return filesystemList;
}