Examples with VNXFileSshApi com.emc.storageos.vnx.xmlapi.VNXFileSshApi used on opensource projects

Example 6 with VNXFileSshApi

use of com.emc.storageos.vnx.xmlapi.VNXFileSshApi in project coprhd-controller by CoprHD.

the class VNXFileCommunicationInterface method discoverUnmanagedNewExports.

private void discoverUnmanagedNewExports(AccessProfile profile) {
    // Get Storage System
    URI storageSystemId = profile.getSystemId();
    StorageSystem storageSystem = _dbClient.queryObject(StorageSystem.class, storageSystemId);
    if (null == storageSystem) {
        return;
    }
    String detailedStatusMessage = "Discovery of VNX Unmanaged Exports started";
    _logger.info(detailedStatusMessage);
    // Used to Save the rules to DB
    List<UnManagedFileExportRule> newUnManagedExportRules = new ArrayList<UnManagedFileExportRule>();
    List<UnManagedFileExportRule> oldUnManagedExportRules = new ArrayList<UnManagedFileExportRule>();
    try {
        // Verification Utility
        UnManagedExportVerificationUtility validationUtility = new UnManagedExportVerificationUtility(_dbClient);
        // Discover port groups (data mover ids) and group names (data mover names)
        Set<StorageHADomain> activeDataMovers = discoverActiveDataMovers(storageSystem);
        // Reused from discoverAll
        // Discover ports (data mover interfaces) with the data movers in the active set.
        Map<String, List<StoragePort>> ports = discoverPorts(storageSystem, activeDataMovers);
        _logger.info("No of newly discovered port {}", ports.get(NEW).size());
        _logger.info("No of existing discovered port {}", ports.get(EXISTING).size());
        if (!ports.get(NEW).isEmpty()) {
            _dbClient.createObject(ports.get(NEW));
        }
        List<StoragePort> allPortsList = ports.get(NEW);
        allPortsList.addAll(ports.get(EXISTING));
        Map<String, List<StoragePort>> allPorts = new ConcurrentHashMap<String, List<StoragePort>>();
        for (StoragePort sPort : allPortsList) {
            _logger.debug("DM Storage Port  {}  StorageHADomain {}", sPort.getPortNetworkId(), sPort.getStorageHADomain());
            List<StoragePort> spList = allPorts.get(sPort.getStorageHADomain().toString());
            if (spList == null) {
                spList = new ArrayList<>();
            }
            spList.add(sPort);
            allPorts.put(sPort.getStorageHADomain().toString(), spList);
        }
        Map<String, List<StorageHADomain>> allVdms = discoverVdmPortGroups(storageSystem, activeDataMovers);
        if (!allVdms.get(NEW).isEmpty()) {
            _dbClient.createObject(allVdms.get(NEW));
        }
        Set<StorageHADomain> allActiveVDMs = new HashSet();
        allActiveVDMs.addAll(allVdms.get(NEW));
        allActiveVDMs.addAll(allVdms.get(EXISTING));
        activeDataMovers.addAll(allVdms.get(NEW));
        activeDataMovers.addAll(allVdms.get(EXISTING));
        Map<String, List<StoragePort>> allVdmPorts = discoverVdmPorts(storageSystem, allActiveVDMs);
        if (!allVdmPorts.get(NEW).isEmpty()) {
            _dbClient.createObject(allVdmPorts.get(NEW));
        }
        List<StoragePort> allVDMPortsList = allVdmPorts.get(NEW);
        allVDMPortsList.addAll(allVdmPorts.get(EXISTING));
        for (StoragePort sPort : allVDMPortsList) {
            List<StoragePort> spList = allPorts.get(sPort.getStorageHADomain().toString());
            _logger.debug("VDM Storage Port  {}  StorageHADomain {}", sPort.getPortNetworkId(), sPort.getStorageHADomain());
            if (spList == null) {
                spList = new ArrayList<>();
            }
            spList.add(sPort);
            allPorts.put(sPort.getStorageHADomain().toString(), spList);
        }
        List<UnManagedFileSystem> unManagedExportBatch = new ArrayList<>();
        for (StorageHADomain mover : activeDataMovers) {
            // Get storage port and name for the DM
            if (allPorts.get(mover.getId().toString()) == null || allPorts.get(mover.getId().toString()).isEmpty()) {
                // Did not find a single storage port for this DM, ignore it
                _logger.debug("No Ports found for {} {}", mover.getName(), mover.getAdapterName());
                continue;
            } else {
                _logger.debug("Number of  Ports found for {} : {} ", mover.getName() + ":" + mover.getAdapterName(), allPorts.get(mover.getId().toString()).size());
            }
            Collections.shuffle(allPorts.get(mover.getId().toString()));
            StoragePort storagePort = allPorts.get(mover.getId().toString()).get(0);
            if (storagePort == null) {
                // Did not find a single storage port for this DM, ignore it
                _logger.debug("StoragePort is null");
                continue;
            }
            // storagePort.setStorageHADomain(mover.getId());
            // get vnas uri
            URI moverURI = getNASUri(mover, storageSystem);
            // Retrieve FS-mountpath map for the Data Mover.
            _logger.info("Retrieving FS-mountpath map for Data Mover {}.", mover.getAdapterName());
            VNXFileSshApi sshDmApi = new VNXFileSshApi();
            sshDmApi.setConnParams(storageSystem.getIpAddress(), storageSystem.getUsername(), storageSystem.getPassword());
            Map<String, String> fileSystemMountpathMap = sshDmApi.getFsMountpathMap(mover.getAdapterName());
            Map<String, Map<String, String>> moverExportDetails = sshDmApi.getNFSExportsForPath(mover.getAdapterName());
            Map<String, String> nameIdMap = getFsNameFsNativeIdMap(storageSystem);
            // Loop through the map and, if the file exists in DB, retrieve the
            // export, process export, and associate export with the FS
            Set<String> fsNames = fileSystemMountpathMap.keySet();
            for (String fsName : fsNames) {
                // Retrieve FS from DB. If FS found, retrieve export and process
                String fsMountPath = fileSystemMountpathMap.get(fsName);
                // Get FS ID for nativeGUID
                // VNXFileSystem vnxFileSystems = discoverNamedFileSystem(storageSystem, fsName);
                String fsId = nameIdMap.get(fsName);
                _logger.debug("Resolved FileSystem name {} to native Id {}", fsName, fsId);
                UnManagedFileSystem vnxufs = null;
                if (fsId != null) {
                    String fsNativeGuid = NativeGUIDGenerator.generateNativeGuidForPreExistingFileSystem(storageSystem.getSystemType(), storageSystem.getSerialNumber().toUpperCase(), fsId);
                    vnxufs = checkUnManagedFileSystemExistsInDB(fsNativeGuid);
                }
                if (vnxufs != null) {
                    // Get export info
                    int noOfExports = 0;
                    boolean inValidExports = false;
                    for (String expPath : moverExportDetails.keySet()) {
                        if (!expPath.contains(fsMountPath)) {
                            // Ingore this path as it is not among the exports
                            continue;
                        } else {
                            // We should process only FS and its sub-directory exports only.
                            String subDir = expPath.substring(fsMountPath.length());
                            if (!subDir.isEmpty() && !subDir.startsWith("/")) {
                                continue;
                            }
                            _logger.info("Path : {} ", expPath);
                            noOfExports++;
                        }
                        // Used as for rules validation
                        List<UnManagedFileExportRule> unManagedExportRules = new ArrayList<UnManagedFileExportRule>();
                        Map<String, String> fsExportInfo = moverExportDetails.get(expPath);
                        if ((fsExportInfo != null) && (fsExportInfo.size() > 0)) {
                            // If multiple security flavors, do not add to ViPR DB
                            String securityFlavors = fsExportInfo.get(VNXFileConstants.SECURITY_TYPE);
                            if (securityFlavors == null || securityFlavors.length() == 0) {
                                securityFlavors = "sys";
                            }
                            if (securityFlavors != null) {
                                String fsMountPoint = storagePort.getPortNetworkId() + ":" + expPath;
                                _logger.info("Associating FS export map for VNX UMFS {}", vnxufs.getLabel());
                                associateExportWithFS(vnxufs, expPath, fsExportInfo, expPath, storagePort);
                                _logger.debug("Export map for VNX UMFS {} = {}", vnxufs.getLabel(), vnxufs.getFsUnManagedExportMap());
                                List<UnManagedFileExportRule> exportRules = applyAllSecurityRules(vnxufs.getId(), expPath, fsMountPoint, securityFlavors, fsExportInfo);
                                _logger.info("Number of export rules discovered for file system {} is {}", vnxufs.getId() + ":" + vnxufs.getLabel(), exportRules.size());
                                for (UnManagedFileExportRule dbExportRule : exportRules) {
                                    _logger.info("Unmanaged File Export Rule : {}", dbExportRule);
                                    String fsExportRulenativeId = dbExportRule.getFsExportIndex();
                                    _logger.info("Native Id using to build Native Guid {}", fsExportRulenativeId);
                                    String fsUnManagedFileExportRuleNativeGuid = NativeGUIDGenerator.generateNativeGuidForPreExistingFileExportRule(storageSystem, fsExportRulenativeId);
                                    _logger.info("Native GUID {}", fsUnManagedFileExportRuleNativeGuid);
                                    dbExportRule.setNativeGuid(fsUnManagedFileExportRuleNativeGuid);
                                    dbExportRule.setFileSystemId(vnxufs.getId());
                                    dbExportRule.setId(URIUtil.createId(UnManagedFileExportRule.class));
                                    // Build all export rules list.
                                    unManagedExportRules.add(dbExportRule);
                                }
                                // apply as per API SVC Validations.
                                if (!unManagedExportRules.isEmpty()) {
                                    boolean isAllRulesValid = validationUtility.validateUnManagedExportRules(unManagedExportRules, false);
                                    if (isAllRulesValid) {
                                        _logger.info("Validating rules success for export {}", expPath);
                                        for (UnManagedFileExportRule exportRule : unManagedExportRules) {
                                            UnManagedFileExportRule existingRule = checkUnManagedFsExportRuleExistsInDB(_dbClient, exportRule.getNativeGuid());
                                            if (existingRule == null) {
                                                newUnManagedExportRules.add(exportRule);
                                            } else {
                                                // Remove the existing rule.
                                                existingRule.setInactive(true);
                                                _dbClient.persistObject(existingRule);
                                                newUnManagedExportRules.add(exportRule);
                                            }
                                        }
                                        vnxufs.setHasExports(true);
                                        vnxufs.putFileSystemCharacterstics(UnManagedFileSystem.SupportedFileSystemCharacterstics.IS_FILESYSTEM_EXPORTED.toString(), TRUE);
                                        // Set the correct storage port
                                        if (null != storagePort) {
                                            StringSet storagePorts = new StringSet();
                                            storagePorts.add(storagePort.getId().toString());
                                            vnxufs.getFileSystemInformation().put(UnManagedFileSystem.SupportedFileSystemInformation.STORAGE_PORT.toString(), storagePorts);
                                        }
                                        _dbClient.persistObject(vnxufs);
                                        _logger.info("File System {} has Exports and their size is {}", vnxufs.getId(), newUnManagedExportRules.size());
                                    } else {
                                        _logger.warn("Validating rules failed for export {}. Ignoring to import these rules into ViPR DB", vnxufs);
                                        inValidExports = true;
                                    }
                                } else {
                                    _logger.warn("Export discovery failed for  {}. Ignoring to import these rules into ViPR DB", vnxufs);
                                    inValidExports = true;
                                }
                                // Adding this additional logic to avoid OOM
                                if (newUnManagedExportRules.size() == MAX_UMFS_RECORD_SIZE) {
                                    _logger.info("Saving Number of New UnManagedFileExportRule(s) {}", newUnManagedExportRules.size());
                                    _dbClient.createObject(newUnManagedExportRules);
                                    newUnManagedExportRules.clear();
                                }
                                // Adding this additional logic to avoid OOM
                                if (oldUnManagedExportRules.size() == MAX_UMFS_RECORD_SIZE) {
                                    _logger.info("Saving Number of Existing UnManagedFileExportRule(s) {}", oldUnManagedExportRules.size());
                                    _dbClient.persistObject(oldUnManagedExportRules);
                                    oldUnManagedExportRules.clear();
                                }
                            }
                        }
                    }
                    _logger.info("No of exports found for path {} = {} ", fsMountPath, noOfExports);
                    if (noOfExports == 0) {
                        _logger.info("FileSystem {} does not have any exports ", vnxufs.getLabel());
                        vnxufs.setHasExports(false);
                    }
                    // Don't consider the unmanaged file systems with invalid exports!!!
                    if (inValidExports) {
                        _logger.info("Ignoring unmanaged file system {}, due to invalid exports", vnxufs.getLabel());
                        vnxufs.setInactive(true);
                    }
                    // set the vNAS uri in umfs
                    StringSet moverSet = new StringSet();
                    moverSet.add(moverURI.toString());
                    vnxufs.putFileSystemInfo(UnManagedFileSystem.SupportedFileSystemInformation.NAS.toString(), moverSet);
                    _logger.info("nas server id {} and fs name {}", mover.getName(), fsName);
                    unManagedExportBatch.add(vnxufs);
                    if (unManagedExportBatch.size() >= VNXFileConstants.VNX_FILE_BATCH_SIZE) {
                        // Add UnManagedFileSystem batch
                        // Update UnManagedFilesystem
                        _dbClient.persistObject(unManagedExportBatch);
                        unManagedExportBatch.clear();
                    }
                }
            }
        }
        if (!unManagedExportBatch.isEmpty()) {
            // Update UnManagedFilesystem
            _dbClient.persistObject(unManagedExportBatch);
            unManagedExportBatch.clear();
        }
        if (!newUnManagedExportRules.isEmpty()) {
            // create new UnManagedExportFules
            _logger.info("Saving Number of New UnManagedFileExportRule(s) {}", newUnManagedExportRules.size());
            _dbClient.createObject(newUnManagedExportRules);
            newUnManagedExportRules.clear();
        }
        if (!oldUnManagedExportRules.isEmpty()) {
            // Update exisiting UnManagedExportFules
            _logger.info("Saving Number of Old UnManagedFileExportRule(s) {}", oldUnManagedExportRules.size());
            _dbClient.persistObject(oldUnManagedExportRules);
            oldUnManagedExportRules.clear();
        }
        // discovery succeeds
        storageSystem.setDiscoveryStatus(DiscoveredDataObject.DataCollectionJobStatus.COMPLETE.toString());
        detailedStatusMessage = String.format("Discovery completed successfully for VNXFile export: %s", storageSystemId.toString());
    } catch (Exception ex) {
        if (storageSystem != null) {
            cleanupDiscovery(storageSystem);
        }
        storageSystem.setDiscoveryStatus(DiscoveredDataObject.DataCollectionJobStatus.ERROR.toString());
        detailedStatusMessage = String.format("Discovery failed for VNXFile exports %s because %s", storageSystemId.toString(), ex.getLocalizedMessage());
        _logger.error(detailedStatusMessage, ex);
    } finally {
        if (storageSystem != null) {
            try {
                // set detailed message
                storageSystem.setLastDiscoveryStatusMessage(detailedStatusMessage);
                _dbClient.persistObject(storageSystem);
            } catch (Exception ex) {
                _logger.error("Error while persisting object to DB", ex);
            }
        }
    }
}

Example 7 with VNXFileSshApi

use of com.emc.storageos.vnx.xmlapi.VNXFileSshApi in project coprhd-controller by CoprHD.

the class VNXFileCommunicationInterface method discoverAll.

public void discoverAll(AccessProfile accessProfile) throws BaseCollectionException {
    URI storageSystemId = null;
    StorageSystem storageSystem = null;
    String detailedStatusMessage = "Unknown Status";
    try {
        _logger.info("Access Profile Details :  IpAddress : {}, PortNumber : {}", accessProfile.getIpAddress(), accessProfile.getPortNumber());
        storageSystemId = accessProfile.getSystemId();
        storageSystem = _dbClient.queryObject(StorageSystem.class, storageSystemId);
        // Retrieve control station information.
        discoverControlStation(storageSystem);
        // Model number
        VNXFileSshApi sshDmApi = new VNXFileSshApi();
        sshDmApi.setConnParams(storageSystem.getIpAddress(), storageSystem.getUsername(), storageSystem.getPassword());
        String model = sshDmApi.getModelInfo();
        storageSystem.setModel(model);
        boolean connectionStatus = getVnxFileSMISConnection(accessProfile, storageSystem);
        if (connectionStatus) {
            storageSystem.setSmisConnectionStatus(ConnectionStatus.CONNECTED.toString());
        } else {
            storageSystem.setSmisConnectionStatus(ConnectionStatus.NOTCONNECTED.toString());
        }
        _dbClient.persistObject(storageSystem);
        if (!storageSystem.getReachableStatus()) {
            throw new VNXFileCollectionException("Failed to connect to " + storageSystem.getIpAddress());
        }
        // Get All Existing DataMovers
        Map<String, StorageHADomain> allExistingDataMovers = getAllDataMovers(storageSystem);
        for (StorageHADomain activeDM : allExistingDataMovers.values()) {
            _logger.info("Existing DataMovers in database {}", activeDM.getName());
        }
        // Discover port groups (data movers)
        StringSet fileSharingProtocols = new StringSet();
        Map<String, List<StorageHADomain>> groups = discoverPortGroups(storageSystem, fileSharingProtocols);
        _logger.info("No of newly discovered groups {}", groups.get(NEW).size());
        _logger.info("No of existing discovered groups {}", groups.get(EXISTING).size());
        if (!groups.get(NEW).isEmpty()) {
            _dbClient.createObject(groups.get(NEW));
            for (StorageHADomain newDm : groups.get(NEW)) {
                _logger.info("New DM {} ", newDm.getAdapterName());
            }
        }
        if (!groups.get(EXISTING).isEmpty()) {
            _dbClient.persistObject(groups.get(EXISTING));
            for (StorageHADomain existingDm : groups.get(EXISTING)) {
                _logger.info("Existing DM {} ", existingDm.getAdapterName());
            }
        }
        // Discover storage pools.
        List<StoragePool> poolsToMatchWithVpool = new ArrayList<StoragePool>();
        List<StoragePool> allPools = new ArrayList<StoragePool>();
        Map<String, List<StoragePool>> pools = discoverStoragePools(storageSystem, poolsToMatchWithVpool, fileSharingProtocols);
        _logger.info("No of newly discovered pools {}", pools.get(NEW).size());
        _logger.info("No of existing discovered pools {}", pools.get(EXISTING).size());
        if (!pools.get(NEW).isEmpty()) {
            allPools.addAll(pools.get(NEW));
            _dbClient.createObject(pools.get(NEW));
        }
        if (!pools.get(EXISTING).isEmpty()) {
            allPools.addAll(pools.get(EXISTING));
            _dbClient.persistObject(pools.get(EXISTING));
        }
        List<StoragePool> notVisiblePools = DiscoveryUtils.checkStoragePoolsNotVisible(allPools, _dbClient, storageSystemId);
        if (notVisiblePools != null && !notVisiblePools.isEmpty()) {
            poolsToMatchWithVpool.addAll(notVisiblePools);
        }
        // Keep a set of active data movers. Data movers in 'standby' state are not added to the
        // database since they cannot be used in this state.
        Set<StorageHADomain> activeDataMovers = new HashSet<StorageHADomain>();
        activeDataMovers.addAll(groups.get(NEW));
        activeDataMovers.addAll(groups.get(EXISTING));
        int i = 0;
        for (StorageHADomain activeDM : activeDataMovers) {
            _logger.info("DataMover {} : {}", i++, activeDM.getName());
        }
        // Discover ports (data mover interfaces) with the data movers in the active set.
        Map<String, List<StoragePort>> ports = discoverPorts(storageSystem, activeDataMovers);
        _logger.info("No of newly discovered port {}", ports.get(NEW).size());
        _logger.info("No of existing discovered port {}", ports.get(EXISTING).size());
        if (!ports.get(NEW).isEmpty()) {
            _dbClient.createObject(ports.get(NEW));
        }
        if (!ports.get(EXISTING).isEmpty()) {
            _dbClient.persistObject(ports.get(EXISTING));
        }
        // Discover VDM and Ports
        Map<String, StorageHADomain> allVdmsInDb = this.getAllVDMs(storageSystem);
        for (StorageHADomain activeVDM : allVdmsInDb.values()) {
            _logger.info("Existing DataMovers in the Database {}", activeVDM.getName());
        }
        Map<String, List<StorageHADomain>> vdms = discoverVdmPortGroups(storageSystem, activeDataMovers);
        _logger.info("No of newly Vdm discovered groups {}", vdms.get(NEW).size());
        _logger.info("No of existing vdm discovered groups {}", vdms.get(EXISTING).size());
        if (!vdms.get(NEW).isEmpty()) {
            _dbClient.createObject(vdms.get(NEW));
        }
        if (!vdms.get(EXISTING).isEmpty()) {
            _dbClient.persistObject(vdms.get(EXISTING));
            for (StorageHADomain existingVdm : vdms.get(EXISTING)) {
                _logger.info("Existing VDM {}", existingVdm.getAdapterName());
            }
        }
        // Keep a set of active data movers. Data movers in 'standby' state are not added to the
        // database since they cannot be used in this state.
        Set<StorageHADomain> activeVDMs = new HashSet<StorageHADomain>();
        List<StorageHADomain> newVdms = vdms.get(NEW);
        for (StorageHADomain vdm : newVdms) {
            _logger.debug("New VDM : {}", vdm.getName());
            activeVDMs.add(vdm);
        }
        List<StorageHADomain> existingVdms = vdms.get(EXISTING);
        for (StorageHADomain vdm : existingVdms) {
            _logger.debug("Existing VDM : {}", vdm.getName());
            activeVDMs.add(vdm);
        }
        // Discover VDM Interfaces
        // Discover ports (data mover interfaces) with the data movers in the active set.
        Map<String, List<StoragePort>> vdmPorts = discoverVdmPorts(storageSystem, activeVDMs);
        _logger.info("No of newly discovered port {}", vdmPorts.get(NEW).size());
        _logger.info("No of existing discovered port {}", vdmPorts.get(EXISTING).size());
        if (!vdmPorts.get(NEW).isEmpty()) {
            _dbClient.createObject(vdmPorts.get(NEW));
            for (StoragePort port : vdmPorts.get(NEW)) {
                _logger.debug("New VDM Port : {}", port.getPortName());
            }
        }
        if (!vdmPorts.get(EXISTING).isEmpty()) {
            _dbClient.persistObject(vdmPorts.get(EXISTING));
            for (StoragePort port : vdmPorts.get(EXISTING)) {
                _logger.info("EXISTING VDM Port : {}", port.getPortName());
            }
        }
        List<StoragePort> allExistingPorts = new ArrayList<StoragePort>(ports.get(EXISTING));
        allExistingPorts.addAll(vdmPorts.get(EXISTING));
        List<StoragePort> allNewPorts = new ArrayList<StoragePort>(ports.get(NEW));
        allNewPorts.addAll(vdmPorts.get(NEW));
        List<StoragePort> allPorts = new ArrayList<StoragePort>(allExistingPorts);
        allPorts.addAll(allNewPorts);
        List<StoragePort> notVisiblePorts = DiscoveryUtils.checkStoragePortsNotVisible(allPorts, _dbClient, storageSystemId);
        allExistingPorts.addAll(notVisiblePorts);
        StoragePortAssociationHelper.updatePortAssociations(allNewPorts, _dbClient);
        StoragePortAssociationHelper.updatePortAssociations(allExistingPorts, _dbClient);
        StringBuffer errorMessage = new StringBuffer();
        ImplicitPoolMatcher.matchModifiedStoragePoolsWithAllVpool(poolsToMatchWithVpool, _dbClient, _coordinator, storageSystemId, errorMessage);
        // Update the virtual nas association with virtual arrays!!!
        // For existing virtual nas ports!!
        StoragePortAssociationHelper.runUpdateVirtualNasAssociationsProcess(allExistingPorts, null, _dbClient);
        // discovery succeeds
        detailedStatusMessage = String.format("Discovery completed successfully for Storage System: %s", storageSystemId.toString());
    } catch (Exception e) {
        if (storageSystem != null) {
            cleanupDiscovery(storageSystem);
        }
        detailedStatusMessage = String.format("Discovery failed for Storage System: %s because %s", storageSystemId.toString(), e.getLocalizedMessage());
        _logger.error(detailedStatusMessage, e);
        throw new VNXFileCollectionException(detailedStatusMessage);
    } finally {
        if (storageSystem != null) {
            try {
                // set detailed message
                storageSystem.setLastDiscoveryStatusMessage(detailedStatusMessage);
                _dbClient.persistObject(storageSystem);
            } catch (DatabaseException ex) {
                _logger.error("Error while persisting object to DB", ex);
            }
        }
    }
}