use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedFileSystem in project coprhd-controller by CoprHD.
the class NetAppFileCommunicationInterface method discoverUmanagedFileSystems.
private void discoverUmanagedFileSystems(AccessProfile profile) {
URI storageSystemId = profile.getSystemId();
StorageSystem storageSystem = _dbClient.queryObject(StorageSystem.class, storageSystemId);
if (null == storageSystem) {
return;
}
String detailedStatusMessage = "Discovery of NetApp Unmanaged FileSystem started";
List<UnManagedFileSystem> unManagedFileSystems = new ArrayList<UnManagedFileSystem>();
List<UnManagedFileSystem> existingUnManagedFileSystems = new ArrayList<UnManagedFileSystem>();
int newFileSystemsCount = 0;
int existingFileSystemsCount = 0;
Set<URI> allDiscoveredUnManagedFileSystems = new HashSet<URI>();
NetAppApi netAppApi = new NetAppApi.Builder(storageSystem.getIpAddress(), storageSystem.getPortNumber(), storageSystem.getUsername(), storageSystem.getPassword()).https(true).build();
Collection<String> attrs = new ArrayList<String>();
for (String property : ntpPropertiesList) {
attrs.add(SupportedNtpFileSystemInformation.getFileSystemInformation(property));
}
try {
StoragePort storagePort = getStoragePortPool(storageSystem);
URIQueryResultList storagePoolURIs = new URIQueryResultList();
_dbClient.queryByConstraint(ContainmentConstraint.Factory.getStorageDeviceStoragePoolConstraint(storageSystem.getId()), storagePoolURIs);
HashMap<String, StoragePool> pools = new HashMap();
Iterator<URI> poolsItr = storagePoolURIs.iterator();
while (poolsItr.hasNext()) {
URI storagePoolURI = poolsItr.next();
StoragePool storagePool = _dbClient.queryObject(StoragePool.class, storagePoolURI);
pools.put(storagePool.getNativeGuid(), storagePool);
}
// Retrieve all the file system and vFiler info.
List<Map<String, String>> fileSystemInfo = netAppApi.listVolumeInfo(null, attrs);
List<VFilerInfo> vFilers = netAppApi.listVFilers(null);
for (Map<String, String> fileSystemChar : fileSystemInfo) {
String poolName = fileSystemChar.get(SupportedNtpFileSystemInformation.getFileSystemInformation(SupportedNtpFileSystemInformation.STORAGE_POOL.toString()));
String filesystem = fileSystemChar.get(SupportedNtpFileSystemInformation.getFileSystemInformation(SupportedNtpFileSystemInformation.NAME.toString()));
String poolNativeGuid = NativeGUIDGenerator.generateNativeGuid(storageSystem, poolName, NativeGUIDGenerator.POOL);
StoragePool pool = pools.get(poolNativeGuid);
String nativeId;
if (filesystem.startsWith(VOL_ROOT)) {
nativeId = filesystem;
} else {
nativeId = VOL_ROOT + filesystem;
}
String fsNativeGuid = NativeGUIDGenerator.generateNativeGuid(storageSystem.getSystemType(), storageSystem.getSerialNumber(), nativeId);
// Ignore export for root volume and don't pull it into ViPR db.
if (nativeId.contains(ROOT_VOL)) {
_logger.info("Ignore and not discover root filesystem on NTP array");
continue;
}
// to create an UnManaged Filesystems.
if (checkStorageFileSystemExistsInDB(fsNativeGuid)) {
continue;
}
_logger.debug("retrieve info for file system: " + filesystem);
String vFiler = getOwningVfiler(filesystem, fileSystemInfo);
if (vFiler != null && !vFiler.equalsIgnoreCase(DEFAULT_FILER)) {
_logger.info("Ignoring {} because it is owned by {}", filesystem, vFiler);
continue;
}
String address = getVfilerAddress(vFiler, vFilers);
if (vFiler != null && !vFiler.isEmpty()) {
// Need to use storage port for vFiler.
String portNativeGuid = NativeGUIDGenerator.generateNativeGuid(storageSystem, address, NativeGUIDGenerator.PORT);
storagePort = getVfilerStoragePort(storageSystem, portNativeGuid, vFiler);
}
String fsUnManagedFsNativeGuid = NativeGUIDGenerator.generateNativeGuidForPreExistingFileSystem(storageSystem.getSystemType(), storageSystem.getSerialNumber().toUpperCase(), nativeId);
UnManagedFileSystem unManagedFs = checkUnManagedFileSystemExistsInDB(fsUnManagedFsNativeGuid);
boolean alreadyExist = unManagedFs == null ? false : true;
unManagedFs = createUnManagedFileSystem(unManagedFs, profile, fsUnManagedFsNativeGuid, nativeId, storageSystem, pool, filesystem, storagePort, fileSystemChar);
if (alreadyExist) {
existingUnManagedFileSystems.add(unManagedFs);
existingFileSystemsCount++;
} else {
unManagedFileSystems.add(unManagedFs);
newFileSystemsCount++;
}
allDiscoveredUnManagedFileSystems.add(unManagedFs.getId());
/**
* Persist 200 objects and clear them to avoid memory issue
*/
validateListSizeLimitAndPersist(unManagedFileSystems, existingUnManagedFileSystems, Constants.DEFAULT_PARTITION_SIZE * 2);
}
// Process those active unmanaged fs objects available in database but not in newly discovered items, to
// mark them inactive.
markUnManagedFSObjectsInActive(storageSystem, allDiscoveredUnManagedFileSystems);
_logger.info("New unmanaged Netapp file systems count: {}", newFileSystemsCount);
_logger.info("Update unmanaged Netapp file systems count: {}", existingFileSystemsCount);
if (!unManagedFileSystems.isEmpty()) {
// Add UnManagedFileSystem
_partitionManager.insertInBatches(unManagedFileSystems, Constants.DEFAULT_PARTITION_SIZE, _dbClient, UNMANAGED_FILESYSTEM);
}
if (!existingUnManagedFileSystems.isEmpty()) {
// Update UnManagedFilesystem
_partitionManager.updateAndReIndexInBatches(existingUnManagedFileSystems, Constants.DEFAULT_PARTITION_SIZE, _dbClient, UNMANAGED_FILESYSTEM);
}
// discovery succeeds
detailedStatusMessage = String.format("Discovery completed successfully for NetApp: %s", storageSystemId.toString());
} catch (NetAppException ve) {
if (null != storageSystem) {
cleanupDiscovery(storageSystem);
}
_logger.error("discoverStorage failed. Storage system: " + storageSystemId);
throw ve;
} catch (Exception e) {
if (null != storageSystem) {
cleanupDiscovery(storageSystem);
}
_logger.error("discoverStorage failed. Storage system: " + storageSystemId, e);
throw NetAppException.exceptions.discoveryFailed(storageSystemId.toString(), e);
} finally {
if (storageSystem != null) {
try {
// set detailed message
storageSystem.setLastDiscoveryStatusMessage(detailedStatusMessage);
_dbClient.persistObject(storageSystem);
} catch (Exception ex) {
_logger.error("Error while persisting object to DB", ex);
}
}
}
}
use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedFileSystem in project coprhd-controller by CoprHD.
the class NetAppFileCommunicationInterface method createUnManagedFileSystem.
/**
* create StorageFileSystem Info Object
*
* @param unManagedFileSystem
* @param unManagedFileSystemNativeGuid
* @param storageSystemUri
* @param storagePool
* @param fileSystem
* @param storagePort
* @param fileSystemChars
* @return UnManagedFileSystem
* @throws IOException
* @throws NetAppException
*/
private UnManagedFileSystem createUnManagedFileSystem(UnManagedFileSystem unManagedFileSystem, AccessProfile profile, String unManagedFileSystemNativeGuid, String unManangedFileSystemNativeId, StorageSystem system, StoragePool pool, String fileSystem, StoragePort storagePort, Map<String, String> fileSystemChars) throws IOException, NetAppException {
if (null == unManagedFileSystem) {
unManagedFileSystem = new UnManagedFileSystem();
unManagedFileSystem.setId(URIUtil.createId(UnManagedFileSystem.class));
unManagedFileSystem.setNativeGuid(unManagedFileSystemNativeGuid);
unManagedFileSystem.setStorageSystemUri(system.getId());
unManagedFileSystem.setHasExports(false);
unManagedFileSystem.setHasShares(false);
}
Map<String, StringSet> unManagedFileSystemInformation = new HashMap<String, StringSet>();
StringMap unManagedFileSystemCharacteristics = new StringMap();
// TODO: We are not able to extract this information from filesystem
// properties from netapp api from iwave.
// may need to go over all the snapshots/exports to determine if we have
// any associated filesystems.
unManagedFileSystemCharacteristics.put(SupportedFileSystemCharacterstics.IS_SNAP_SHOT.toString(), FALSE);
unManagedFileSystemCharacteristics.put(SupportedFileSystemCharacterstics.IS_THINLY_PROVISIONED.toString(), FALSE);
unManagedFileSystemCharacteristics.put(UnManagedFileSystem.SupportedFileSystemCharacterstics.IS_INGESTABLE.toString(), TRUE);
// On netapp Systems this currently true.
unManagedFileSystemCharacteristics.put(UnManagedFileSystem.SupportedFileSystemCharacterstics.IS_FILESYSTEM_EXPORTED.toString(), FALSE);
if (null != storagePort) {
StringSet storagePorts = new StringSet();
storagePorts.add(storagePort.getId().toString());
unManagedFileSystemInformation.put(SupportedFileSystemInformation.STORAGE_PORT.toString(), storagePorts);
}
if (null != pool) {
StringSet pools = new StringSet();
pools.add(pool.getId().toString());
unManagedFileSystemInformation.put(UnManagedFileSystem.SupportedFileSystemInformation.STORAGE_POOL.toString(), pools);
unManagedFileSystem.setStoragePoolUri(pool.getId());
StringSet matchedVPools = DiscoveryUtils.getMatchedVirtualPoolsForPool(_dbClient, pool.getId());
_logger.debug("Matched Pools : {}", Joiner.on("\t").join(matchedVPools));
if (null == matchedVPools || matchedVPools.isEmpty()) {
// clear all existing supported vpool list.
unManagedFileSystem.getSupportedVpoolUris().clear();
} else {
// replace with new StringSet
unManagedFileSystem.getSupportedVpoolUris().replace(matchedVPools);
_logger.info("Replaced Pools :" + Joiner.on("\t").join(unManagedFileSystem.getSupportedVpoolUris()));
}
}
if (null != system) {
StringSet systemTypes = new StringSet();
systemTypes.add(system.getSystemType());
unManagedFileSystemInformation.put(SupportedFileSystemInformation.SYSTEM_TYPE.toString(), systemTypes);
}
// Get FileSystem used Space.
if (null != fileSystemChars.get(SupportedNtpFileSystemInformation.getFileSystemInformation(SupportedNtpFileSystemInformation.ALLOCATED_CAPACITY.toString()))) {
StringSet allocatedCapacity = new StringSet();
allocatedCapacity.add(fileSystemChars.get(SupportedNtpFileSystemInformation.getFileSystemInformation(SupportedNtpFileSystemInformation.ALLOCATED_CAPACITY.toString())));
unManagedFileSystemInformation.put(SupportedFileSystemInformation.ALLOCATED_CAPACITY.toString(), allocatedCapacity);
}
// Get FileSystem used Space.
if (null != fileSystemChars.get(SupportedNtpFileSystemInformation.getFileSystemInformation(SupportedNtpFileSystemInformation.PROVISIONED_CAPACITY.toString()))) {
StringSet provisionedCapacity = new StringSet();
String totalCapacity = fileSystemChars.get(SupportedNtpFileSystemInformation.getFileSystemInformation(SupportedNtpFileSystemInformation.PROVISIONED_CAPACITY.toString()));
String snapShotReserveBlocks = fileSystemChars.get(SupportedNtpFileSystemInformation.getFileSystemInformation(SupportedNtpFileSystemInformation.SNAPSHOT_BLOCKS_RESERVED.toString()));
// Snapshot reserved Blocks - 1 block is 1024 bytes, convert it to bytes
String fsProvisionedCapacity = Long.toString(Long.parseLong(totalCapacity) + (Long.parseLong(snapShotReserveBlocks) * BYTESCONVERTER));
provisionedCapacity.add(fsProvisionedCapacity);
unManagedFileSystemInformation.put(SupportedFileSystemInformation.PROVISIONED_CAPACITY.toString(), provisionedCapacity);
}
// Save off FileSystem Name, Path, Mount and label information
if (null != fileSystemChars.get(SupportedNtpFileSystemInformation.getFileSystemInformation(SupportedNtpFileSystemInformation.NAME.toString()))) {
StringSet fsName = new StringSet();
String fileSystemName = fileSystemChars.get(SupportedNtpFileSystemInformation.getFileSystemInformation(SupportedNtpFileSystemInformation.NAME.toString()));
fsName.add(fileSystemName);
unManagedFileSystem.setLabel(fileSystemName);
StringSet fsPath = new StringSet();
fsPath.add(unManangedFileSystemNativeId);
StringSet fsMountPath = new StringSet();
fsMountPath.add(VOL_ROOT + fileSystem);
unManagedFileSystemInformation.put(SupportedFileSystemInformation.NAME.toString(), fsName);
unManagedFileSystemInformation.put(SupportedFileSystemInformation.NATIVE_ID.toString(), fsPath);
unManagedFileSystemInformation.put(SupportedFileSystemInformation.DEVICE_LABEL.toString(), fsName);
unManagedFileSystemInformation.put(SupportedFileSystemInformation.PATH.toString(), fsPath);
unManagedFileSystemInformation.put(SupportedFileSystemInformation.MOUNT_PATH.toString(), fsMountPath);
}
// Add fileSystemInformation and Characteristics.
unManagedFileSystem.addFileSystemInformation(unManagedFileSystemInformation);
unManagedFileSystem.setFileSystemCharacterstics(unManagedFileSystemCharacteristics);
return unManagedFileSystem;
}
use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedFileSystem in project coprhd-controller by CoprHD.
the class ExtendedCommunicationInterfaceImpl method markUnManagedFSObjectsInActive.
/**
* Synchronize the existing active DB Un-Managed FS objects with the newly discovered
* Un-Managed FS objects listed by the Array.
*/
protected void markUnManagedFSObjectsInActive(StorageSystem storageSystem, Set<URI> discoveredUnManagedFileSystems) {
_logger.info(" -- Processing {} discovered Un-Managed FS Objects from -- {}", discoveredUnManagedFileSystems.size(), storageSystem.getLabel());
if (discoveredUnManagedFileSystems.isEmpty()) {
return;
}
// Get all available existing unmanaged FS URIs for this array from DB
URIQueryResultList allAvailableUnManagedFileSystemsInDB = new URIQueryResultList();
_dbClient.queryByConstraint(ContainmentConstraint.Factory.getStorageDeviceUnManagedFileSystemConstraint(storageSystem.getId()), allAvailableUnManagedFileSystemsInDB);
List<UnManagedFileSystem> unManagedFileSystems = new ArrayList<UnManagedFileSystem>();
int totalObjs = 0;
while (allAvailableUnManagedFileSystemsInDB.iterator().hasNext()) {
URI unManagedFileSystemUri = allAvailableUnManagedFileSystemsInDB.iterator().next();
if (!discoveredUnManagedFileSystems.contains(unManagedFileSystemUri)) {
UnManagedFileSystem uFS = _dbClient.queryObject(UnManagedFileSystem.class, unManagedFileSystemUri);
_logger.info("Found a stale un-managed active file system in DB {} - Marking this to In-Active", uFS.getNativeGuid());
uFS.setInactive(true);
unManagedFileSystems.add(uFS);
if (unManagedFileSystems.size() == 1000) {
totalObjs += 1000;
_partitionManager.updateInBatches(unManagedFileSystems, Constants.DEFAULT_PARTITION_SIZE, _dbClient, UNMANAGED_FILESYSTEM);
unManagedFileSystems.clear();
}
}
}
totalObjs += unManagedFileSystems.size();
if (!unManagedFileSystems.isEmpty()) {
_partitionManager.updateInBatches(unManagedFileSystems, Constants.DEFAULT_PARTITION_SIZE, _dbClient, UNMANAGED_FILESYSTEM);
}
_logger.info("Total number of stale unmanaged file systems processed {}", totalObjs);
}
use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedFileSystem in project coprhd-controller by CoprHD.
the class DataDomainCommunicationInterface method discoverUnManagedCifsShares.
private void discoverUnManagedCifsShares(DataDomainClient ddClient, StorageSystem storageSystem) throws DataDomainApiException {
storageSystem.setDiscoveryStatus(DiscoveredDataObject.DataCollectionJobStatus.IN_PROGRESS.toString());
String detailedStatusMessage = "Discovery of Data Domain Unmanaged Cifs Shares started";
storageSystem.setLastDiscoveryStatusMessage(detailedStatusMessage);
// Used to cache UMFS once retrieved from DB
Map<String, UnManagedFileSystem> existingUnManagedFileSystems = new HashMap<String, UnManagedFileSystem>();
// Used to Save the CIFS ACLs to DB
List<UnManagedCifsShareACL> newUnManagedCifsACLs = new ArrayList<UnManagedCifsShareACL>();
List<UnManagedCifsShareACL> oldUnManagedCifsACLs = new ArrayList<UnManagedCifsShareACL>();
try {
// Get exports on the array and loop through each export.
DDShareList shareList = ddClient.getShares(storageSystem.getNativeGuid());
for (DDShareInfo shareInfo : shareList.getShares()) {
DDShareInfoDetail share = ddClient.getShare(storageSystem.getNativeGuid(), shareInfo.getId());
if (share.getPathStatus() != DataDomainApiConstants.PATH_EXISTS) {
continue;
}
String fsUnManagedFsNativeGuid = NativeGUIDGenerator.generateNativeGuidForPreExistingFileSystem(storageSystem.getSystemType(), storageSystem.getSerialNumber().toUpperCase(), share.getMtreeId());
// Get UMFS from cache if possible, otherwise try to retrieve from DB
UnManagedFileSystem unManagedFS = existingUnManagedFileSystems.get(fsUnManagedFsNativeGuid);
if (unManagedFS == null) {
unManagedFS = getUnManagedFileSystemFromDB(fsUnManagedFsNativeGuid);
}
if (unManagedFS != null) {
// Add UMFS to cache
existingUnManagedFileSystems.put(fsUnManagedFsNativeGuid, unManagedFS);
StringSet storagePortIds = unManagedFS.getFileSystemInformation().get(UnManagedFileSystem.SupportedFileSystemInformation.STORAGE_PORT.toString());
StoragePort storagePort = null;
for (String portId : storagePortIds) {
StoragePort sp = _dbClient.queryObject(StoragePort.class, URI.create(portId));
if (sp != null && !sp.getInactive()) {
storagePort = sp;
break;
}
}
associateCifsExportWithFS(unManagedFS, share, storagePort);
unManagedFS.setHasShares(true);
unManagedFS.putFileSystemCharacterstics(UnManagedFileSystem.SupportedFileSystemCharacterstics.IS_FILESYSTEM_EXPORTED.toString(), TRUE);
_log.debug("Export map for VNX UMFS {} = {}", unManagedFS.getLabel(), unManagedFS.getUnManagedSmbShareMap());
List<UnManagedCifsShareACL> cifsACLs = applyCifsSecurityRules(unManagedFS, share, storagePort);
_log.info("Number of export rules discovered for file system {} is {}", unManagedFS.getId() + ":" + unManagedFS.getLabel(), cifsACLs.size());
for (UnManagedCifsShareACL cifsAcl : cifsACLs) {
_log.info("Unmanaged File share acls : {}", cifsAcl);
String fsShareNativeId = cifsAcl.getFileSystemShareACLIndex();
_log.info("UMFS Share ACL index {}", fsShareNativeId);
String fsUnManagedFileShareNativeGuid = NativeGUIDGenerator.generateNativeGuidForPreExistingFileShare(storageSystem, fsShareNativeId);
_log.info("Native GUID {}", fsUnManagedFileShareNativeGuid);
cifsAcl.setNativeGuid(fsUnManagedFileShareNativeGuid);
// Check whether the CIFS share ACL was present in ViPR DB.
UnManagedCifsShareACL existingACL = checkUnManagedFsCifsACLExistsInDB(_dbClient, cifsAcl.getNativeGuid());
if (existingACL == null) {
newUnManagedCifsACLs.add(cifsAcl);
} else {
newUnManagedCifsACLs.add(cifsAcl);
existingACL.setInactive(true);
oldUnManagedCifsACLs.add(existingACL);
}
}
// Update the UnManaged file system
_dbClient.persistObject(unManagedFS);
}
if (newUnManagedCifsACLs.size() >= VNXFileConstants.VNX_FILE_BATCH_SIZE) {
// create new UnManagedCifsShareACL
_log.info("Saving Number of New UnManagedCifsShareACL(s) {}", newUnManagedCifsACLs.size());
_dbClient.createObject(newUnManagedCifsACLs);
newUnManagedCifsACLs.clear();
}
if (oldUnManagedCifsACLs.size() >= VNXFileConstants.VNX_FILE_BATCH_SIZE) {
// Update existing UnManagedCifsShareACL
_log.info("Saving Number of Old UnManagedCifsShareACL(s) {}", oldUnManagedCifsACLs.size());
_dbClient.persistObject(oldUnManagedCifsACLs);
oldUnManagedCifsACLs.clear();
}
}
if (!newUnManagedCifsACLs.isEmpty()) {
// create new UnManagedCifsShareACL
_log.info("Saving Number of New UnManagedCifsShareACL(s) {}", newUnManagedCifsACLs.size());
_dbClient.createObject(newUnManagedCifsACLs);
newUnManagedCifsACLs.clear();
}
if (!oldUnManagedCifsACLs.isEmpty()) {
// Update existing UnManagedCifsShareACL
_log.info("Saving Number of Old UnManagedCifsShareACL(s) {}", oldUnManagedCifsACLs.size());
_dbClient.persistObject(oldUnManagedCifsACLs);
oldUnManagedCifsACLs.clear();
}
storageSystem.setDiscoveryStatus(DiscoveredDataObject.DataCollectionJobStatus.COMPLETE.toString());
// discovery succeeded
detailedStatusMessage = String.format("Discovery completed successfully for Data Domain: %s", storageSystem.getId().toString());
storageSystem.setLastDiscoveryStatusMessage(detailedStatusMessage);
} catch (DataDomainApiException dde) {
_log.error("discoverStorage failed. Storage system: " + storageSystem.getId());
} catch (Exception e) {
_log.error("discoverStorage failed. Storage system: " + storageSystem.getId(), e);
} finally {
if (storageSystem != null) {
try {
_dbClient.persistObject(storageSystem);
} catch (Exception ex) {
_log.error("Error while persisting object to DB", ex);
}
}
}
}
use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedFileSystem in project coprhd-controller by CoprHD.
the class DataDomainCommunicationInterface method getUnManagedFileSystemFromDB.
/**
* Retrieve the FS from DB if it exists already
*
* @param nativeGuid
* @return unManageFileSystem
* @throws IOException
*/
protected UnManagedFileSystem getUnManagedFileSystemFromDB(String nativeGuid) {
UnManagedFileSystem filesystemInfo = null;
URIQueryResultList result = new URIQueryResultList();
_dbClient.queryByConstraint(AlternateIdConstraint.Factory.getFileSystemInfoNativeGUIdConstraint(nativeGuid), result);
List<URI> filesystemUris = new ArrayList<URI>();
Iterator<URI> iter = result.iterator();
while (iter.hasNext()) {
URI unFileSystemtURI = iter.next();
filesystemUris.add(unFileSystemtURI);
}
if (!filesystemUris.isEmpty()) {
filesystemInfo = _dbClient.queryObject(UnManagedFileSystem.class, filesystemUris.get(0));
}
return filesystemInfo;
}
Aggregations