use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedFileExportRule in project coprhd-controller by CoprHD.
the class VNXUnityUnManagedObjectDiscoverer method discoverAllExportRules.
public void discoverAllExportRules(AccessProfile accessProfile, DbClient dbClient, PartitionManager partitionManager) {
StorageSystem storageSystem = dbClient.queryObject(StorageSystem.class, accessProfile.getSystemId());
VNXeApiClient apiClient = getVnxUnityClient(accessProfile);
log.info("discoverAllExportRules for storage system {} - start", storageSystem.getId());
unManagedExportRulesInsert = new ArrayList<UnManagedFileExportRule>();
unManagedExportRulesUpdate = new ArrayList<UnManagedFileExportRule>();
unManagedFilesystemsUpdate = new ArrayList<UnManagedFileSystem>();
List<VNXeNfsShare> nfsExports = apiClient.getAllNfsShares();
// Verification Utility
UnManagedExportVerificationUtility validationUtility = new UnManagedExportVerificationUtility(dbClient);
for (VNXeNfsShare exp : nfsExports) {
log.info("Discovered fS export {}", exp.toString());
VNXeFileSystem fs = null;
if (exp.getFilesystem() != null) {
fs = apiClient.getFileSystemByFSId(exp.getFilesystem().getId());
String fsNativeGuid = NativeGUIDGenerator.generateNativeGuid(storageSystem.getSystemType(), storageSystem.getSerialNumber(), fs.getId());
try {
if (checkStorageFileSystemExistsInDB(fsNativeGuid, dbClient)) {
log.info("Skipping file system {} as it is already managed by ViPR", fsNativeGuid);
continue;
}
// Create UnManaged FS
String fsUnManagedFsNativeGuid = NativeGUIDGenerator.generateNativeGuidForPreExistingFileSystem(storageSystem.getSystemType(), storageSystem.getSerialNumber().toUpperCase(), fs.getId());
UnManagedFileSystem unManagedFs = checkUnManagedFileSystemExistsInDB(dbClient, fsUnManagedFsNativeGuid);
StoragePort storagePort = getStoragePortPool(storageSystem, dbClient, apiClient, fs);
String mountPath = extractValueFromStringSet(SupportedFileSystemInformation.MOUNT_PATH.toString(), unManagedFs.getFileSystemInformation());
String exportPath = exp.getPath();
if (!exportPath.equalsIgnoreCase("/")) {
mountPath = mountPath + exportPath;
}
String mountPoint = storagePort.getPortNetworkId() + ":" + mountPath;
String nfsShareId = exp.getId();
String fsUnManagedFileExportRuleNativeGuid = NativeGUIDGenerator.generateNativeGuidForPreExistingFileExportRule(storageSystem, nfsShareId);
log.info("Native GUID {}", fsUnManagedFileExportRuleNativeGuid);
UnManagedFileExportRule unManagedExportRule = checkUnManagedFsExportRuleExistsInDB(dbClient, fsUnManagedFileExportRuleNativeGuid);
UnManagedFileExportRule unManagedExpRule = null;
List<UnManagedFileExportRule> unManagedExportRules = new ArrayList<UnManagedFileExportRule>();
if (unManagedExportRule == null) {
unManagedExportRule = new UnManagedFileExportRule();
unManagedExportRule.setNativeGuid(fsUnManagedFileExportRuleNativeGuid);
unManagedExportRule.setFileSystemId(unManagedFs.getId());
unManagedExportRule.setId(URIUtil.createId(UnManagedFileExportRule.class));
unManagedExpRule = createExportRules(unManagedFs.getId(), apiClient, exp, unManagedExportRule, mountPath, mountPoint, nfsShareId, storagePort.getPortName());
unManagedExportRulesInsert.add(unManagedExpRule);
} else {
unManagedExpRule = createExportRules(unManagedFs.getId(), apiClient, exp, unManagedExportRule, mountPath, mountPoint, nfsShareId, storagePort.getPortName());
unManagedExportRulesUpdate.add(unManagedExpRule);
}
log.info("Unmanaged File Export Rule : {}", unManagedExportRule);
// Build all export rules list.
unManagedExportRules.add(unManagedExpRule);
// apply as per API SVC Validations.
if (!unManagedExportRules.isEmpty()) {
boolean isAllRulesValid = validationUtility.validateUnManagedExportRules(unManagedExportRules, false);
if (isAllRulesValid) {
log.info("Validating rules success for export {}", unManagedFs.getPath());
unManagedFs.setHasExports(true);
unManagedFs.putFileSystemCharacterstics(UnManagedFileSystem.SupportedFileSystemCharacterstics.IS_FILESYSTEM_EXPORTED.toString(), Boolean.TRUE.toString());
unManagedFilesystemsUpdate.add(unManagedFs);
log.info("File System {} has Exports and their size is {}", unManagedFs.getId(), unManagedExportRules.size());
} else {
log.warn("Validating rules failed for export {}. Ignroing to import these rules into ViPR DB", unManagedFs);
unManagedFs.setInactive(true);
unManagedFilesystemsUpdate.add(unManagedFs);
}
}
} catch (IOException e) {
log.error("IOException occured in discoverAllExportRules()", e);
}
}
}
if (!unManagedExportRulesInsert.isEmpty()) {
// Add UnManage export rules
partitionManager.insertInBatches(unManagedExportRulesInsert, Constants.DEFAULT_PARTITION_SIZE, dbClient, UNMANAGED_EXPORT_RULE);
}
if (!unManagedExportRulesUpdate.isEmpty()) {
// Update UnManage export rules
partitionManager.updateInBatches(unManagedExportRulesUpdate, Constants.DEFAULT_PARTITION_SIZE, dbClient, UNMANAGED_EXPORT_RULE);
}
if (!unManagedFilesystemsUpdate.isEmpty()) {
// Update UnManagedFilesystem
partitionManager.updateInBatches(unManagedFilesystemsUpdate, Constants.DEFAULT_PARTITION_SIZE, dbClient, UNMANAGED_FILESYSTEM);
}
}
use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedFileExportRule in project coprhd-controller by CoprHD.
the class VNXUnityUnManagedObjectDiscoverer method checkUnManagedFsExportRuleExistsInDB.
/**
* check Pre Existing Storage Export Rule exists in DB
*
* @param nativeGuid
* @return unManagedFileExportRule
* @throws IOException
*/
protected UnManagedFileExportRule checkUnManagedFsExportRuleExistsInDB(DbClient dbClient, String fsExportRuleNativeId) {
UnManagedFileExportRule unManagedExportRule = null;
URIQueryResultList result = new URIQueryResultList();
dbClient.queryByConstraint(AlternateIdConstraint.Factory.getFileExporRuleNativeGUIdConstraint(fsExportRuleNativeId), result);
Iterator<URI> iter = result.iterator();
while (iter.hasNext()) {
URI unExportRuleURI = iter.next();
unManagedExportRule = dbClient.queryObject(UnManagedFileExportRule.class, unExportRuleURI);
return unManagedExportRule;
}
return unManagedExportRule;
}
use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedFileExportRule in project coprhd-controller by CoprHD.
the class UnManagedExportVerificationUtility method scanForDuplicateSecFlavor.
private boolean scanForDuplicateSecFlavor(List<UnManagedFileExportRule> unManagedExportRules) {
_log.info("Validating Sec Flavor");
List<String> secFlavorsFound = new ArrayList<>();
for (UnManagedFileExportRule rule : unManagedExportRules) {
String secRuleToValidate = rule.getSecFlavor();
// MULTIPLE_EXPORTS_WITH_SAME_SEC_FLAVOR
if (!secFlavorsFound.contains(secRuleToValidate)) {
secFlavorsFound.add(rule.getSecFlavor());
_log.info("Secuity rules found as of now {}, size {}", secFlavorsFound, secFlavorsFound.size());
} else {
_log.warn("Duplicate SecFlavor found {}", secRuleToValidate);
return false;
}
}
return true;
}
use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedFileExportRule in project coprhd-controller by CoprHD.
the class VNXFileCommunicationInterface method discoverUnmanagedNewExports.
private void discoverUnmanagedNewExports(AccessProfile profile) {
// Get Storage System
URI storageSystemId = profile.getSystemId();
StorageSystem storageSystem = _dbClient.queryObject(StorageSystem.class, storageSystemId);
if (null == storageSystem) {
return;
}
String detailedStatusMessage = "Discovery of VNX Unmanaged Exports started";
_logger.info(detailedStatusMessage);
// Used to Save the rules to DB
List<UnManagedFileExportRule> newUnManagedExportRules = new ArrayList<UnManagedFileExportRule>();
List<UnManagedFileExportRule> oldUnManagedExportRules = new ArrayList<UnManagedFileExportRule>();
try {
// Verification Utility
UnManagedExportVerificationUtility validationUtility = new UnManagedExportVerificationUtility(_dbClient);
// Discover port groups (data mover ids) and group names (data mover names)
Set<StorageHADomain> activeDataMovers = discoverActiveDataMovers(storageSystem);
// Reused from discoverAll
// Discover ports (data mover interfaces) with the data movers in the active set.
Map<String, List<StoragePort>> ports = discoverPorts(storageSystem, activeDataMovers);
_logger.info("No of newly discovered port {}", ports.get(NEW).size());
_logger.info("No of existing discovered port {}", ports.get(EXISTING).size());
if (!ports.get(NEW).isEmpty()) {
_dbClient.createObject(ports.get(NEW));
}
List<StoragePort> allPortsList = ports.get(NEW);
allPortsList.addAll(ports.get(EXISTING));
Map<String, List<StoragePort>> allPorts = new ConcurrentHashMap<String, List<StoragePort>>();
for (StoragePort sPort : allPortsList) {
_logger.debug("DM Storage Port {} StorageHADomain {}", sPort.getPortNetworkId(), sPort.getStorageHADomain());
List<StoragePort> spList = allPorts.get(sPort.getStorageHADomain().toString());
if (spList == null) {
spList = new ArrayList<>();
}
spList.add(sPort);
allPorts.put(sPort.getStorageHADomain().toString(), spList);
}
Map<String, List<StorageHADomain>> allVdms = discoverVdmPortGroups(storageSystem, activeDataMovers);
if (!allVdms.get(NEW).isEmpty()) {
_dbClient.createObject(allVdms.get(NEW));
}
Set<StorageHADomain> allActiveVDMs = new HashSet();
allActiveVDMs.addAll(allVdms.get(NEW));
allActiveVDMs.addAll(allVdms.get(EXISTING));
activeDataMovers.addAll(allVdms.get(NEW));
activeDataMovers.addAll(allVdms.get(EXISTING));
Map<String, List<StoragePort>> allVdmPorts = discoverVdmPorts(storageSystem, allActiveVDMs);
if (!allVdmPorts.get(NEW).isEmpty()) {
_dbClient.createObject(allVdmPorts.get(NEW));
}
List<StoragePort> allVDMPortsList = allVdmPorts.get(NEW);
allVDMPortsList.addAll(allVdmPorts.get(EXISTING));
for (StoragePort sPort : allVDMPortsList) {
List<StoragePort> spList = allPorts.get(sPort.getStorageHADomain().toString());
_logger.debug("VDM Storage Port {} StorageHADomain {}", sPort.getPortNetworkId(), sPort.getStorageHADomain());
if (spList == null) {
spList = new ArrayList<>();
}
spList.add(sPort);
allPorts.put(sPort.getStorageHADomain().toString(), spList);
}
List<UnManagedFileSystem> unManagedExportBatch = new ArrayList<>();
for (StorageHADomain mover : activeDataMovers) {
// Get storage port and name for the DM
if (allPorts.get(mover.getId().toString()) == null || allPorts.get(mover.getId().toString()).isEmpty()) {
// Did not find a single storage port for this DM, ignore it
_logger.debug("No Ports found for {} {}", mover.getName(), mover.getAdapterName());
continue;
} else {
_logger.debug("Number of Ports found for {} : {} ", mover.getName() + ":" + mover.getAdapterName(), allPorts.get(mover.getId().toString()).size());
}
Collections.shuffle(allPorts.get(mover.getId().toString()));
StoragePort storagePort = allPorts.get(mover.getId().toString()).get(0);
if (storagePort == null) {
// Did not find a single storage port for this DM, ignore it
_logger.debug("StoragePort is null");
continue;
}
// storagePort.setStorageHADomain(mover.getId());
// get vnas uri
URI moverURI = getNASUri(mover, storageSystem);
// Retrieve FS-mountpath map for the Data Mover.
_logger.info("Retrieving FS-mountpath map for Data Mover {}.", mover.getAdapterName());
VNXFileSshApi sshDmApi = new VNXFileSshApi();
sshDmApi.setConnParams(storageSystem.getIpAddress(), storageSystem.getUsername(), storageSystem.getPassword());
Map<String, String> fileSystemMountpathMap = sshDmApi.getFsMountpathMap(mover.getAdapterName());
Map<String, Map<String, String>> moverExportDetails = sshDmApi.getNFSExportsForPath(mover.getAdapterName());
Map<String, String> nameIdMap = getFsNameFsNativeIdMap(storageSystem);
// Loop through the map and, if the file exists in DB, retrieve the
// export, process export, and associate export with the FS
Set<String> fsNames = fileSystemMountpathMap.keySet();
for (String fsName : fsNames) {
// Retrieve FS from DB. If FS found, retrieve export and process
String fsMountPath = fileSystemMountpathMap.get(fsName);
// Get FS ID for nativeGUID
// VNXFileSystem vnxFileSystems = discoverNamedFileSystem(storageSystem, fsName);
String fsId = nameIdMap.get(fsName);
_logger.debug("Resolved FileSystem name {} to native Id {}", fsName, fsId);
UnManagedFileSystem vnxufs = null;
if (fsId != null) {
String fsNativeGuid = NativeGUIDGenerator.generateNativeGuidForPreExistingFileSystem(storageSystem.getSystemType(), storageSystem.getSerialNumber().toUpperCase(), fsId);
vnxufs = checkUnManagedFileSystemExistsInDB(fsNativeGuid);
}
if (vnxufs != null) {
// Get export info
int noOfExports = 0;
boolean inValidExports = false;
for (String expPath : moverExportDetails.keySet()) {
if (!expPath.contains(fsMountPath)) {
// Ingore this path as it is not among the exports
continue;
} else {
// We should process only FS and its sub-directory exports only.
String subDir = expPath.substring(fsMountPath.length());
if (!subDir.isEmpty() && !subDir.startsWith("/")) {
continue;
}
_logger.info("Path : {} ", expPath);
noOfExports++;
}
// Used as for rules validation
List<UnManagedFileExportRule> unManagedExportRules = new ArrayList<UnManagedFileExportRule>();
Map<String, String> fsExportInfo = moverExportDetails.get(expPath);
if ((fsExportInfo != null) && (fsExportInfo.size() > 0)) {
// If multiple security flavors, do not add to ViPR DB
String securityFlavors = fsExportInfo.get(VNXFileConstants.SECURITY_TYPE);
if (securityFlavors == null || securityFlavors.length() == 0) {
securityFlavors = "sys";
}
if (securityFlavors != null) {
String fsMountPoint = storagePort.getPortNetworkId() + ":" + expPath;
_logger.info("Associating FS export map for VNX UMFS {}", vnxufs.getLabel());
associateExportWithFS(vnxufs, expPath, fsExportInfo, expPath, storagePort);
_logger.debug("Export map for VNX UMFS {} = {}", vnxufs.getLabel(), vnxufs.getFsUnManagedExportMap());
List<UnManagedFileExportRule> exportRules = applyAllSecurityRules(vnxufs.getId(), expPath, fsMountPoint, securityFlavors, fsExportInfo);
_logger.info("Number of export rules discovered for file system {} is {}", vnxufs.getId() + ":" + vnxufs.getLabel(), exportRules.size());
for (UnManagedFileExportRule dbExportRule : exportRules) {
_logger.info("Unmanaged File Export Rule : {}", dbExportRule);
String fsExportRulenativeId = dbExportRule.getFsExportIndex();
_logger.info("Native Id using to build Native Guid {}", fsExportRulenativeId);
String fsUnManagedFileExportRuleNativeGuid = NativeGUIDGenerator.generateNativeGuidForPreExistingFileExportRule(storageSystem, fsExportRulenativeId);
_logger.info("Native GUID {}", fsUnManagedFileExportRuleNativeGuid);
dbExportRule.setNativeGuid(fsUnManagedFileExportRuleNativeGuid);
dbExportRule.setFileSystemId(vnxufs.getId());
dbExportRule.setId(URIUtil.createId(UnManagedFileExportRule.class));
// Build all export rules list.
unManagedExportRules.add(dbExportRule);
}
// apply as per API SVC Validations.
if (!unManagedExportRules.isEmpty()) {
boolean isAllRulesValid = validationUtility.validateUnManagedExportRules(unManagedExportRules, false);
if (isAllRulesValid) {
_logger.info("Validating rules success for export {}", expPath);
for (UnManagedFileExportRule exportRule : unManagedExportRules) {
UnManagedFileExportRule existingRule = checkUnManagedFsExportRuleExistsInDB(_dbClient, exportRule.getNativeGuid());
if (existingRule == null) {
newUnManagedExportRules.add(exportRule);
} else {
// Remove the existing rule.
existingRule.setInactive(true);
_dbClient.persistObject(existingRule);
newUnManagedExportRules.add(exportRule);
}
}
vnxufs.setHasExports(true);
vnxufs.putFileSystemCharacterstics(UnManagedFileSystem.SupportedFileSystemCharacterstics.IS_FILESYSTEM_EXPORTED.toString(), TRUE);
// Set the correct storage port
if (null != storagePort) {
StringSet storagePorts = new StringSet();
storagePorts.add(storagePort.getId().toString());
vnxufs.getFileSystemInformation().put(UnManagedFileSystem.SupportedFileSystemInformation.STORAGE_PORT.toString(), storagePorts);
}
_dbClient.persistObject(vnxufs);
_logger.info("File System {} has Exports and their size is {}", vnxufs.getId(), newUnManagedExportRules.size());
} else {
_logger.warn("Validating rules failed for export {}. Ignoring to import these rules into ViPR DB", vnxufs);
inValidExports = true;
}
} else {
_logger.warn("Export discovery failed for {}. Ignoring to import these rules into ViPR DB", vnxufs);
inValidExports = true;
}
// Adding this additional logic to avoid OOM
if (newUnManagedExportRules.size() == MAX_UMFS_RECORD_SIZE) {
_logger.info("Saving Number of New UnManagedFileExportRule(s) {}", newUnManagedExportRules.size());
_dbClient.createObject(newUnManagedExportRules);
newUnManagedExportRules.clear();
}
// Adding this additional logic to avoid OOM
if (oldUnManagedExportRules.size() == MAX_UMFS_RECORD_SIZE) {
_logger.info("Saving Number of Existing UnManagedFileExportRule(s) {}", oldUnManagedExportRules.size());
_dbClient.persistObject(oldUnManagedExportRules);
oldUnManagedExportRules.clear();
}
}
}
}
_logger.info("No of exports found for path {} = {} ", fsMountPath, noOfExports);
if (noOfExports == 0) {
_logger.info("FileSystem {} does not have any exports ", vnxufs.getLabel());
vnxufs.setHasExports(false);
}
// Don't consider the unmanaged file systems with invalid exports!!!
if (inValidExports) {
_logger.info("Ignoring unmanaged file system {}, due to invalid exports", vnxufs.getLabel());
vnxufs.setInactive(true);
}
// set the vNAS uri in umfs
StringSet moverSet = new StringSet();
moverSet.add(moverURI.toString());
vnxufs.putFileSystemInfo(UnManagedFileSystem.SupportedFileSystemInformation.NAS.toString(), moverSet);
_logger.info("nas server id {} and fs name {}", mover.getName(), fsName);
unManagedExportBatch.add(vnxufs);
if (unManagedExportBatch.size() >= VNXFileConstants.VNX_FILE_BATCH_SIZE) {
// Add UnManagedFileSystem batch
// Update UnManagedFilesystem
_dbClient.persistObject(unManagedExportBatch);
unManagedExportBatch.clear();
}
}
}
}
if (!unManagedExportBatch.isEmpty()) {
// Update UnManagedFilesystem
_dbClient.persistObject(unManagedExportBatch);
unManagedExportBatch.clear();
}
if (!newUnManagedExportRules.isEmpty()) {
// create new UnManagedExportFules
_logger.info("Saving Number of New UnManagedFileExportRule(s) {}", newUnManagedExportRules.size());
_dbClient.createObject(newUnManagedExportRules);
newUnManagedExportRules.clear();
}
if (!oldUnManagedExportRules.isEmpty()) {
// Update exisiting UnManagedExportFules
_logger.info("Saving Number of Old UnManagedFileExportRule(s) {}", oldUnManagedExportRules.size());
_dbClient.persistObject(oldUnManagedExportRules);
oldUnManagedExportRules.clear();
}
// discovery succeeds
storageSystem.setDiscoveryStatus(DiscoveredDataObject.DataCollectionJobStatus.COMPLETE.toString());
detailedStatusMessage = String.format("Discovery completed successfully for VNXFile export: %s", storageSystemId.toString());
} catch (Exception ex) {
if (storageSystem != null) {
cleanupDiscovery(storageSystem);
}
storageSystem.setDiscoveryStatus(DiscoveredDataObject.DataCollectionJobStatus.ERROR.toString());
detailedStatusMessage = String.format("Discovery failed for VNXFile exports %s because %s", storageSystemId.toString(), ex.getLocalizedMessage());
_logger.error(detailedStatusMessage, ex);
} finally {
if (storageSystem != null) {
try {
// set detailed message
storageSystem.setLastDiscoveryStatusMessage(detailedStatusMessage);
_dbClient.persistObject(storageSystem);
} catch (Exception ex) {
_logger.error("Error while persisting object to DB", ex);
}
}
}
}
use of com.emc.storageos.db.client.model.UnManagedDiscoveredObjects.UnManagedFileExportRule in project coprhd-controller by CoprHD.
the class VNXFileCommunicationInterface method createUnManagedExportRule.
private UnManagedFileExportRule createUnManagedExportRule(URI id, String exportPath, String mountPoint, String securityFlavor, String anonUser, StringSet accessHosts, StringSet roHosts, StringSet rwHosts, StringSet rootHosts) {
UnManagedFileExportRule umfsExpRule = new UnManagedFileExportRule();
// Don't create the ID here ...
umfsExpRule.setFileSystemId(id);
umfsExpRule.setAnon(anonUser);
umfsExpRule.setExportPath(exportPath);
umfsExpRule.setMountPoint(mountPoint);
umfsExpRule.setSecFlavor(securityFlavor);
if (anonUser != null) {
if (anonUser.equalsIgnoreCase(VNXFileConstants.ROOT_ANON_USER)) {
umfsExpRule.setAnon(VNXFileConstants.ROOT);
} else {
umfsExpRule.setAnon(anonUser);
}
} else {
umfsExpRule.setAnon(VNXFileConstants.NOBODY);
}
if (accessHosts != null && roHosts == null) {
if (rwHosts == null) {
umfsExpRule.setReadWriteHosts(accessHosts);
} else {
umfsExpRule.setReadOnlyHosts(accessHosts);
}
}
if (roHosts != null) {
StringSet readOnlyHosts = roHosts;
if (accessHosts != null) {
for (String accHost : accessHosts) {
if (!(readOnlyHosts.contains(accHost))) {
readOnlyHosts.add(accHost);
}
}
}
umfsExpRule.setReadOnlyHosts(readOnlyHosts);
}
if (rwHosts != null) {
umfsExpRule.setReadWriteHosts(rwHosts);
}
if (rootHosts != null) {
umfsExpRule.setRootHosts(rootHosts);
}
return umfsExpRule;
}
Aggregations