use of com.sequenceiq.cloudbreak.template.filesystem.FileSystemConfigQueryObject in project cloudbreak by hortonworks.
the class CloudStorageDecoratorTest method testUpdateCloudStorageLocationsWhenRequestContainsTemplatedLocationThenItShouldBeReplaced.
@Test
void testUpdateCloudStorageLocationsWhenRequestContainsTemplatedLocationThenItShouldBeReplaced() {
CloudStorageCdpService storageLocationType = CloudStorageCdpService.DEFAULT_FS;
String eStorageLocationValue = "s3a://some-dir/some-other-dir/{{{clusterName}}}";
StorageLocationBase storageLocationBase = new StorageLocationBase();
storageLocationBase.setType(storageLocationType);
storageLocationBase.setValue(eStorageLocationValue);
List<StorageLocationBase> storageLocations = new ArrayList<>(1);
storageLocations.add(storageLocationBase);
CloudStorageRequest request = new CloudStorageRequest();
request.setLocations(storageLocations);
SdxClusterResponse sdxResponse = new SdxClusterResponse();
String storageLocationValue = "s3a://some-dir/some-other-dir/" + CLUSTER_NAME;
sdxResponse.setCloudStorageBaseLocation(storageLocationValue);
sdxResponse.setCloudStorageFileSystemType(FileSystemType.S3);
ConfigQueryEntry cqe = new ConfigQueryEntry();
cqe.setType(CloudStorageCdpService.DEFAULT_FS);
cqe.setDefaultPath(storageLocationValue);
Set<ConfigQueryEntry> cqes = new LinkedHashSet<>(1);
cqes.add(cqe);
when(blueprintService.queryFileSystemParameters(BLUEPRINT_NAME, CLUSTER_NAME, storageLocationValue, FileSystemType.S3.name(), "", true, false, 0L)).thenReturn(cqes);
Pair<Blueprint, String> mockBt = mock(Pair.class);
when(blueprintService.getBlueprintAndText(BLUEPRINT_NAME, 0L)).thenReturn(mockBt);
FileSystemConfigQueryObject mockfscqo = mock(FileSystemConfigQueryObject.class);
when(blueprintService.createFileSystemConfigQueryObject(mockBt, CLUSTER_NAME, sdxResponse.getCloudStorageBaseLocation(), sdxResponse.getCloudStorageFileSystemType().name(), "", true, false)).thenReturn(mockfscqo);
when(cmCloudStorageConfigProvider.queryParameters(any(), eq(mockfscqo))).thenReturn(cqes);
CloudStorageRequest result = underTest.updateCloudStorageLocations(BLUEPRINT_NAME, CLUSTER_NAME, request, List.of(sdxResponse));
assertNotNull(result);
assertEquals(0, result.getLocations().stream().filter(slb -> slb.getValue().contains("{{{") && slb.getValue().contains("}}}")).count());
}
use of com.sequenceiq.cloudbreak.template.filesystem.FileSystemConfigQueryObject in project cloudbreak by hortonworks.
the class CmCloudStorageConfigDetailsTest method testWhenHiveMetasoreAndRangerAdminIsPresentedDoubleAndAttachedThenShouldReturnWithRangerConfigs.
@Test
public void testWhenHiveMetasoreAndRangerAdminIsPresentedDoubleAndAttachedThenShouldReturnWithRangerConfigs() {
Map<String, Set<String>> map = new HashMap<>();
map.put("master", Sets.newHashSet(HIVE_METASTORE, RANGER_ADMIN, NAMENODE));
map.put("slave_1", Sets.newHashSet(HIVE_METASTORE, RANGER_ADMIN));
prepareBlueprintProcessorFactoryMock(map);
FileSystemConfigQueryObject fileSystemConfigQueryObject = FileSystemConfigQueryObject.Builder.builder().withStorageName(STORAGE_NAME).withClusterName(CLUSTER_NAME).withBlueprintText(BLUEPRINT_TEXT).withAttachedCluster(true).withFileSystemType(FileSystemType.S3.name()).build();
Set<ConfigQueryEntry> bigCluster = underTest.queryParameters(fileSystemConfigQueryObject);
Assert.assertEquals(1L, bigCluster.size());
Set<ConfigQueryEntry> rangerAdmins = serviceEntry(bigCluster, RANGER_ADMIN);
Assert.assertEquals(1, rangerAdmins.size());
Assert.assertTrue(rangerAdmins.stream().anyMatch(cqe -> cqe.getDefaultPath().equals("hwx-remote/ranger/audit")));
}
use of com.sequenceiq.cloudbreak.template.filesystem.FileSystemConfigQueryObject in project cloudbreak by hortonworks.
the class CmCloudStorageConfigDetailsTest method testStandaloneClusterPaths.
@Test
public void testStandaloneClusterPaths() {
prepareBlueprintProcessorFactoryMock(RANGER_ADMIN, ZEPPELIN_SERVER, HBASE_MASTER, NAMENODE);
FileSystemConfigQueryObject fileSystemConfigQueryObject = FileSystemConfigQueryObject.Builder.builder().withStorageName(STORAGE_NAME).withClusterName(CLUSTER_NAME).withBlueprintText(BLUEPRINT_TEXT).withDatalakeCluster(false).withAttachedCluster(false).withFileSystemType(FileSystemType.S3.name()).build();
Set<ConfigQueryEntry> bigCluster = underTest.queryParameters(fileSystemConfigQueryObject);
Assert.assertEquals(3L, bigCluster.size());
Set<ConfigQueryEntry> rangerAdmin = serviceEntry(bigCluster, RANGER_ADMIN);
Set<ConfigQueryEntry> zeppelin = serviceEntry(bigCluster, ZEPPELIN_SERVER);
Set<ConfigQueryEntry> hbaseMaster = serviceEntry(bigCluster, HBASE_MASTER);
Assert.assertEquals(1, rangerAdmin.size());
Assert.assertTrue(rangerAdmin.stream().map(ConfigQueryEntry::getDefaultPath).anyMatch("hwx-remote/ranger/audit"::equals));
Assert.assertEquals(1, zeppelin.size());
Assert.assertTrue(zeppelin.stream().map(ConfigQueryEntry::getDefaultPath).anyMatch("hwx-remote/bigCluster/zeppelin/notebook"::equals));
Assert.assertEquals(1, hbaseMaster.size());
Assert.assertTrue(hbaseMaster.stream().map(ConfigQueryEntry::getDefaultPath).anyMatch("hwx-remote/bigCluster/hbase"::equals));
}
use of com.sequenceiq.cloudbreak.template.filesystem.FileSystemConfigQueryObject in project cloudbreak by hortonworks.
the class CmCloudStorageConfigDetailsTest method testAttachedClusterPaths.
@Test
public void testAttachedClusterPaths() {
prepareBlueprintProcessorFactoryMock(RANGER_ADMIN, RESOURCEMANAGER, ZEPPELIN_SERVER, HBASE_MASTER, NAMENODE);
FileSystemConfigQueryObject fileSystemConfigQueryObject = FileSystemConfigQueryObject.Builder.builder().withStorageName(STORAGE_NAME).withClusterName(CLUSTER_NAME).withBlueprintText(BLUEPRINT_TEXT).withDatalakeCluster(false).withAttachedCluster(true).withFileSystemType(FileSystemType.S3.name()).build();
Set<ConfigQueryEntry> bigCluster = underTest.queryParameters(fileSystemConfigQueryObject);
Assert.assertEquals(4L, bigCluster.size());
Set<ConfigQueryEntry> rangerAdmin = serviceEntry(bigCluster, RANGER_ADMIN);
Set<ConfigQueryEntry> yarnLogs = serviceEntry(bigCluster, RESOURCEMANAGER);
Set<ConfigQueryEntry> zeppelin = serviceEntry(bigCluster, ZEPPELIN_SERVER);
Set<ConfigQueryEntry> hbaseMaster = serviceEntry(bigCluster, HBASE_MASTER);
Assert.assertEquals(1, rangerAdmin.size());
Assert.assertTrue(rangerAdmin.stream().map(ConfigQueryEntry::getDefaultPath).anyMatch("hwx-remote/ranger/audit"::equals));
Assert.assertEquals(1, yarnLogs.size());
Assert.assertTrue(yarnLogs.stream().map(ConfigQueryEntry::getDefaultPath).anyMatch("hwx-remote/oplogs/yarn-app-logs"::equals));
Assert.assertEquals(1, zeppelin.size());
Assert.assertTrue(zeppelin.stream().map(ConfigQueryEntry::getDefaultPath).anyMatch("hwx-remote/bigCluster/zeppelin/notebook"::equals));
Assert.assertEquals(1, hbaseMaster.size());
Assert.assertTrue(hbaseMaster.stream().map(ConfigQueryEntry::getDefaultPath).anyMatch("hwx-remote/bigCluster/hbase"::equals));
}
use of com.sequenceiq.cloudbreak.template.filesystem.FileSystemConfigQueryObject in project cloudbreak by hortonworks.
the class CmCloudStorageConfigDetailsTest method testSubFolder.
@Test
public void testSubFolder() {
prepareBlueprintProcessorFactoryMock(RANGER_ADMIN, ZEPPELIN_SERVER, HBASE_MASTER, NAMENODE);
FileSystemConfigQueryObject fileSystemConfigQueryObject = FileSystemConfigQueryObject.Builder.builder().withStorageName("hwx-remote.dfs.core.windows.net/subfolder").withClusterName(CLUSTER_NAME).withBlueprintText(BLUEPRINT_TEXT).withDatalakeCluster(false).withAttachedCluster(false).withFileSystemType(FileSystemType.ADLS_GEN_2.name()).build();
Set<ConfigQueryEntry> bigCluster = underTest.queryParameters(fileSystemConfigQueryObject);
Assert.assertEquals(3L, bigCluster.size());
Set<ConfigQueryEntry> rangerAdmin = serviceEntry(bigCluster, RANGER_ADMIN);
Set<ConfigQueryEntry> zeppelin = serviceEntry(bigCluster, ZEPPELIN_SERVER);
Set<ConfigQueryEntry> hbaseMaster = serviceEntry(bigCluster, HBASE_MASTER);
Assert.assertEquals(1, rangerAdmin.size());
Assert.assertTrue(rangerAdmin.stream().map(ConfigQueryEntry::getDefaultPath).anyMatch("hwx-remote.dfs.core.windows.net/subfolder/ranger/audit"::equals));
Assert.assertEquals(1, zeppelin.size());
Assert.assertTrue(zeppelin.stream().map(ConfigQueryEntry::getDefaultPath).anyMatch("hwx-remote.dfs.core.windows.net/subfolder/bigCluster/zeppelin/notebook"::equals));
Assert.assertEquals(1, hbaseMaster.size());
Assert.assertTrue(hbaseMaster.stream().map(ConfigQueryEntry::getDefaultPath).anyMatch("hwx-remote.dfs.core.windows.net/subfolder/bigCluster/hbase"::equals));
}
Aggregations