Search in sources :

Example 1 with ConfigQueryEntry

use of com.sequenceiq.common.api.cloudstorage.query.ConfigQueryEntry in project cloudbreak by hortonworks.

the class ConfigQueryEntryToFileSystemParameterV4ResponseConverterTest method createConfigQueryEntry.

private ConfigQueryEntry createConfigQueryEntry() {
    ConfigQueryEntry entry = new ConfigQueryEntry();
    entry.setType(CloudStorageCdpService.RANGER_AUDIT);
    entry.setDefaultPath(TEST_PATH_VALUE);
    entry.setProtocol(TEST_PROTOCOL);
    entry.setDescription(TEST_DESCRIPTION);
    entry.setPropertyDisplayName(TEST_DISPLAY_NAME);
    entry.setPropertyFile(TEST_PROPERTY_FILE);
    entry.setPropertyName(TEST_PROPERTY_NAME);
    entry.setRelatedMissingServices(new HashSet<>(Arrays.asList(TEST_RELATED_SERVICE, TEST_RELATED_SERVICE2)));
    entry.setRelatedServices(new HashSet<>(Arrays.asList(TEST_RELATED_SERVICE, TEST_RELATED_SERVICE2)));
    entry.setSupportedStorages(Collections.singleton(TEST_STORAGE));
    return entry;
}
Also used : ConfigQueryEntry(com.sequenceiq.common.api.cloudstorage.query.ConfigQueryEntry)

Example 2 with ConfigQueryEntry

use of com.sequenceiq.common.api.cloudstorage.query.ConfigQueryEntry in project cloudbreak by hortonworks.

the class CloudStorageConfigDetails method queryParameters.

public Set<ConfigQueryEntry> queryParameters(BlueprintTextProcessor blueprintTextProcessor, ConfigQueryEntries configQueryEntries, FileSystemConfigQueryObject request) {
    Set<ConfigQueryEntry> filtered = new HashSet<>();
    Map<String, Set<String>> componentsByHostGroup = blueprintTextProcessor.getComponentsByHostGroup();
    boolean attachedCluster = request.isAttachedCluster();
    for (Map.Entry<String, Set<String>> serviceHostgroupEntry : componentsByHostGroup.entrySet()) {
        for (String service : serviceHostgroupEntry.getValue()) {
            Set<ConfigQueryEntry> collectedEntries = configQueryEntries.getEntries().stream().filter(configQueryEntry -> configQueryEntry.getRelatedServices().stream().anyMatch(relatedService -> relatedService.equalsIgnoreCase(service))).filter(configQueryEntry -> {
                if ((configQueryEntry.isRequiredForAttachedCluster() && attachedCluster) || !attachedCluster) {
                    return true;
                }
                return false;
            }).filter(configQueryEntry -> configQueryEntry.getSupportedStorages().contains(request.getFileSystemType().toUpperCase())).collect(Collectors.toSet());
            filtered.addAll(collectedEntries);
        }
    }
    Set<ConfigQueryEntry> collectedEntries = configQueryEntries.getEntries().stream().filter(configQueryEntry -> blueprintDoesNotContainActual(configQueryEntry.getRelatedMissingServices(), componentsByHostGroup)).collect(Collectors.toSet());
    filtered.addAll(collectedEntries);
    String fileSystemTypeRequest = request.getFileSystemType();
    FileSystemType fileSystemType = FileSystemType.valueOf(fileSystemTypeRequest);
    String protocol = fileSystemType.getProtocol();
    Map<String, Object> templateObject = getTemplateObject(request, protocol);
    for (ConfigQueryEntry configQueryEntry : filtered) {
        try {
            boolean secure = request.isSecure();
            configQueryEntry.setProtocol(secure ? protocol + "s" : protocol);
            configQueryEntry.setSecure(secure);
            configQueryEntry.setDefaultPath(generateConfigWithParameters(configQueryEntry.getDefaultPath(), fileSystemType, templateObject));
        } catch (IOException e) {
        }
    }
    filtered = filtered.stream().sorted(Comparator.comparing(ConfigQueryEntry::getPropertyName)).collect(Collectors.toCollection(LinkedHashSet::new));
    return filtered;
}
Also used : Logger(org.slf4j.Logger) HandlebarTemplate(com.sequenceiq.cloudbreak.handlebar.HandlebarTemplate) ConfigQueryEntries(com.sequenceiq.common.api.cloudstorage.query.ConfigQueryEntries) LoggerFactory(org.slf4j.LoggerFactory) Set(java.util.Set) IOException(java.io.IOException) HashMap(java.util.HashMap) BlueprintTextProcessor(com.sequenceiq.cloudbreak.template.processor.BlueprintTextProcessor) Collectors(java.util.stream.Collectors) StringUtils(org.apache.commons.lang3.StringUtils) HashSet(java.util.HashSet) Service(org.springframework.stereotype.Service) Map(java.util.Map) FileSystemType(com.sequenceiq.common.model.FileSystemType) ConfigQueryEntry(com.sequenceiq.common.api.cloudstorage.query.ConfigQueryEntry) HandlebarUtils(com.sequenceiq.cloudbreak.handlebar.HandlebarUtils) Handlebars(com.github.jknack.handlebars.Handlebars) Comparator(java.util.Comparator) LinkedHashSet(java.util.LinkedHashSet) Template(com.github.jknack.handlebars.Template) LinkedHashSet(java.util.LinkedHashSet) ConfigQueryEntry(com.sequenceiq.common.api.cloudstorage.query.ConfigQueryEntry) Set(java.util.Set) HashSet(java.util.HashSet) LinkedHashSet(java.util.LinkedHashSet) IOException(java.io.IOException) FileSystemType(com.sequenceiq.common.model.FileSystemType) HashMap(java.util.HashMap) Map(java.util.Map) HashSet(java.util.HashSet) LinkedHashSet(java.util.LinkedHashSet)

Example 3 with ConfigQueryEntry

use of com.sequenceiq.common.api.cloudstorage.query.ConfigQueryEntry in project cloudbreak by hortonworks.

the class CloudStorageDecoratorTest method testUpdateCloudStorageLocationsWhenRequestContainsTemplatedLocationThenItShouldBeReplaced.

@Test
void testUpdateCloudStorageLocationsWhenRequestContainsTemplatedLocationThenItShouldBeReplaced() {
    CloudStorageCdpService storageLocationType = CloudStorageCdpService.DEFAULT_FS;
    String eStorageLocationValue = "s3a://some-dir/some-other-dir/{{{clusterName}}}";
    StorageLocationBase storageLocationBase = new StorageLocationBase();
    storageLocationBase.setType(storageLocationType);
    storageLocationBase.setValue(eStorageLocationValue);
    List<StorageLocationBase> storageLocations = new ArrayList<>(1);
    storageLocations.add(storageLocationBase);
    CloudStorageRequest request = new CloudStorageRequest();
    request.setLocations(storageLocations);
    SdxClusterResponse sdxResponse = new SdxClusterResponse();
    String storageLocationValue = "s3a://some-dir/some-other-dir/" + CLUSTER_NAME;
    sdxResponse.setCloudStorageBaseLocation(storageLocationValue);
    sdxResponse.setCloudStorageFileSystemType(FileSystemType.S3);
    ConfigQueryEntry cqe = new ConfigQueryEntry();
    cqe.setType(CloudStorageCdpService.DEFAULT_FS);
    cqe.setDefaultPath(storageLocationValue);
    Set<ConfigQueryEntry> cqes = new LinkedHashSet<>(1);
    cqes.add(cqe);
    when(blueprintService.queryFileSystemParameters(BLUEPRINT_NAME, CLUSTER_NAME, storageLocationValue, FileSystemType.S3.name(), "", true, false, 0L)).thenReturn(cqes);
    Pair<Blueprint, String> mockBt = mock(Pair.class);
    when(blueprintService.getBlueprintAndText(BLUEPRINT_NAME, 0L)).thenReturn(mockBt);
    FileSystemConfigQueryObject mockfscqo = mock(FileSystemConfigQueryObject.class);
    when(blueprintService.createFileSystemConfigQueryObject(mockBt, CLUSTER_NAME, sdxResponse.getCloudStorageBaseLocation(), sdxResponse.getCloudStorageFileSystemType().name(), "", true, false)).thenReturn(mockfscqo);
    when(cmCloudStorageConfigProvider.queryParameters(any(), eq(mockfscqo))).thenReturn(cqes);
    CloudStorageRequest result = underTest.updateCloudStorageLocations(BLUEPRINT_NAME, CLUSTER_NAME, request, List.of(sdxResponse));
    assertNotNull(result);
    assertEquals(0, result.getLocations().stream().filter(slb -> slb.getValue().contains("{{{") && slb.getValue().contains("}}}")).count());
}
Also used : LinkedHashSet(java.util.LinkedHashSet) ConfigQueryEntry(com.sequenceiq.common.api.cloudstorage.query.ConfigQueryEntry) CloudStorageCdpService(com.sequenceiq.common.model.CloudStorageCdpService) CloudStorageRequest(com.sequenceiq.common.api.cloudstorage.CloudStorageRequest) Blueprint(com.sequenceiq.cloudbreak.domain.Blueprint) ArrayList(java.util.ArrayList) FileSystemConfigQueryObject(com.sequenceiq.cloudbreak.template.filesystem.FileSystemConfigQueryObject) SdxClusterResponse(com.sequenceiq.sdx.api.model.SdxClusterResponse) StorageLocationBase(com.sequenceiq.common.api.cloudstorage.StorageLocationBase) Test(org.junit.jupiter.api.Test)

Example 4 with ConfigQueryEntry

use of com.sequenceiq.common.api.cloudstorage.query.ConfigQueryEntry in project cloudbreak by hortonworks.

the class CmCloudStorageConfigDetailsTest method testWhenHiveMetasoreAndRangerAdminIsPresentedDoubleAndAttachedThenShouldReturnWithRangerConfigs.

@Test
public void testWhenHiveMetasoreAndRangerAdminIsPresentedDoubleAndAttachedThenShouldReturnWithRangerConfigs() {
    Map<String, Set<String>> map = new HashMap<>();
    map.put("master", Sets.newHashSet(HIVE_METASTORE, RANGER_ADMIN, NAMENODE));
    map.put("slave_1", Sets.newHashSet(HIVE_METASTORE, RANGER_ADMIN));
    prepareBlueprintProcessorFactoryMock(map);
    FileSystemConfigQueryObject fileSystemConfigQueryObject = FileSystemConfigQueryObject.Builder.builder().withStorageName(STORAGE_NAME).withClusterName(CLUSTER_NAME).withBlueprintText(BLUEPRINT_TEXT).withAttachedCluster(true).withFileSystemType(FileSystemType.S3.name()).build();
    Set<ConfigQueryEntry> bigCluster = underTest.queryParameters(fileSystemConfigQueryObject);
    Assert.assertEquals(1L, bigCluster.size());
    Set<ConfigQueryEntry> rangerAdmins = serviceEntry(bigCluster, RANGER_ADMIN);
    Assert.assertEquals(1, rangerAdmins.size());
    Assert.assertTrue(rangerAdmins.stream().anyMatch(cqe -> cqe.getDefaultPath().equals("hwx-remote/ranger/audit")));
}
Also used : CloudbreakResourceReaderService(com.sequenceiq.cloudbreak.service.CloudbreakResourceReaderService) Mock(org.mockito.Mock) RunWith(org.junit.runner.RunWith) Set(java.util.Set) ReflectionTestUtils(org.springframework.test.util.ReflectionTestUtils) IOException(java.io.IOException) HashMap(java.util.HashMap) Test(org.junit.Test) Mockito.when(org.mockito.Mockito.when) Collectors(java.util.stream.Collectors) CmTemplateProcessorFactory(com.sequenceiq.cloudbreak.cmtemplate.CmTemplateProcessorFactory) Sets(com.google.common.collect.Sets) Map(java.util.Map) FileSystemType(com.sequenceiq.common.model.FileSystemType) CloudStorageConfigDetails(com.sequenceiq.cloudbreak.template.filesystem.CloudStorageConfigDetails) FileSystemConfigQueryObject(com.sequenceiq.cloudbreak.template.filesystem.FileSystemConfigQueryObject) FileReaderUtils(com.sequenceiq.cloudbreak.util.FileReaderUtils) ConfigQueryEntry(com.sequenceiq.common.api.cloudstorage.query.ConfigQueryEntry) CmTemplateProcessor(com.sequenceiq.cloudbreak.cmtemplate.CmTemplateProcessor) Assert(org.junit.Assert) MockitoJUnitRunner(org.mockito.junit.MockitoJUnitRunner) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) Mockito.mock(org.mockito.Mockito.mock) Before(org.junit.Before) ConfigQueryEntry(com.sequenceiq.common.api.cloudstorage.query.ConfigQueryEntry) Set(java.util.Set) HashMap(java.util.HashMap) FileSystemConfigQueryObject(com.sequenceiq.cloudbreak.template.filesystem.FileSystemConfigQueryObject) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) Test(org.junit.Test)

Example 5 with ConfigQueryEntry

use of com.sequenceiq.common.api.cloudstorage.query.ConfigQueryEntry in project cloudbreak by hortonworks.

the class CmCloudStorageConfigDetailsTest method testStandaloneClusterPaths.

@Test
public void testStandaloneClusterPaths() {
    prepareBlueprintProcessorFactoryMock(RANGER_ADMIN, ZEPPELIN_SERVER, HBASE_MASTER, NAMENODE);
    FileSystemConfigQueryObject fileSystemConfigQueryObject = FileSystemConfigQueryObject.Builder.builder().withStorageName(STORAGE_NAME).withClusterName(CLUSTER_NAME).withBlueprintText(BLUEPRINT_TEXT).withDatalakeCluster(false).withAttachedCluster(false).withFileSystemType(FileSystemType.S3.name()).build();
    Set<ConfigQueryEntry> bigCluster = underTest.queryParameters(fileSystemConfigQueryObject);
    Assert.assertEquals(3L, bigCluster.size());
    Set<ConfigQueryEntry> rangerAdmin = serviceEntry(bigCluster, RANGER_ADMIN);
    Set<ConfigQueryEntry> zeppelin = serviceEntry(bigCluster, ZEPPELIN_SERVER);
    Set<ConfigQueryEntry> hbaseMaster = serviceEntry(bigCluster, HBASE_MASTER);
    Assert.assertEquals(1, rangerAdmin.size());
    Assert.assertTrue(rangerAdmin.stream().map(ConfigQueryEntry::getDefaultPath).anyMatch("hwx-remote/ranger/audit"::equals));
    Assert.assertEquals(1, zeppelin.size());
    Assert.assertTrue(zeppelin.stream().map(ConfigQueryEntry::getDefaultPath).anyMatch("hwx-remote/bigCluster/zeppelin/notebook"::equals));
    Assert.assertEquals(1, hbaseMaster.size());
    Assert.assertTrue(hbaseMaster.stream().map(ConfigQueryEntry::getDefaultPath).anyMatch("hwx-remote/bigCluster/hbase"::equals));
}
Also used : ConfigQueryEntry(com.sequenceiq.common.api.cloudstorage.query.ConfigQueryEntry) FileSystemConfigQueryObject(com.sequenceiq.cloudbreak.template.filesystem.FileSystemConfigQueryObject) Test(org.junit.Test)

Aggregations

ConfigQueryEntry (com.sequenceiq.common.api.cloudstorage.query.ConfigQueryEntry)21 FileSystemConfigQueryObject (com.sequenceiq.cloudbreak.template.filesystem.FileSystemConfigQueryObject)16 Test (org.junit.Test)12 FileSystemType (com.sequenceiq.common.model.FileSystemType)7 Set (java.util.Set)6 IOException (java.io.IOException)5 HashMap (java.util.HashMap)5 LinkedHashSet (java.util.LinkedHashSet)5 Map (java.util.Map)5 Collectors (java.util.stream.Collectors)5 Mock (org.mockito.Mock)5 Mockito.mock (org.mockito.Mockito.mock)5 Mockito.when (org.mockito.Mockito.when)5 Sets (com.google.common.collect.Sets)4 CmTemplateProcessor (com.sequenceiq.cloudbreak.cmtemplate.CmTemplateProcessor)4 CmTemplateProcessorFactory (com.sequenceiq.cloudbreak.cmtemplate.CmTemplateProcessorFactory)4 Blueprint (com.sequenceiq.cloudbreak.domain.Blueprint)4 CloudbreakResourceReaderService (com.sequenceiq.cloudbreak.service.CloudbreakResourceReaderService)4 CloudStorageConfigDetails (com.sequenceiq.cloudbreak.template.filesystem.CloudStorageConfigDetails)4 FileReaderUtils (com.sequenceiq.cloudbreak.util.FileReaderUtils)4