Examples with ConfigQueryEntry com.sequenceiq.common.api.cloudstorage.query.ConfigQueryEntry used on opensource projects

Example 6 with ConfigQueryEntry

use of com.sequenceiq.common.api.cloudstorage.query.ConfigQueryEntry in project cloudbreak by hortonworks.

the class CmCloudStorageConfigDetailsTest method testAttachedClusterPaths.

@Test
public void testAttachedClusterPaths() {
    prepareBlueprintProcessorFactoryMock(RANGER_ADMIN, RESOURCEMANAGER, ZEPPELIN_SERVER, HBASE_MASTER, NAMENODE);
    FileSystemConfigQueryObject fileSystemConfigQueryObject = FileSystemConfigQueryObject.Builder.builder().withStorageName(STORAGE_NAME).withClusterName(CLUSTER_NAME).withBlueprintText(BLUEPRINT_TEXT).withDatalakeCluster(false).withAttachedCluster(true).withFileSystemType(FileSystemType.S3.name()).build();
    Set<ConfigQueryEntry> bigCluster = underTest.queryParameters(fileSystemConfigQueryObject);
    Assert.assertEquals(4L, bigCluster.size());
    Set<ConfigQueryEntry> rangerAdmin = serviceEntry(bigCluster, RANGER_ADMIN);
    Set<ConfigQueryEntry> yarnLogs = serviceEntry(bigCluster, RESOURCEMANAGER);
    Set<ConfigQueryEntry> zeppelin = serviceEntry(bigCluster, ZEPPELIN_SERVER);
    Set<ConfigQueryEntry> hbaseMaster = serviceEntry(bigCluster, HBASE_MASTER);
    Assert.assertEquals(1, rangerAdmin.size());
    Assert.assertTrue(rangerAdmin.stream().map(ConfigQueryEntry::getDefaultPath).anyMatch("hwx-remote/ranger/audit"::equals));
    Assert.assertEquals(1, yarnLogs.size());
    Assert.assertTrue(yarnLogs.stream().map(ConfigQueryEntry::getDefaultPath).anyMatch("hwx-remote/oplogs/yarn-app-logs"::equals));
    Assert.assertEquals(1, zeppelin.size());
    Assert.assertTrue(zeppelin.stream().map(ConfigQueryEntry::getDefaultPath).anyMatch("hwx-remote/bigCluster/zeppelin/notebook"::equals));
    Assert.assertEquals(1, hbaseMaster.size());
    Assert.assertTrue(hbaseMaster.stream().map(ConfigQueryEntry::getDefaultPath).anyMatch("hwx-remote/bigCluster/hbase"::equals));
}

Example 7 with ConfigQueryEntry

use of com.sequenceiq.common.api.cloudstorage.query.ConfigQueryEntry in project cloudbreak by hortonworks.

the class CmCloudStorageConfigDetailsTest method testSubFolder.

@Test
public void testSubFolder() {
    prepareBlueprintProcessorFactoryMock(RANGER_ADMIN, ZEPPELIN_SERVER, HBASE_MASTER, NAMENODE);
    FileSystemConfigQueryObject fileSystemConfigQueryObject = FileSystemConfigQueryObject.Builder.builder().withStorageName("hwx-remote.dfs.core.windows.net/subfolder").withClusterName(CLUSTER_NAME).withBlueprintText(BLUEPRINT_TEXT).withDatalakeCluster(false).withAttachedCluster(false).withFileSystemType(FileSystemType.ADLS_GEN_2.name()).build();
    Set<ConfigQueryEntry> bigCluster = underTest.queryParameters(fileSystemConfigQueryObject);
    Assert.assertEquals(3L, bigCluster.size());
    Set<ConfigQueryEntry> rangerAdmin = serviceEntry(bigCluster, RANGER_ADMIN);
    Set<ConfigQueryEntry> zeppelin = serviceEntry(bigCluster, ZEPPELIN_SERVER);
    Set<ConfigQueryEntry> hbaseMaster = serviceEntry(bigCluster, HBASE_MASTER);
    Assert.assertEquals(1, rangerAdmin.size());
    Assert.assertTrue(rangerAdmin.stream().map(ConfigQueryEntry::getDefaultPath).anyMatch("hwx-remote.dfs.core.windows.net/subfolder/ranger/audit"::equals));
    Assert.assertEquals(1, zeppelin.size());
    Assert.assertTrue(zeppelin.stream().map(ConfigQueryEntry::getDefaultPath).anyMatch("hwx-remote.dfs.core.windows.net/subfolder/bigCluster/zeppelin/notebook"::equals));
    Assert.assertEquals(1, hbaseMaster.size());
    Assert.assertTrue(hbaseMaster.stream().map(ConfigQueryEntry::getDefaultPath).anyMatch("hwx-remote.dfs.core.windows.net/subfolder/bigCluster/hbase"::equals));
}

Example 8 with ConfigQueryEntry

use of com.sequenceiq.common.api.cloudstorage.query.ConfigQueryEntry in project cloudbreak by hortonworks.

the class CmCloudStorageConfigDetailsTest method testWhenHiveMetasoreAndRangerAdminIsPresentedDoubleAndNotAttachedThenShouldReturnWithRangerConfigs.

@Test
public void testWhenHiveMetasoreAndRangerAdminIsPresentedDoubleAndNotAttachedThenShouldReturnWithRangerConfigs() {
    Map<String, Set<String>> map = new HashMap<>();
    map.put("master", Sets.newHashSet(HIVE_METASTORE, RANGER_ADMIN, NAMENODE));
    map.put("slave_1", Sets.newHashSet(HIVE_METASTORE, RANGER_ADMIN));
    prepareBlueprintProcessorFactoryMock(map);
    FileSystemConfigQueryObject fileSystemConfigQueryObject = FileSystemConfigQueryObject.Builder.builder().withStorageName(STORAGE_NAME).withClusterName(CLUSTER_NAME).withBlueprintText(BLUEPRINT_TEXT).withAttachedCluster(false).withFileSystemType(FileSystemType.S3.name()).build();
    Set<ConfigQueryEntry> bigCluster = underTest.queryParameters(fileSystemConfigQueryObject);
    Assert.assertEquals(4L, bigCluster.size());
    Set<ConfigQueryEntry> rangerAdmins = serviceEntry(bigCluster, RANGER_ADMIN);
    Set<ConfigQueryEntry> hive = serviceEntry(bigCluster, HIVE_METASTORE);
    Assert.assertEquals(1, rangerAdmins.size());
    Assert.assertTrue(rangerAdmins.stream().anyMatch(cqe -> cqe.getDefaultPath().equals("hwx-remote/ranger/audit")));
    Assert.assertEquals(3, hive.size());
    Assert.assertTrue(hive.stream().anyMatch(cqe -> cqe.getDefaultPath().equals("hwx-remote/warehouse/tablespace/managed/hive")));
    Assert.assertTrue(hive.stream().anyMatch(cqe -> cqe.getDefaultPath().equals("hwx-remote/warehouse/tablespace/external/hive")));
}

Example 9 with ConfigQueryEntry

use of com.sequenceiq.common.api.cloudstorage.query.ConfigQueryEntry in project cloudbreak by hortonworks.

the class CloudStorageDecorator method filterConfigsWithTemplatePlaceholder.

private Set<ConfigQueryEntry> filterConfigsWithTemplatePlaceholder(CloudStorageRequest request, Set<ConfigQueryEntry> recommendations) {
    Set<ConfigQueryEntry> filtered = new LinkedHashSet<>();
    for (StorageLocationBase location : request.getLocations()) {
        recommendations.stream().filter(configQueryEntry -> configQueryEntry.getType().equals(location.getType())).findFirst().ifPresent(configQueryEntry -> {
            ConfigQueryEntry custom = configQueryEntry.copy();
            custom.setDefaultPath(location.getValue());
            filtered.add(custom);
        });
    }
    return filtered;
}

Example 10 with ConfigQueryEntry

use of com.sequenceiq.common.api.cloudstorage.query.ConfigQueryEntry in project cloudbreak by hortonworks.

the class CloudStorageDecorator method updateCloudStorageLocations.

public CloudStorageRequest updateCloudStorageLocations(String blueprintName, String clusterName, CloudStorageRequest request, List<SdxClusterResponse> datalakes) {
    if (hasDatalake(datalakes)) {
        Pair<String, FileSystemType> sdxBaseLocationFileSystemType = getBaseLocationWithFileSystemTypeFromSdx(datalakes.get(0));
        Set<ConfigQueryEntry> recommendations = getRecommendations(blueprintName, clusterName, sdxBaseLocationFileSystemType);
        if (storageLocationsNotDefined(request)) {
            if (request == null) {
                request = new CloudStorageRequest();
            }
            if (request.getLocations() == null) {
                request.setLocations(new ArrayList<>());
            }
            for (ConfigQueryEntry recommendation : recommendations) {
                request.getLocations().add(createStorageLocationBaseByTypeAndDefaultPath(recommendation.getType(), recommendation.getDefaultPath()));
            }
        } else {
            Map<CloudStorageCdpService, String> templatedLocations = findLocationsThatContainsTemplatedValue(request);
            if (!templatedLocations.isEmpty()) {
                LOGGER.info("Cloud storage location(s) has found with template placeholder(s). About to replace them with the recommended one(s).");
                Set<ConfigQueryEntry> filtered = filterConfigsWithTemplatePlaceholder(request, recommendations);
                Set<ConfigQueryEntry> replaced = queryParameters(filtered, blueprintName, clusterName, sdxBaseLocationFileSystemType);
                replaceTemplatedLocationValuesWithFilledValues(request, replaced, templatedLocations);
            }
        }
    }
    return request;
}