Examples with FileSystemConfigQueryObject com.sequenceiq.cloudbreak.template.filesystem.FileSystemConfigQueryObject used on opensource projects

Example 6 with FileSystemConfigQueryObject

use of com.sequenceiq.cloudbreak.template.filesystem.FileSystemConfigQueryObject in project cloudbreak by hortonworks.

the class CmCloudStorageConfigDetailsTest method testWhenHiveMetasoreAndRangerAdminIsPresentedDoubleAndNotAttachedThenShouldReturnWithRangerConfigs.

@Test
public void testWhenHiveMetasoreAndRangerAdminIsPresentedDoubleAndNotAttachedThenShouldReturnWithRangerConfigs() {
    Map<String, Set<String>> map = new HashMap<>();
    map.put("master", Sets.newHashSet(HIVE_METASTORE, RANGER_ADMIN, NAMENODE));
    map.put("slave_1", Sets.newHashSet(HIVE_METASTORE, RANGER_ADMIN));
    prepareBlueprintProcessorFactoryMock(map);
    FileSystemConfigQueryObject fileSystemConfigQueryObject = FileSystemConfigQueryObject.Builder.builder().withStorageName(STORAGE_NAME).withClusterName(CLUSTER_NAME).withBlueprintText(BLUEPRINT_TEXT).withAttachedCluster(false).withFileSystemType(FileSystemType.S3.name()).build();
    Set<ConfigQueryEntry> bigCluster = underTest.queryParameters(fileSystemConfigQueryObject);
    Assert.assertEquals(4L, bigCluster.size());
    Set<ConfigQueryEntry> rangerAdmins = serviceEntry(bigCluster, RANGER_ADMIN);
    Set<ConfigQueryEntry> hive = serviceEntry(bigCluster, HIVE_METASTORE);
    Assert.assertEquals(1, rangerAdmins.size());
    Assert.assertTrue(rangerAdmins.stream().anyMatch(cqe -> cqe.getDefaultPath().equals("hwx-remote/ranger/audit")));
    Assert.assertEquals(3, hive.size());
    Assert.assertTrue(hive.stream().anyMatch(cqe -> cqe.getDefaultPath().equals("hwx-remote/warehouse/tablespace/managed/hive")));
    Assert.assertTrue(hive.stream().anyMatch(cqe -> cqe.getDefaultPath().equals("hwx-remote/warehouse/tablespace/external/hive")));
}

Example 7 with FileSystemConfigQueryObject

use of com.sequenceiq.cloudbreak.template.filesystem.FileSystemConfigQueryObject in project cloudbreak by hortonworks.

the class CloudStorageDecorator method queryParameters.

private Set<ConfigQueryEntry> queryParameters(Set<ConfigQueryEntry> filtered, String blueprintName, String clusterName, Pair<String, FileSystemType> sdxBaseLocationFileSystemType) {
    Pair<Blueprint, String> bt = blueprintService.getBlueprintAndText(blueprintName, 0L);
    FileSystemConfigQueryObject fsConfigO = blueprintService.createFileSystemConfigQueryObject(bt, clusterName, sdxBaseLocationFileSystemType.getLeft(), sdxBaseLocationFileSystemType.getRight().name(), "", true, false);
    return cmCloudStorageConfigProvider.queryParameters(filtered, fsConfigO);
}

Example 8 with FileSystemConfigQueryObject

use of com.sequenceiq.cloudbreak.template.filesystem.FileSystemConfigQueryObject in project cloudbreak by hortonworks.

the class CloudStorageDecoratorTest method testUpdateCloudStorageLocationsWhenRequestContainsOneTemplatedLocationAndOneWithoutTemplatePlaceholderThenThatShouldBeReplaced.

@Test
void testUpdateCloudStorageLocationsWhenRequestContainsOneTemplatedLocationAndOneWithoutTemplatePlaceholderThenThatShouldBeReplaced() {
    String templatedStorageLocationBaseValue = "s3a://some-dir/some-other-dir/";
    CloudStorageCdpService templatedStorageLocationType = CloudStorageCdpService.DEFAULT_FS;
    String templatedStorageLocationValue = templatedStorageLocationBaseValue + "{{{clusterName}}}";
    StorageLocationBase templatedStorageLocationBase = new StorageLocationBase();
    templatedStorageLocationBase.setType(templatedStorageLocationType);
    templatedStorageLocationBase.setValue(templatedStorageLocationValue);
    CloudStorageCdpService storageLocationType = CloudStorageCdpService.FLINK_JOBMANAGER_ARCHIVE;
    String eStorageLocationValue = "s3a://some-awesome-dir/some-other-awesome-dir/" + CLUSTER_NAME;
    StorageLocationBase storageLocationBase = new StorageLocationBase();
    storageLocationBase.setType(storageLocationType);
    storageLocationBase.setValue(eStorageLocationValue);
    List<StorageLocationBase> storageLocations = new ArrayList<>(2);
    storageLocations.add(storageLocationBase);
    storageLocations.add(templatedStorageLocationBase);
    ConfigQueryEntry cqe = new ConfigQueryEntry();
    cqe.setType(CloudStorageCdpService.DEFAULT_FS);
    cqe.setDefaultPath(templatedStorageLocationBaseValue + CLUSTER_NAME);
    CloudStorageRequest request = new CloudStorageRequest();
    request.setLocations(storageLocations);
    SdxClusterResponse sdxReponse = new SdxClusterResponse();
    String storageLocationValue = eStorageLocationValue;
    sdxReponse.setCloudStorageBaseLocation(storageLocationValue);
    sdxReponse.setCloudStorageFileSystemType(FileSystemType.S3);
    ConfigQueryEntry cqeFlink = new ConfigQueryEntry();
    cqeFlink.setType(CloudStorageCdpService.FLINK_JOBMANAGER_ARCHIVE);
    cqeFlink.setDefaultPath(storageLocationValue);
    Set<ConfigQueryEntry> cqes = new LinkedHashSet<>(1);
    cqes.add(cqe);
    cqes.add(cqeFlink);
    when(blueprintService.queryFileSystemParameters(BLUEPRINT_NAME, CLUSTER_NAME, storageLocationValue, FileSystemType.S3.name(), "", true, false, 0L)).thenReturn(cqes);
    Pair<Blueprint, String> mockBt = mock(Pair.class);
    when(blueprintService.getBlueprintAndText(BLUEPRINT_NAME, 0L)).thenReturn(mockBt);
    FileSystemConfigQueryObject mockfscqo = mock(FileSystemConfigQueryObject.class);
    when(blueprintService.createFileSystemConfigQueryObject(mockBt, CLUSTER_NAME, sdxReponse.getCloudStorageBaseLocation(), sdxReponse.getCloudStorageFileSystemType().name(), "", true, false)).thenReturn(mockfscqo);
    when(cmCloudStorageConfigProvider.queryParameters(any(), eq(mockfscqo))).thenReturn(cqes);
    CloudStorageRequest result = underTest.updateCloudStorageLocations(BLUEPRINT_NAME, CLUSTER_NAME, request, List.of(sdxReponse));
    assertNotNull(result);
    assertEquals(0, result.getLocations().stream().filter(slb -> slb.getValue().contains("{{{") && slb.getValue().contains("}}}")).count());
}

Example 9 with FileSystemConfigQueryObject

use of com.sequenceiq.cloudbreak.template.filesystem.FileSystemConfigQueryObject in project cloudbreak by hortonworks.

the class BlueprintService method queryFileSystemParameters.

public Set<ConfigQueryEntry> queryFileSystemParameters(String blueprintName, String clusterName, String baseLocation, String fileSystemType, String accountName, boolean attachedCluster, boolean secure, Long workspaceId) {
    Set<ConfigQueryEntry> result = new HashSet<>();
    Pair<Blueprint, String> bp = getBlueprintAndText(blueprintName, workspaceId);
    if (blueprintUtils.isClouderaManagerClusterTemplate(bp.getRight())) {
        FileSystemConfigQueryObject fileSystemConfigQueryObject = createFileSystemConfigQueryObject(bp, clusterName, baseLocation, fileSystemType, accountName, attachedCluster, secure);
        result = cmCloudStorageConfigProvider.queryParameters(fileSystemConfigQueryObject);
    }
    return result;
}

Example 10 with FileSystemConfigQueryObject

use of com.sequenceiq.cloudbreak.template.filesystem.FileSystemConfigQueryObject in project cloudbreak by hortonworks.

the class CmCloudStorageConfigDetailsTest method testDatalakeClusterPaths.

@Test
public void testDatalakeClusterPaths() {
    prepareBlueprintProcessorFactoryMock(RANGER_ADMIN, RESOURCEMANAGER, NAMENODE);
    FileSystemConfigQueryObject fileSystemConfigQueryObject = FileSystemConfigQueryObject.Builder.builder().withStorageName(STORAGE_NAME).withClusterName(CLUSTER_NAME).withBlueprintText(BLUEPRINT_TEXT).withDatalakeCluster(true).withFileSystemType(FileSystemType.S3.name()).build();
    Set<ConfigQueryEntry> bigCluster = underTest.queryParameters(fileSystemConfigQueryObject);
    Assert.assertEquals(2L, bigCluster.size());
    Set<ConfigQueryEntry> rangerAdmin = serviceEntry(bigCluster, RANGER_ADMIN);
    Set<ConfigQueryEntry> yarn = serviceEntry(bigCluster, RESOURCEMANAGER);
    Assert.assertEquals(1, rangerAdmin.size());
    Assert.assertTrue(rangerAdmin.stream().map(ConfigQueryEntry::getDefaultPath).anyMatch("hwx-remote/ranger/audit"::equals));
    Assert.assertEquals(1, yarn.size());
    Assert.assertTrue(yarn.stream().map(ConfigQueryEntry::getDefaultPath).anyMatch("hwx-remote/oplogs/yarn-app-logs"::equals));
}