Examples with StorageLocationView com.sequenceiq.cloudbreak.template.filesystem.StorageLocationView used on opensource projects

Example 11 with StorageLocationView

use of com.sequenceiq.cloudbreak.template.filesystem.StorageLocationView in project cloudbreak by hortonworks.

the class CentralCmTemplateUpdaterTest method getKafkaPropertiesWhenNoHdfsInClusterShouldPresentCoreSettings.

@Test
public void getKafkaPropertiesWhenNoHdfsInClusterShouldPresentCoreSettings() {
    CoreConfigProvider coreConfigProvider = new CoreConfigProvider();
    ReflectionTestUtils.setField(coreConfigProvider, "s3ConfigProvider", s3ConfigProvider);
    List<CmTemplateComponentConfigProvider> cmTemplateComponentConfigProviders = List.of(coreConfigProvider);
    ReflectionTestUtils.setField(cmTemplateComponentConfigProviderProcessor, "providers", cmTemplateComponentConfigProviders);
    S3FileSystem s3FileSystem = new S3FileSystem();
    s3FileSystem.setInstanceProfile("profile");
    s3FileSystem.setS3GuardDynamoTableName("cb-table");
    s3FileSystem.setStorageContainer("cloudbreak-bucket");
    StorageLocation storageLocation = new StorageLocation();
    storageLocation.setProperty("core_defaultfs");
    storageLocation.setValue("s3a://cloudbreak-bucket/kafka");
    storageLocation.setConfigFile("core_settings");
    StorageLocationView storageLocationView = new StorageLocationView(storageLocation);
    BaseFileSystemConfigurationsView baseFileSystemConfigurationsView = new S3FileSystemConfigurationsView(s3FileSystem, Sets.newHashSet(storageLocationView), false);
    when(templatePreparationObject.getFileSystemConfigurationView()).thenReturn(Optional.of(baseFileSystemConfigurationsView));
    when(templatePreparationObject.getGatewayView()).thenReturn(new GatewayView(new Gateway(), "signkey", new HashSet<>()));
    Set<HostgroupView> hostgroupViews = new HashSet<>();
    hostgroupViews.add(new HostgroupView("master", 1, InstanceGroupType.GATEWAY, 1));
    when(templatePreparationObject.getHostgroupViews()).thenReturn(hostgroupViews);
    when(templatePreparationObject.getGatewayView()).thenReturn(new GatewayView(new Gateway(), "signkey", new HashSet<>()));
    when(blueprintView.getBlueprintText()).thenReturn(getBlueprintText("input/kafka-without-hdfs.bp"));
    String generated = generator.getBlueprintText(templatePreparationObject);
    String expected = new CmTemplateProcessor(getBlueprintText("output/kafka-without-hdfs.bp")).getTemplate().toString();
    String output = new CmTemplateProcessor(generated).getTemplate().toString();
    Assert.assertEquals(expected, output);
}

Example 12 with StorageLocationView

use of com.sequenceiq.cloudbreak.template.filesystem.StorageLocationView in project cloudbreak by hortonworks.

the class StackToTemplatePreparationObjectConverter method updateFileSystemViewWithBackupLocation.

private void updateFileSystemViewWithBackupLocation(DetailedEnvironmentResponse detailedEnvironmentResponse, BaseFileSystemConfigurationsView fileSystemConfigurationView) {
    if (fileSystemConfigurationView != null) {
        BackupResponse backupResponse = detailedEnvironmentResponse.getBackup();
        TelemetryResponse telemetryResponse = detailedEnvironmentResponse.getTelemetry();
        Optional<String> backupLocation = Optional.empty();
        if (backupResponse != null && backupResponse.getStorageLocation() != null) {
            backupLocation = Optional.of(backupResponse.getStorageLocation());
        } else if (telemetryResponse != null && telemetryResponse.getLogging() != null) {
            backupLocation = Optional.of(telemetryResponse.getLogging().getStorageLocation());
        }
        if (backupLocation.isPresent()) {
            StorageLocation storageLocation = new StorageLocation();
            storageLocation.setValue(backupLocation.get());
            storageLocation.setProperty(RangerCloudStorageServiceConfigProvider.DEFAULT_BACKUP_DIR);
            StorageLocationView backupLocationView = new StorageLocationView(storageLocation);
            fileSystemConfigurationView.getLocations().add(backupLocationView);
        }
    }
}

Example 13 with StorageLocationView

use of com.sequenceiq.cloudbreak.template.filesystem.StorageLocationView in project cloudbreak by hortonworks.

the class Spark3OnYarnRoleConfigProviderTest method getTemplatePreparationObject.

private TemplatePreparationObject getTemplatePreparationObject(String... locations) {
    HostgroupView master = new HostgroupView("master", 1, InstanceGroupType.GATEWAY, 1);
    HostgroupView worker = new HostgroupView("worker", 2, InstanceGroupType.CORE, 2);
    List<StorageLocationView> storageLocations = new ArrayList<>();
    if (locations.length >= 1) {
        StorageLocation hmsExternalWarehouseDir = new StorageLocation();
        hmsExternalWarehouseDir.setProperty("hive.metastore.warehouse.external.dir");
        hmsExternalWarehouseDir.setValue(locations[0]);
        storageLocations.add(new StorageLocationView(hmsExternalWarehouseDir));
    }
    S3FileSystemConfigurationsView fileSystemConfigurationsView = new S3FileSystemConfigurationsView(new S3FileSystem(), storageLocations, false);
    return Builder.builder().withFileSystemConfigurationView(fileSystemConfigurationsView).withHostgroupViews(Set.of(master, worker)).build();
}

Example 14 with StorageLocationView

use of com.sequenceiq.cloudbreak.template.filesystem.StorageLocationView in project cloudbreak by hortonworks.

the class TezRoleConfigProviderTest method getTemplatePreparationObject.

private TemplatePreparationObject getTemplatePreparationObject(String... locations) {
    HostgroupView master = new HostgroupView("master", 1, InstanceGroupType.GATEWAY, 1);
    HostgroupView worker = new HostgroupView("worker", 2, InstanceGroupType.CORE, 2);
    List<StorageLocationView> storageLocations = new ArrayList<>();
    if (locations.length >= 1) {
        StorageLocation hmsExternalWarehouseDir = new StorageLocation();
        hmsExternalWarehouseDir.setProperty("hive.metastore.warehouse.external.dir");
        hmsExternalWarehouseDir.setValue(locations[0]);
        storageLocations.add(new StorageLocationView(hmsExternalWarehouseDir));
    }
    S3FileSystemConfigurationsView fileSystemConfigurationsView = new S3FileSystemConfigurationsView(new S3FileSystem(), storageLocations, false);
    return Builder.builder().withFileSystemConfigurationView(fileSystemConfigurationsView).withHostgroupViews(Set.of(master, worker)).build();
}

Example 15 with StorageLocationView

use of com.sequenceiq.cloudbreak.template.filesystem.StorageLocationView in project cloudbreak by hortonworks.

the class ZeppelinCloudStorageRoleConfigProviderTest method assertZeppelinStorageValues.

protected void assertZeppelinStorageValues(String storagePath) {
    List<StorageLocationView> locations = new ArrayList<>();
    StorageLocation zeppelinNotebookDir = new StorageLocation();
    zeppelinNotebookDir.setProperty("zeppelin.notebook.dir");
    zeppelinNotebookDir.setValue(storagePath);
    locations.add(new StorageLocationView(zeppelinNotebookDir));
    BaseFileSystemConfigurationsView fileSystemConfigurationsView;
    if (storagePath.startsWith("s3a")) {
        fileSystemConfigurationsView = new S3FileSystemConfigurationsView(new S3FileSystem(), locations, false);
    } else if (storagePath.startsWith("gcs")) {
        fileSystemConfigurationsView = new GcsFileSystemConfigurationsView(new GcsFileSystem(), locations, false);
    } else {
        fileSystemConfigurationsView = new AdlsGen2FileSystemConfigurationsView(new AdlsGen2FileSystem(), locations, false);
    }
    TemplatePreparationObject preparationObject = getTemplatePreparationObject(fileSystemConfigurationsView);
    String inputJson = getBlueprintText("input/clouderamanager-ds.bp");
    CmTemplateProcessor cmTemplateProcessor = new CmTemplateProcessor(inputJson);
    Map<String, List<ApiClusterTemplateConfig>> roleConfigs = underTest.getRoleConfigs(cmTemplateProcessor, preparationObject);
    List<ApiClusterTemplateConfig> zeppelinStorageConfigs = roleConfigs.get("zeppelin-ZEPPELIN_SERVER-BASE");
    assertEquals(1, zeppelinStorageConfigs.size());
    assertEquals("zeppelin.notebook.dir", zeppelinStorageConfigs.get(0).getName());
    assertEquals(storagePath, zeppelinStorageConfigs.get(0).getValue());
}