Examples with S3FileSystemConfigurationsView com.sequenceiq.cloudbreak.template.filesystem.s3.S3FileSystemConfigurationsView used on opensource projects

Example 21 with S3FileSystemConfigurationsView

use of com.sequenceiq.cloudbreak.template.filesystem.s3.S3FileSystemConfigurationsView in project cloudbreak by hortonworks.

the class RangerCloudStorageServiceConfigProviderTest method getTemplatePreparationObjectForAws.

private TemplatePreparationObject.Builder getTemplatePreparationObjectForAws(boolean includeLocations, boolean above721, boolean includeBackup) {
    HostgroupView gateway = new HostgroupView("gateway", 1, InstanceGroupType.GATEWAY, Set.of("g"));
    HostgroupView master = new HostgroupView("master", 1, InstanceGroupType.CORE, Set.of("m1", "m2"));
    HostgroupView worker = new HostgroupView("worker", 2, InstanceGroupType.CORE, Set.of("w1", "w2", "w3"));
    List<StorageLocationView> locations = new ArrayList<>();
    if (includeLocations) {
        locations.add(new StorageLocationView(getRangerAuditCloudStorageDirAws()));
        if (above721) {
            locations.add(new StorageLocationView(buildStorageLocation("hive.metastore.warehouse.dir", "s3a://bucket/warehouse/tablespace/managed/hive")));
            locations.add(new StorageLocationView(buildStorageLocation("hive.repl.replica.functions.root.dir", "s3a://bucket/hive_replica_functions_dir")));
            locations.add(new StorageLocationView(buildStorageLocation("hive.metastore.warehouse.external.dir", "s3a://bucket/warehouse/tablespace/external/hive")));
            locations.add(new StorageLocationView(buildStorageLocation("hbase.rootdir", "s3a://bucket/hbase")));
            if (includeBackup) {
                locations.add(new StorageLocationView(buildStorageLocation(DEFAULT_BACKUP_DIR, "s3a://bucket/backup/location")));
            }
        }
    }
    S3FileSystemConfigurationsView fileSystemConfigurationsView = new S3FileSystemConfigurationsView(new S3FileSystem(), locations, false);
    GeneralClusterConfigs generalClusterConfigs = new GeneralClusterConfigs();
    generalClusterConfigs.setPrimaryGatewayInstanceDiscoveryFQDN(Optional.of("fqdn"));
    return Builder.builder().withFileSystemConfigurationView(fileSystemConfigurationsView).withHostgroupViews(Set.of(gateway, master, worker)).withGeneralClusterConfigs(generalClusterConfigs);
}

Example 22 with S3FileSystemConfigurationsView

use of com.sequenceiq.cloudbreak.template.filesystem.s3.S3FileSystemConfigurationsView in project cloudbreak by hortonworks.

the class CentralCmTemplateUpdaterTest method getCmTemplateIfCoresettingspresentedShouldNotOverrideThePropertyWithEmptyString.

@Test
public void getCmTemplateIfCoresettingspresentedShouldNotOverrideThePropertyWithEmptyString() {
    List<StorageLocationView> locations = new ArrayList<>();
    StorageLocation hbaseRootDir = new StorageLocation();
    hbaseRootDir.setProperty("hbase.rootdir");
    hbaseRootDir.setValue("s3a://bucket/cluster1/hbase");
    locations.add(new StorageLocationView(hbaseRootDir));
    StorageLocation coresettings = new StorageLocation();
    coresettings.setProperty("core_defaultfs");
    coresettings.setValue("s3a://bucket/cluster1/hbase");
    locations.add(new StorageLocationView(coresettings));
    S3FileSystemConfigurationsView fileSystemConfigurationsView = new S3FileSystemConfigurationsView(new S3FileSystem(), locations, false);
    when(templatePreparationObject.getFileSystemConfigurationView()).thenReturn(Optional.of(fileSystemConfigurationsView));
    when(blueprintView.getBlueprintText()).thenReturn(getBlueprintText("input/core-settings-empty.bp"));
    ApiClusterTemplate generated = testGetCmTemplate();
    assertMatchesBlueprintAtPath("output/core-settings-empty.bp", generated);
}

Example 23 with S3FileSystemConfigurationsView

use of com.sequenceiq.cloudbreak.template.filesystem.s3.S3FileSystemConfigurationsView in project cloudbreak by hortonworks.

the class CentralCmTemplateUpdaterTest method setUp.

@Before
public void setUp() {
    when(entitlementService.sdxHbaseCloudStorageEnabled(anyString())).thenReturn(true);
    List<CmTemplateComponentConfigProvider> cmTemplateComponentConfigProviders = List.of(new HiveMetastoreConfigProvider(), coreConfigProvider, hbaseCloudStorageProvider);
    when(cmTemplateProcessorFactory.get(anyString())).thenAnswer(i -> new CmTemplateProcessor(i.getArgument(0)));
    when(templatePreparationObject.getBlueprintView()).thenReturn(blueprintView);
    when(templatePreparationObject.getHostgroupViews()).thenReturn(toHostgroupViews(getHostgroupMappings()));
    when(templatePreparationObject.getGeneralClusterConfigs()).thenReturn(generalClusterConfigs);
    doNothing().when(s3ConfigProvider).getServiceConfigs(any(TemplatePreparationObject.class), any(StringBuilder.class));
    RDSConfig rdsConfig = TestUtil.rdsConfig(DatabaseType.HIVE);
    when(templatePreparationObject.getRdsConfigs()).thenReturn(Set.of(rdsConfig));
    when(templatePreparationObject.getRdsConfig(DatabaseType.HIVE)).thenReturn(rdsConfig);
    when(templatePreparationObject.getCustomConfigurationsView()).thenReturn(Optional.of(customConfigurationsView));
    List<StorageLocationView> locations = new ArrayList<>();
    StorageLocation hbaseRootDir = new StorageLocation();
    hbaseRootDir.setProperty("hbase.rootdir");
    hbaseRootDir.setValue("s3a://bucket/cluster1/hbase");
    locations.add(new StorageLocationView(hbaseRootDir));
    S3FileSystemConfigurationsView fileSystemConfigurationsView = new S3FileSystemConfigurationsView(new S3FileSystem(), locations, false);
    when(templatePreparationObject.getFileSystemConfigurationView()).thenReturn(Optional.of(fileSystemConfigurationsView));
    when(generalClusterConfigs.getClusterName()).thenReturn("testcluster");
    when(generalClusterConfigs.getPassword()).thenReturn("Admin123!");
    when(generalClusterConfigs.getAccountId()).thenReturn(Optional.of("1234"));
    clouderaManagerRepo = new ClouderaManagerRepo();
    clouderaManagerRepo.setVersion("6.1.0");
    ProductDetailsView productDetailsView = new ProductDetailsView(clouderaManagerRepo, List.of());
    when(templatePreparationObject.getProductDetailsView()).thenReturn(productDetailsView);
    ReflectionTestUtils.setField(cmTemplateComponentConfigProviderProcessor, "providers", cmTemplateComponentConfigProviders);
    ReflectionTestUtils.setField(cmTemplateConfigInjectorProcessor, "injectors", List.of());
    ReflectionTestUtils.setField(cmHostGroupRoleConfigProviderProcessor, "providers", List.of());
}

Example 24 with S3FileSystemConfigurationsView

use of com.sequenceiq.cloudbreak.template.filesystem.s3.S3FileSystemConfigurationsView in project cloudbreak by hortonworks.

the class CentralCmTemplateUpdaterTest method getCmTemplateIfCoresettingspresentedShouldNotOverrideTheProperty.

@Test
public void getCmTemplateIfCoresettingspresentedShouldNotOverrideTheProperty() {
    List<StorageLocationView> locations = new ArrayList<>();
    StorageLocation hbaseRootDir = new StorageLocation();
    hbaseRootDir.setProperty("hbase.rootdir");
    hbaseRootDir.setValue("s3a://bucket/cluster1/hbase");
    locations.add(new StorageLocationView(hbaseRootDir));
    StorageLocation coresettings = new StorageLocation();
    coresettings.setProperty("core_defaultfs");
    coresettings.setValue("s3a://bucket/cluster1/hbase");
    locations.add(new StorageLocationView(coresettings));
    S3FileSystemConfigurationsView fileSystemConfigurationsView = new S3FileSystemConfigurationsView(new S3FileSystem(), locations, false);
    when(templatePreparationObject.getFileSystemConfigurationView()).thenReturn(Optional.of(fileSystemConfigurationsView));
    when(blueprintView.getBlueprintText()).thenReturn(getBlueprintText("input/core-settings.bp"));
    ApiClusterTemplate generated = testGetCmTemplate();
    assertMatchesBlueprintAtPath("output/core-settings.bp", generated);
}

Example 25 with S3FileSystemConfigurationsView

use of com.sequenceiq.cloudbreak.template.filesystem.s3.S3FileSystemConfigurationsView in project cloudbreak by hortonworks.

the class S3ConfigProvider method configureS3GuardCoreSiteParameters.

private void configureS3GuardCoreSiteParameters(TemplatePreparationObject source, StringBuilder hdfsCoreSiteSafetyValveValue) {
    S3FileSystemConfigurationsView s3FileSystemConfigurationsView = (S3FileSystemConfigurationsView) source.getFileSystemConfigurationView().get();
    if (isS3GuardTableConfigured(s3FileSystemConfigurationsView)) {
        hdfsCoreSiteSafetyValveValue.append(ConfigUtils.getSafetyValveProperty(S3GUARD_METADATASTORE_IMPL_PARAM, S3GUARD_METADATASTORE_IMPL_VALUE));
        addTags(source, hdfsCoreSiteSafetyValveValue);
        hdfsCoreSiteSafetyValveValue.append(ConfigUtils.getSafetyValveProperty(S3GUARD_TABLE_CREATE_PARAM, S3GUARD_TABLE_CREATE_VALUE));
        hdfsCoreSiteSafetyValveValue.append(ConfigUtils.getSafetyValveProperty(S3GUARD_TABLE_NAME_PARAM, s3FileSystemConfigurationsView.getS3GuardDynamoTableName()));
        source.getPlacementView().ifPresent(placementView -> hdfsCoreSiteSafetyValveValue.append(ConfigUtils.getSafetyValveProperty(S3GUARD_TABLE_REGION_PARAM, placementView.getRegion())));
        ConfigUtils.getStorageLocationForServiceProperty(source, HMS_METASTORE_DIR).ifPresent(location -> hdfsCoreSiteSafetyValveValue.append(ConfigUtils.getSafetyValveProperty(S3GUARD_AUTHORITATIVE_PATH_PARAM, location.getValue())));
    }
}