Examples with S3FileSystem com.sequenceiq.common.api.filesystem.S3FileSystem used on opensource projects

Example 21 with S3FileSystem

use of com.sequenceiq.common.api.filesystem.S3FileSystem in project cloudbreak by hortonworks.

the class ProfilerSchedulerCloudStorageRoleConfigProviderTest method getTemplatePreparationObject.

private TemplatePreparationObject getTemplatePreparationObject(boolean includeLocations) {
    HostgroupView master = new HostgroupView("master", 1, InstanceGroupType.GATEWAY, 1);
    HostgroupView worker = new HostgroupView("worker", 2, InstanceGroupType.CORE, 2);
    List<StorageLocationView> locations = new ArrayList<>();
    if (includeLocations) {
        locations.add(new StorageLocationView(getProfilerSchedulerFileSystemUri()));
    }
    S3FileSystemConfigurationsView fileSystemConfigurationsView = new S3FileSystemConfigurationsView(new S3FileSystem(), locations, false);
    return Builder.builder().withFileSystemConfigurationView(fileSystemConfigurationsView).withHostgroupViews(Set.of(master, worker)).build();
}

Example 22 with S3FileSystem

use of com.sequenceiq.common.api.filesystem.S3FileSystem in project cloudbreak by hortonworks.

the class RangerCloudStorageServiceConfigProviderTest method getTemplatePreparationObjectForAws.

private TemplatePreparationObject.Builder getTemplatePreparationObjectForAws(boolean includeLocations, boolean above721, boolean includeBackup) {
    HostgroupView gateway = new HostgroupView("gateway", 1, InstanceGroupType.GATEWAY, Set.of("g"));
    HostgroupView master = new HostgroupView("master", 1, InstanceGroupType.CORE, Set.of("m1", "m2"));
    HostgroupView worker = new HostgroupView("worker", 2, InstanceGroupType.CORE, Set.of("w1", "w2", "w3"));
    List<StorageLocationView> locations = new ArrayList<>();
    if (includeLocations) {
        locations.add(new StorageLocationView(getRangerAuditCloudStorageDirAws()));
        if (above721) {
            locations.add(new StorageLocationView(buildStorageLocation("hive.metastore.warehouse.dir", "s3a://bucket/warehouse/tablespace/managed/hive")));
            locations.add(new StorageLocationView(buildStorageLocation("hive.repl.replica.functions.root.dir", "s3a://bucket/hive_replica_functions_dir")));
            locations.add(new StorageLocationView(buildStorageLocation("hive.metastore.warehouse.external.dir", "s3a://bucket/warehouse/tablespace/external/hive")));
            locations.add(new StorageLocationView(buildStorageLocation("hbase.rootdir", "s3a://bucket/hbase")));
            if (includeBackup) {
                locations.add(new StorageLocationView(buildStorageLocation(DEFAULT_BACKUP_DIR, "s3a://bucket/backup/location")));
            }
        }
    }
    S3FileSystemConfigurationsView fileSystemConfigurationsView = new S3FileSystemConfigurationsView(new S3FileSystem(), locations, false);
    GeneralClusterConfigs generalClusterConfigs = new GeneralClusterConfigs();
    generalClusterConfigs.setPrimaryGatewayInstanceDiscoveryFQDN(Optional.of("fqdn"));
    return Builder.builder().withFileSystemConfigurationView(fileSystemConfigurationsView).withHostgroupViews(Set.of(gateway, master, worker)).withGeneralClusterConfigs(generalClusterConfigs);
}

Example 23 with S3FileSystem

use of com.sequenceiq.common.api.filesystem.S3FileSystem in project cloudbreak by hortonworks.

the class CentralCmTemplateUpdaterTest method getCmTemplateIfCoresettingspresentedShouldNotOverrideThePropertyWithEmptyString.

@Test
public void getCmTemplateIfCoresettingspresentedShouldNotOverrideThePropertyWithEmptyString() {
    List<StorageLocationView> locations = new ArrayList<>();
    StorageLocation hbaseRootDir = new StorageLocation();
    hbaseRootDir.setProperty("hbase.rootdir");
    hbaseRootDir.setValue("s3a://bucket/cluster1/hbase");
    locations.add(new StorageLocationView(hbaseRootDir));
    StorageLocation coresettings = new StorageLocation();
    coresettings.setProperty("core_defaultfs");
    coresettings.setValue("s3a://bucket/cluster1/hbase");
    locations.add(new StorageLocationView(coresettings));
    S3FileSystemConfigurationsView fileSystemConfigurationsView = new S3FileSystemConfigurationsView(new S3FileSystem(), locations, false);
    when(templatePreparationObject.getFileSystemConfigurationView()).thenReturn(Optional.of(fileSystemConfigurationsView));
    when(blueprintView.getBlueprintText()).thenReturn(getBlueprintText("input/core-settings-empty.bp"));
    ApiClusterTemplate generated = testGetCmTemplate();
    assertMatchesBlueprintAtPath("output/core-settings-empty.bp", generated);
}

Example 24 with S3FileSystem

use of com.sequenceiq.common.api.filesystem.S3FileSystem in project cloudbreak by hortonworks.

the class CentralCmTemplateUpdaterTest method setUp.

@Before
public void setUp() {
    when(entitlementService.sdxHbaseCloudStorageEnabled(anyString())).thenReturn(true);
    List<CmTemplateComponentConfigProvider> cmTemplateComponentConfigProviders = List.of(new HiveMetastoreConfigProvider(), coreConfigProvider, hbaseCloudStorageProvider);
    when(cmTemplateProcessorFactory.get(anyString())).thenAnswer(i -> new CmTemplateProcessor(i.getArgument(0)));
    when(templatePreparationObject.getBlueprintView()).thenReturn(blueprintView);
    when(templatePreparationObject.getHostgroupViews()).thenReturn(toHostgroupViews(getHostgroupMappings()));
    when(templatePreparationObject.getGeneralClusterConfigs()).thenReturn(generalClusterConfigs);
    doNothing().when(s3ConfigProvider).getServiceConfigs(any(TemplatePreparationObject.class), any(StringBuilder.class));
    RDSConfig rdsConfig = TestUtil.rdsConfig(DatabaseType.HIVE);
    when(templatePreparationObject.getRdsConfigs()).thenReturn(Set.of(rdsConfig));
    when(templatePreparationObject.getRdsConfig(DatabaseType.HIVE)).thenReturn(rdsConfig);
    when(templatePreparationObject.getCustomConfigurationsView()).thenReturn(Optional.of(customConfigurationsView));
    List<StorageLocationView> locations = new ArrayList<>();
    StorageLocation hbaseRootDir = new StorageLocation();
    hbaseRootDir.setProperty("hbase.rootdir");
    hbaseRootDir.setValue("s3a://bucket/cluster1/hbase");
    locations.add(new StorageLocationView(hbaseRootDir));
    S3FileSystemConfigurationsView fileSystemConfigurationsView = new S3FileSystemConfigurationsView(new S3FileSystem(), locations, false);
    when(templatePreparationObject.getFileSystemConfigurationView()).thenReturn(Optional.of(fileSystemConfigurationsView));
    when(generalClusterConfigs.getClusterName()).thenReturn("testcluster");
    when(generalClusterConfigs.getPassword()).thenReturn("Admin123!");
    when(generalClusterConfigs.getAccountId()).thenReturn(Optional.of("1234"));
    clouderaManagerRepo = new ClouderaManagerRepo();
    clouderaManagerRepo.setVersion("6.1.0");
    ProductDetailsView productDetailsView = new ProductDetailsView(clouderaManagerRepo, List.of());
    when(templatePreparationObject.getProductDetailsView()).thenReturn(productDetailsView);
    ReflectionTestUtils.setField(cmTemplateComponentConfigProviderProcessor, "providers", cmTemplateComponentConfigProviders);
    ReflectionTestUtils.setField(cmTemplateConfigInjectorProcessor, "injectors", List.of());
    ReflectionTestUtils.setField(cmHostGroupRoleConfigProviderProcessor, "providers", List.of());
}

Example 25 with S3FileSystem

use of com.sequenceiq.common.api.filesystem.S3FileSystem in project cloudbreak by hortonworks.

the class CentralCmTemplateUpdaterTest method getCmTemplateIfCoresettingspresentedShouldNotOverrideTheProperty.

@Test
public void getCmTemplateIfCoresettingspresentedShouldNotOverrideTheProperty() {
    List<StorageLocationView> locations = new ArrayList<>();
    StorageLocation hbaseRootDir = new StorageLocation();
    hbaseRootDir.setProperty("hbase.rootdir");
    hbaseRootDir.setValue("s3a://bucket/cluster1/hbase");
    locations.add(new StorageLocationView(hbaseRootDir));
    StorageLocation coresettings = new StorageLocation();
    coresettings.setProperty("core_defaultfs");
    coresettings.setValue("s3a://bucket/cluster1/hbase");
    locations.add(new StorageLocationView(coresettings));
    S3FileSystemConfigurationsView fileSystemConfigurationsView = new S3FileSystemConfigurationsView(new S3FileSystem(), locations, false);
    when(templatePreparationObject.getFileSystemConfigurationView()).thenReturn(Optional.of(fileSystemConfigurationsView));
    when(blueprintView.getBlueprintText()).thenReturn(getBlueprintText("input/core-settings.bp"));
    ApiClusterTemplate generated = testGetCmTemplate();
    assertMatchesBlueprintAtPath("output/core-settings.bp", generated);
}