Examples with S3FileSystem com.sequenceiq.common.api.filesystem.S3FileSystem used on opensource projects

Example 6 with S3FileSystem

use of com.sequenceiq.common.api.filesystem.S3FileSystem in project cloudbreak by hortonworks.

the class TemplateCoreTestUtil method s3FileSystem.

public static S3FileSystem s3FileSystem() {
    S3FileSystem s3FileSystem = new S3FileSystem();
    s3FileSystem.setInstanceProfile("instanceprofile");
    return s3FileSystem;
}

Example 7 with S3FileSystem

use of com.sequenceiq.common.api.filesystem.S3FileSystem in project cloudbreak by hortonworks.

the class HiveLLAPServiceConfigProviderTest method getTemplatePreparationObject.

private TemplatePreparationObject getTemplatePreparationObject(boolean includeLocations) {
    HostgroupView master = new HostgroupView("master", 1, InstanceGroupType.GATEWAY, 1);
    HostgroupView worker = new HostgroupView("worker", 2, InstanceGroupType.CORE, 2);
    List<StorageLocationView> locations = new ArrayList<>();
    if (includeLocations) {
        StorageLocation hmsExternalWarehouseDir = new StorageLocation();
        hmsExternalWarehouseDir.setProperty("hive.metastore.warehouse.external.dir");
        hmsExternalWarehouseDir.setValue("s3a://bucket/hive/warehouse/external");
        locations.add(new StorageLocationView(hmsExternalWarehouseDir));
    }
    S3FileSystemConfigurationsView fileSystemConfigurationsView = new S3FileSystemConfigurationsView(new S3FileSystem(), locations, false);
    return Builder.builder().withFileSystemConfigurationView(fileSystemConfigurationsView).withHostgroupViews(Set.of(master, worker)).build();
}

Example 8 with S3FileSystem

use of com.sequenceiq.common.api.filesystem.S3FileSystem in project cloudbreak by hortonworks.

the class HiveMetastoreCloudStorageServiceConfigProviderTest method getTemplatePreparationObject.

private TemplatePreparationObject getTemplatePreparationObject(boolean includeLocations) {
    HostgroupView master = new HostgroupView("master", 1, InstanceGroupType.GATEWAY, 1);
    HostgroupView worker = new HostgroupView("worker", 2, InstanceGroupType.CORE, 2);
    List<StorageLocationView> locations = new ArrayList<>();
    if (includeLocations) {
        locations.add(new StorageLocationView(getHiveWarehouseStorageLocation()));
        locations.add(new StorageLocationView(getHiveWarehouseExternalStorageLocation()));
        locations.add(new StorageLocationView(getHiveWarehouseReplicaStorageLocation()));
    }
    S3FileSystemConfigurationsView fileSystemConfigurationsView = new S3FileSystemConfigurationsView(new S3FileSystem(), locations, false);
    return Builder.builder().withFileSystemConfigurationView(fileSystemConfigurationsView).withHostgroupViews(Set.of(master, worker)).build();
}

Example 9 with S3FileSystem

use of com.sequenceiq.common.api.filesystem.S3FileSystem in project cloudbreak by hortonworks.

the class CentralCmTemplateUpdaterTest method getKafkaPropertiesWhenNoHdfsInClusterShouldPresentCoreSettings.

@Test
public void getKafkaPropertiesWhenNoHdfsInClusterShouldPresentCoreSettings() {
    CoreConfigProvider coreConfigProvider = new CoreConfigProvider();
    ReflectionTestUtils.setField(coreConfigProvider, "s3ConfigProvider", s3ConfigProvider);
    List<CmTemplateComponentConfigProvider> cmTemplateComponentConfigProviders = List.of(coreConfigProvider);
    ReflectionTestUtils.setField(cmTemplateComponentConfigProviderProcessor, "providers", cmTemplateComponentConfigProviders);
    S3FileSystem s3FileSystem = new S3FileSystem();
    s3FileSystem.setInstanceProfile("profile");
    s3FileSystem.setS3GuardDynamoTableName("cb-table");
    s3FileSystem.setStorageContainer("cloudbreak-bucket");
    StorageLocation storageLocation = new StorageLocation();
    storageLocation.setProperty("core_defaultfs");
    storageLocation.setValue("s3a://cloudbreak-bucket/kafka");
    storageLocation.setConfigFile("core_settings");
    StorageLocationView storageLocationView = new StorageLocationView(storageLocation);
    BaseFileSystemConfigurationsView baseFileSystemConfigurationsView = new S3FileSystemConfigurationsView(s3FileSystem, Sets.newHashSet(storageLocationView), false);
    when(templatePreparationObject.getFileSystemConfigurationView()).thenReturn(Optional.of(baseFileSystemConfigurationsView));
    when(templatePreparationObject.getGatewayView()).thenReturn(new GatewayView(new Gateway(), "signkey", new HashSet<>()));
    Set<HostgroupView> hostgroupViews = new HashSet<>();
    hostgroupViews.add(new HostgroupView("master", 1, InstanceGroupType.GATEWAY, 1));
    when(templatePreparationObject.getHostgroupViews()).thenReturn(hostgroupViews);
    when(templatePreparationObject.getGatewayView()).thenReturn(new GatewayView(new Gateway(), "signkey", new HashSet<>()));
    when(blueprintView.getBlueprintText()).thenReturn(getBlueprintText("input/kafka-without-hdfs.bp"));
    String generated = generator.getBlueprintText(templatePreparationObject);
    String expected = new CmTemplateProcessor(getBlueprintText("output/kafka-without-hdfs.bp")).getTemplate().toString();
    String output = new CmTemplateProcessor(generated).getTemplate().toString();
    Assert.assertEquals(expected, output);
}

Example 10 with S3FileSystem

use of com.sequenceiq.common.api.filesystem.S3FileSystem in project cloudbreak by hortonworks.

the class FileSystemConverter method legacyConvertFromConfiguration.

private List<CloudFileSystemView> legacyConvertFromConfiguration(FileSystem source) {
    try {
        CloudFileSystemView fileSystemView;
        if (source.getType().isAdls()) {
            AdlsFileSystem adlsFileSystem = source.getConfigurations().get(AdlsFileSystem.class);
            fileSystemView = convertAdlsLegacy(adlsFileSystem);
        } else if (source.getType().isGcs()) {
            GcsFileSystem gcsFileSystem = source.getConfigurations().get(GcsFileSystem.class);
            fileSystemView = convertGcsLegacy(gcsFileSystem);
        } else if (source.getType().isS3()) {
            S3FileSystem s3FileSystem = source.getConfigurations().get(S3FileSystem.class);
            fileSystemView = convertS3Legacy(s3FileSystem);
        } else if (source.getType().isEfs()) {
            EfsFileSystem efsFileSystem = source.getConfigurations().get(EfsFileSystem.class);
            fileSystemView = convertEfsLegacy(efsFileSystem);
        } else if (source.getType().isWasb()) {
            WasbFileSystem wasbFileSystem = source.getConfigurations().get(WasbFileSystem.class);
            fileSystemView = convertWasbLegacy(wasbFileSystem);
        } else if (source.getType().isAdlsGen2()) {
            AdlsGen2FileSystem adlsGen2FileSystem = source.getConfigurations().get(AdlsGen2FileSystem.class);
            fileSystemView = convertAdlsGen2Legacy(adlsGen2FileSystem);
        } else if (source.getType().isGcs()) {
            GcsFileSystem gcsFileSystem = source.getConfigurations().get(GcsFileSystem.class);
            fileSystemView = convertGcsLegacy(gcsFileSystem);
        } else {
            return Collections.emptyList();
        }
        return List.of(fileSystemView);
    } catch (IOException e) {
        LOGGER.warn("Error occurred when tried to convert filesystem object: {}", e.getMessage());
    }
    return Collections.emptyList();
}