Examples with S3FileSystem com.sequenceiq.common.api.filesystem.S3FileSystem used on opensource projects

Example 16 with S3FileSystem

use of com.sequenceiq.common.api.filesystem.S3FileSystem in project cloudbreak by hortonworks.

the class HiveOnTezServiceConfigProviderTest method getTemplatePreparationObject.

private TemplatePreparationObject getTemplatePreparationObject(boolean includeLocations) {
    HostgroupView master = new HostgroupView("master", 1, InstanceGroupType.GATEWAY, 1);
    HostgroupView worker = new HostgroupView("worker", 2, InstanceGroupType.CORE, 2);
    List<StorageLocationView> locations = new ArrayList<>();
    if (includeLocations) {
        StorageLocation hmsExternalWarehouseDir = new StorageLocation();
        hmsExternalWarehouseDir.setProperty("hive.metastore.warehouse.external.dir");
        hmsExternalWarehouseDir.setValue("s3a://bucket/hive/warehouse/external");
        locations.add(new StorageLocationView(hmsExternalWarehouseDir));
    }
    S3FileSystemConfigurationsView fileSystemConfigurationsView = new S3FileSystemConfigurationsView(new S3FileSystem(), locations, false);
    return Builder.builder().withFileSystemConfigurationView(fileSystemConfigurationsView).withHostgroupViews(Set.of(master, worker)).build();
}

Example 17 with S3FileSystem

use of com.sequenceiq.common.api.filesystem.S3FileSystem in project cloudbreak by hortonworks.

the class ProfilerAdminCloudStorageRoleConfigProviderTest method getTemplatePreparationObject.

private TemplatePreparationObject getTemplatePreparationObject(boolean includeLocations) {
    HostgroupView master = new HostgroupView("master", 1, InstanceGroupType.GATEWAY, 1);
    HostgroupView worker = new HostgroupView("worker", 2, InstanceGroupType.CORE, 2);
    List<StorageLocationView> locations = new ArrayList<>();
    if (includeLocations) {
        locations.add(new StorageLocationView(getProfilerAdminFileSystemUri()));
    }
    S3FileSystemConfigurationsView fileSystemConfigurationsView = new S3FileSystemConfigurationsView(new S3FileSystem(), locations, false);
    return Builder.builder().withFileSystemConfigurationView(fileSystemConfigurationsView).withHostgroupViews(Set.of(master, worker)).build();
}

Example 18 with S3FileSystem

use of com.sequenceiq.common.api.filesystem.S3FileSystem in project cloudbreak by hortonworks.

the class ImpalaCloudStorageServiceConfigProviderTest method getTemplatePreparationObject.

private TemplatePreparationObject getTemplatePreparationObject(boolean includeLocations) {
    HostgroupView master = new HostgroupView("master", 1, InstanceGroupType.GATEWAY, 1);
    HostgroupView coordinator = new HostgroupView("coordinator", 1, InstanceGroupType.CORE, 1);
    HostgroupView executor = new HostgroupView("executor", 2, InstanceGroupType.CORE, 2);
    List<StorageLocationView> locations = new ArrayList<>();
    if (includeLocations) {
        locations.add(new StorageLocationView(getHiveWarehouseStorageLocation()));
        locations.add(new StorageLocationView(getHiveWarehouseExternalStorageLocation()));
    }
    S3FileSystemConfigurationsView fileSystemConfigurationsView = new S3FileSystemConfigurationsView(new S3FileSystem(), locations, false);
    return Builder.builder().withFileSystemConfigurationView(fileSystemConfigurationsView).withHostgroupViews(Set.of(master, coordinator, executor)).build();
}

Example 19 with S3FileSystem

use of com.sequenceiq.common.api.filesystem.S3FileSystem in project cloudbreak by hortonworks.

the class FlinkCloudStorageConfigProviderTest method getTemplatePreparationObject.

private TemplatePreparationObject getTemplatePreparationObject(boolean includeLocations) {
    HostgroupView master = new HostgroupView("master", 1, InstanceGroupType.GATEWAY, 1);
    HostgroupView worker = new HostgroupView("worker", 2, InstanceGroupType.CORE, 2);
    List<StorageLocationView> locations = new ArrayList<>();
    if (includeLocations) {
        locations.add(getStorageLocationView(HISTORY_SERVER_ARCHIVE_FS_DIR_NAME, HISTORY_SERVER_ARCHIVE_FS_DIR_VALUE));
        locations.add(getStorageLocationView(JOBMANAGER_ARCHIVE_FS_DIR_NAME, JOBMANAGER_ARCHIVE_FS_DIR_VALUE));
        locations.add(getStorageLocationView(STATE_CHECKPOINTS_DIR_NAME, STATE_CHECKPOINTS_DIR_VALUE));
        locations.add(getStorageLocationView(STATE_SAVEPOINTS_DIR_NAME, STATE_SAVEPOINTS_DIR_VALUE));
        locations.add(getStorageLocationView(HIGH_AVAILABILITY_STORAGE_DIR_NAME, HIGH_AVAILABILITY_STORAGE_DIR_VALUE));
    }
    S3FileSystemConfigurationsView fileSystemConfigurationsView = new S3FileSystemConfigurationsView(new S3FileSystem(), locations, false);
    return Builder.builder().withFileSystemConfigurationView(fileSystemConfigurationsView).withHostgroupViews(Set.of(master, worker)).build();
}

Example 20 with S3FileSystem

use of com.sequenceiq.common.api.filesystem.S3FileSystem in project cloudbreak by hortonworks.

the class AbstractHbaseConfigProviderTest method getTemplatePreparationObject.

protected TemplatePreparationObject getTemplatePreparationObject(boolean includeLocations, boolean datalakeCluster, String cdhVersion, Map<String, String> defaultTags, CloudPlatform cloudPlatform) {
    HostgroupView master = new HostgroupView("master", 1, InstanceGroupType.GATEWAY, 1);
    HostgroupView worker = new HostgroupView("worker", 2, InstanceGroupType.CORE, 2);
    List<StorageLocationView> locations = new ArrayList<>();
    if (includeLocations) {
        StorageLocation hbaseRootDir = new StorageLocation();
        hbaseRootDir.setProperty("hbase.rootdir");
        hbaseRootDir.setValue("s3a://bucket/cluster1/hbase");
        locations.add(new StorageLocationView(hbaseRootDir));
    }
    S3FileSystemConfigurationsView fileSystemConfigurationsView = new S3FileSystemConfigurationsView(new S3FileSystem(), locations, false);
    SharedServiceConfigsView sharedServicesConfigsView = new SharedServiceConfigsView();
    sharedServicesConfigsView.setDatalakeCluster(datalakeCluster);
    String inputJson = getBlueprintText("input/clouderamanager.bp");
    CmTemplateProcessor cmTemplateProcessor = new CmTemplateProcessor(inputJson);
    cmTemplateProcessor.setCdhVersion(cdhVersion);
    GeneralClusterConfigs generalClusterConfigs = new GeneralClusterConfigs();
    generalClusterConfigs.setAccountId(Optional.of("1234"));
    return TemplatePreparationObject.Builder.builder().withFileSystemConfigurationView(fileSystemConfigurationsView).withBlueprintView(new BlueprintView(inputJson, "", "", cmTemplateProcessor)).withSharedServiceConfigs(sharedServicesConfigsView).withHostgroupViews(Set.of(master, worker)).withGeneralClusterConfigs(generalClusterConfigs).withDefaultTags(defaultTags).withCloudPlatform(cloudPlatform).build();
}