Examples with StorageLocationView com.sequenceiq.cloudbreak.template.filesystem.StorageLocationView used on opensource projects

Example 1 with StorageLocationView

use of com.sequenceiq.cloudbreak.template.filesystem.StorageLocationView in project cloudbreak by hortonworks.

the class ProfilerMetricsCloudStorageRoleConfigProviderTest method getTemplatePreparationObject.

private TemplatePreparationObject getTemplatePreparationObject(boolean includeLocations) {
    HostgroupView master = new HostgroupView("master", 1, InstanceGroupType.GATEWAY, 1);
    HostgroupView worker = new HostgroupView("worker", 2, InstanceGroupType.CORE, 2);
    List<StorageLocationView> locations = new ArrayList<>();
    if (includeLocations) {
        locations.add(new StorageLocationView(getProfilerMetricsFileSystemUri()));
    }
    S3FileSystemConfigurationsView fileSystemConfigurationsView = new S3FileSystemConfigurationsView(new S3FileSystem(), locations, false);
    return Builder.builder().withFileSystemConfigurationView(fileSystemConfigurationsView).withHostgroupViews(Set.of(master, worker)).build();
}

Example 2 with StorageLocationView

use of com.sequenceiq.cloudbreak.template.filesystem.StorageLocationView in project cloudbreak by hortonworks.

the class RangerCloudStorageServiceConfigProviderTest method getTemplatePreparationObjectForAzure.

private TemplatePreparationObject.Builder getTemplatePreparationObjectForAzure(boolean above721) {
    HostgroupView gateway = new HostgroupView("gateway", 1, InstanceGroupType.GATEWAY, Set.of("g"));
    HostgroupView master = new HostgroupView("master", 1, InstanceGroupType.CORE, Set.of("m1", "m2"));
    HostgroupView worker = new HostgroupView("worker", 2, InstanceGroupType.CORE, Set.of("w1", "w2", "w3"));
    List<StorageLocationView> locations = new ArrayList<>();
    locations.add(new StorageLocationView(getRangerAuditCloudStorageDirAzure()));
    if (above721) {
        locations.add(new StorageLocationView(buildStorageLocation("hive.metastore.warehouse.dir", "abfs://data@your-san.dfs.core.windows.net/warehouse/tablespace/managed/hive")));
        locations.add(new StorageLocationView(buildStorageLocation("hive.repl.replica.functions.root.dir", "abfs://data@your-san.dfs.core.windows.net/hive_replica_functions_dir")));
        locations.add(new StorageLocationView(buildStorageLocation("hive.metastore.warehouse.external.dir", "abfs://data@your-san.dfs.core.windows.net/warehouse/tablespace/external/hive")));
        locations.add(new StorageLocationView(buildStorageLocation("hbase.rootdir", "abfs://data@your-san.dfs.core.windows.net/hbase")));
    }
    AdlsGen2FileSystemConfigurationsView fileSystemConfigurationsView = new AdlsGen2FileSystemConfigurationsView(new AdlsGen2FileSystem(), locations, false);
    GeneralClusterConfigs generalClusterConfigs = new GeneralClusterConfigs();
    generalClusterConfigs.setPrimaryGatewayInstanceDiscoveryFQDN(Optional.of("fqdn"));
    return Builder.builder().withFileSystemConfigurationView(fileSystemConfigurationsView).withHostgroupViews(Set.of(gateway, master, worker)).withGeneralClusterConfigs(generalClusterConfigs);
}

Example 3 with StorageLocationView

use of com.sequenceiq.cloudbreak.template.filesystem.StorageLocationView in project cloudbreak by hortonworks.

the class DataDiscoveryServiceCloudStorageRoleConfigProviderTest method getTemplatePreparationObject.

private TemplatePreparationObject getTemplatePreparationObject(boolean includeLocations) {
    HostgroupView master = new HostgroupView("master", 1, InstanceGroupType.GATEWAY, 1);
    HostgroupView worker = new HostgroupView("worker", 2, InstanceGroupType.CORE, 2);
    List<StorageLocationView> locations = new ArrayList<>();
    if (includeLocations) {
        locations.add(new StorageLocationView(getDataDiscoveryServiceFileSystemUri()));
    }
    S3FileSystemConfigurationsView fileSystemConfigurationsView = new S3FileSystemConfigurationsView(new S3FileSystem(), locations, false);
    return Builder.builder().withFileSystemConfigurationView(fileSystemConfigurationsView).withHostgroupViews(Set.of(master, worker)).build();
}

Example 4 with StorageLocationView

use of com.sequenceiq.cloudbreak.template.filesystem.StorageLocationView in project cloudbreak by hortonworks.

the class S3ConfigProviderTest method getTemplatePreparationObject.

private TemplatePreparationObject getTemplatePreparationObject(boolean useS3FileSystem, boolean fillDynamoTableName, boolean includeLocations) {
    HostgroupView master = new HostgroupView("master", 1, InstanceGroupType.GATEWAY, 1);
    HostgroupView worker = new HostgroupView("worker", 2, InstanceGroupType.CORE, 2);
    List<StorageLocationView> locations = new ArrayList<>();
    if (includeLocations) {
        locations.add(new StorageLocationView(getStorageLocation("hive.metastore.warehouse.dir", "s3a://bucket-first/warehouse/managed")));
        locations.add(new StorageLocationView(getStorageLocation("hive.metastore.warehouse.external.dir", "s3a://bucket-first/warehouse/external")));
        locations.add(new StorageLocationView(getStorageLocation("ranger_plugin_hdfs_audit_url", "s3a://bucket-second/ranger/audit")));
    }
    BaseFileSystemConfigurationsView fileSystemConfigurationsView;
    if (useS3FileSystem) {
        S3FileSystem s3FileSystem = new S3FileSystem();
        if (fillDynamoTableName) {
            s3FileSystem.setS3GuardDynamoTableName("dynamoTable");
        }
        fileSystemConfigurationsView = new S3FileSystemConfigurationsView(s3FileSystem, locations, false);
    } else {
        fileSystemConfigurationsView = new AdlsFileSystemConfigurationsView(new AdlsFileSystem(), locations, false);
    }
    Gateway gateway = TestUtil.gatewayEnabledWithExposedKnoxServices("NAMENODE");
    PlacementView placementView = new PlacementView("region", "az");
    return Builder.builder().withFileSystemConfigurationView(fileSystemConfigurationsView).withHostgroupViews(Set.of(master, worker)).withGateway(gateway, "/cb/secret/signkey", new HashSet<>()).withPlacementView(placementView).withDefaultTags(Map.of("apple", "apple1")).build();
}

Example 5 with StorageLocationView

use of com.sequenceiq.cloudbreak.template.filesystem.StorageLocationView in project cloudbreak by hortonworks.

the class SparkOnYarnRoleConfigProviderTest method getTemplatePreparationObject.

private TemplatePreparationObject getTemplatePreparationObject(String... locations) {
    HostgroupView master = new HostgroupView("master", 1, InstanceGroupType.GATEWAY, 1);
    HostgroupView worker = new HostgroupView("worker", 2, InstanceGroupType.CORE, 2);
    List<StorageLocationView> storageLocations = new ArrayList<>();
    if (locations.length >= 1) {
        StorageLocation hmsExternalWarehouseDir = new StorageLocation();
        hmsExternalWarehouseDir.setProperty("hive.metastore.warehouse.external.dir");
        hmsExternalWarehouseDir.setValue(locations[0]);
        storageLocations.add(new StorageLocationView(hmsExternalWarehouseDir));
    }
    S3FileSystemConfigurationsView fileSystemConfigurationsView = new S3FileSystemConfigurationsView(new S3FileSystem(), storageLocations, false);
    return Builder.builder().withFileSystemConfigurationView(fileSystemConfigurationsView).withHostgroupViews(Set.of(master, worker)).build();
}