Examples with AdlsGen2FileSystem com.sequenceiq.common.api.filesystem.AdlsGen2FileSystem used on opensource projects

Example 1 with AdlsGen2FileSystem

use of com.sequenceiq.common.api.filesystem.AdlsGen2FileSystem in project cloudbreak by hortonworks.

the class CloudStorageParametersConverter method adlsGen2ToFileSystem.

public AdlsGen2FileSystem adlsGen2ToFileSystem(AdlsGen2CloudStorageV1Parameters source) {
    AdlsGen2FileSystem adlsGen2FileSystem = new AdlsGen2FileSystem();
    adlsGen2FileSystem.setAccountName(source.getAccountName());
    adlsGen2FileSystem.setAccountKey(source.getAccountKey());
    adlsGen2FileSystem.setManagedIdentity(source.getManagedIdentity());
    return adlsGen2FileSystem;
}

Example 2 with AdlsGen2FileSystem

use of com.sequenceiq.common.api.filesystem.AdlsGen2FileSystem in project cloudbreak by hortonworks.

the class AdlsGen2FileSystemToAdlsGen2CloudStorageParametersConverterTest method createAdlsGen2FileSystem.

private AdlsGen2FileSystem createAdlsGen2FileSystem() {
    AdlsGen2FileSystem adlsGen2FileSystem = new AdlsGen2FileSystem();
    adlsGen2FileSystem.setAccountKey(TEST_ACCOUNT_KEY);
    adlsGen2FileSystem.setAccountName(TEST_ACCOUNT_NAME);
    return adlsGen2FileSystem;
}

Example 3 with AdlsGen2FileSystem

use of com.sequenceiq.common.api.filesystem.AdlsGen2FileSystem in project cloudbreak by hortonworks.

the class RangerCloudStorageServiceConfigProviderTest method getTemplatePreparationObjectForAzure.

private TemplatePreparationObject.Builder getTemplatePreparationObjectForAzure(boolean above721) {
    HostgroupView gateway = new HostgroupView("gateway", 1, InstanceGroupType.GATEWAY, Set.of("g"));
    HostgroupView master = new HostgroupView("master", 1, InstanceGroupType.CORE, Set.of("m1", "m2"));
    HostgroupView worker = new HostgroupView("worker", 2, InstanceGroupType.CORE, Set.of("w1", "w2", "w3"));
    List<StorageLocationView> locations = new ArrayList<>();
    locations.add(new StorageLocationView(getRangerAuditCloudStorageDirAzure()));
    if (above721) {
        locations.add(new StorageLocationView(buildStorageLocation("hive.metastore.warehouse.dir", "abfs://data@your-san.dfs.core.windows.net/warehouse/tablespace/managed/hive")));
        locations.add(new StorageLocationView(buildStorageLocation("hive.repl.replica.functions.root.dir", "abfs://data@your-san.dfs.core.windows.net/hive_replica_functions_dir")));
        locations.add(new StorageLocationView(buildStorageLocation("hive.metastore.warehouse.external.dir", "abfs://data@your-san.dfs.core.windows.net/warehouse/tablespace/external/hive")));
        locations.add(new StorageLocationView(buildStorageLocation("hbase.rootdir", "abfs://data@your-san.dfs.core.windows.net/hbase")));
    }
    AdlsGen2FileSystemConfigurationsView fileSystemConfigurationsView = new AdlsGen2FileSystemConfigurationsView(new AdlsGen2FileSystem(), locations, false);
    GeneralClusterConfigs generalClusterConfigs = new GeneralClusterConfigs();
    generalClusterConfigs.setPrimaryGatewayInstanceDiscoveryFQDN(Optional.of("fqdn"));
    return Builder.builder().withFileSystemConfigurationView(fileSystemConfigurationsView).withHostgroupViews(Set.of(gateway, master, worker)).withGeneralClusterConfigs(generalClusterConfigs);
}

Example 4 with AdlsGen2FileSystem

use of com.sequenceiq.common.api.filesystem.AdlsGen2FileSystem in project cloudbreak by hortonworks.

the class FileSystemConverter method legacyConvertFromConfiguration.

private List<CloudFileSystemView> legacyConvertFromConfiguration(FileSystem source) {
    try {
        CloudFileSystemView fileSystemView;
        if (source.getType().isAdls()) {
            AdlsFileSystem adlsFileSystem = source.getConfigurations().get(AdlsFileSystem.class);
            fileSystemView = convertAdlsLegacy(adlsFileSystem);
        } else if (source.getType().isGcs()) {
            GcsFileSystem gcsFileSystem = source.getConfigurations().get(GcsFileSystem.class);
            fileSystemView = convertGcsLegacy(gcsFileSystem);
        } else if (source.getType().isS3()) {
            S3FileSystem s3FileSystem = source.getConfigurations().get(S3FileSystem.class);
            fileSystemView = convertS3Legacy(s3FileSystem);
        } else if (source.getType().isEfs()) {
            EfsFileSystem efsFileSystem = source.getConfigurations().get(EfsFileSystem.class);
            fileSystemView = convertEfsLegacy(efsFileSystem);
        } else if (source.getType().isWasb()) {
            WasbFileSystem wasbFileSystem = source.getConfigurations().get(WasbFileSystem.class);
            fileSystemView = convertWasbLegacy(wasbFileSystem);
        } else if (source.getType().isAdlsGen2()) {
            AdlsGen2FileSystem adlsGen2FileSystem = source.getConfigurations().get(AdlsGen2FileSystem.class);
            fileSystemView = convertAdlsGen2Legacy(adlsGen2FileSystem);
        } else if (source.getType().isGcs()) {
            GcsFileSystem gcsFileSystem = source.getConfigurations().get(GcsFileSystem.class);
            fileSystemView = convertGcsLegacy(gcsFileSystem);
        } else {
            return Collections.emptyList();
        }
        return List.of(fileSystemView);
    } catch (IOException e) {
        LOGGER.warn("Error occurred when tried to convert filesystem object: {}", e.getMessage());
    }
    return Collections.emptyList();
}

Example 5 with AdlsGen2FileSystem

use of com.sequenceiq.common.api.filesystem.AdlsGen2FileSystem in project cloudbreak by hortonworks.

the class ZeppelinCloudStorageRoleConfigProviderTest method assertZeppelinStorageValues.

protected void assertZeppelinStorageValues(String storagePath) {
    List<StorageLocationView> locations = new ArrayList<>();
    StorageLocation zeppelinNotebookDir = new StorageLocation();
    zeppelinNotebookDir.setProperty("zeppelin.notebook.dir");
    zeppelinNotebookDir.setValue(storagePath);
    locations.add(new StorageLocationView(zeppelinNotebookDir));
    BaseFileSystemConfigurationsView fileSystemConfigurationsView;
    if (storagePath.startsWith("s3a")) {
        fileSystemConfigurationsView = new S3FileSystemConfigurationsView(new S3FileSystem(), locations, false);
    } else if (storagePath.startsWith("gcs")) {
        fileSystemConfigurationsView = new GcsFileSystemConfigurationsView(new GcsFileSystem(), locations, false);
    } else {
        fileSystemConfigurationsView = new AdlsGen2FileSystemConfigurationsView(new AdlsGen2FileSystem(), locations, false);
    }
    TemplatePreparationObject preparationObject = getTemplatePreparationObject(fileSystemConfigurationsView);
    String inputJson = getBlueprintText("input/clouderamanager-ds.bp");
    CmTemplateProcessor cmTemplateProcessor = new CmTemplateProcessor(inputJson);
    Map<String, List<ApiClusterTemplateConfig>> roleConfigs = underTest.getRoleConfigs(cmTemplateProcessor, preparationObject);
    List<ApiClusterTemplateConfig> zeppelinStorageConfigs = roleConfigs.get("zeppelin-ZEPPELIN_SERVER-BASE");
    assertEquals(1, zeppelinStorageConfigs.size());
    assertEquals("zeppelin.notebook.dir", zeppelinStorageConfigs.get(0).getName());
    assertEquals(storagePath, zeppelinStorageConfigs.get(0).getValue());
}