Examples with SdxCloudStorageRequest com.sequenceiq.sdx.api.model.SdxCloudStorageRequest used on opensource projects

Example 6 with SdxCloudStorageRequest

use of com.sequenceiq.sdx.api.model.SdxCloudStorageRequest in project cloudbreak by hortonworks.

the class StorageValidationServiceTest method whenAwsConfiguredWithRoleShouldNotThrowException.

@Test
public void whenAwsConfiguredWithRoleShouldNotThrowException() {
    SdxCloudStorageRequest cloudStorageRequest = new SdxCloudStorageRequest();
    cloudStorageRequest.setBaseLocation("s3a://example-path");
    cloudStorageRequest.setFileSystemType(FileSystemType.S3);
    S3CloudStorageV1Parameters s3 = new S3CloudStorageV1Parameters();
    s3.setInstanceProfile("role");
    cloudStorageRequest.setS3(s3);
    underTest.validateCloudStorage("AWS", cloudStorageRequest);
}

Example 7 with SdxCloudStorageRequest

use of com.sequenceiq.sdx.api.model.SdxCloudStorageRequest in project cloudbreak by hortonworks.

the class CloudStorageManifesterTest method whenConfigIsProvidedReturnFileSystemParameters.

@Test
public void whenConfigIsProvidedReturnFileSystemParameters() {
    when(regionAwareInternalCrnGenerator.getInternalCrnForServiceAsString()).thenReturn("crn");
    when(regionAwareInternalCrnGeneratorFactory.iam()).thenReturn(regionAwareInternalCrnGenerator);
    mockFileSystemResponseForCloudbreakClient();
    SdxCluster sdxCluster = new SdxCluster();
    SdxClusterRequest sdxClusterRequest = new SdxClusterRequest();
    sdxCluster.setInitiatorUserCrn(USER_CRN);
    sdxCluster.setClusterName("sdx-cluster");
    SdxCloudStorageRequest cloudStorageRequest = new SdxCloudStorageRequest();
    cloudStorageRequest.setBaseLocation("s3a://example-path");
    cloudStorageRequest.setFileSystemType(FileSystemType.S3);
    S3CloudStorageV1Parameters s3Params = new S3CloudStorageV1Parameters();
    s3Params.setInstanceProfile("instance:profile");
    cloudStorageRequest.setS3(s3Params);
    sdxClusterRequest.setCloudStorage(cloudStorageRequest);
    DetailedEnvironmentResponse environment = new DetailedEnvironmentResponse();
    environment.setCloudPlatform("AWS");
    ClusterV4Request clusterV4Request = new ClusterV4Request();
    clusterV4Request.setBlueprintName(exampleBlueprintName);
    CloudStorageRequest cloudStorageConfigReq = ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.initCloudStorageRequest(environment, clusterV4Request, sdxCluster, sdxClusterRequest));
    StorageLocationBase singleRequest = cloudStorageConfigReq.getLocations().iterator().next();
    assertEquals(1, cloudStorageConfigReq.getIdentities().size());
    assertEquals(CloudIdentityType.ID_BROKER, cloudStorageConfigReq.getIdentities().iterator().next().getType());
    assertEquals(1, cloudStorageConfigReq.getLocations().size());
    assertEquals(CloudStorageCdpService.RANGER_AUDIT, singleRequest.getType());
    assertEquals("ranger/example-path", singleRequest.getValue());
    verify(storageValidationService).validateCloudStorage("AWS", cloudStorageRequest);
}

Example 8 with SdxCloudStorageRequest

use of com.sequenceiq.sdx.api.model.SdxCloudStorageRequest in project cloudbreak by hortonworks.

the class CloudStorageManifesterTest method whenEnvironmentHasLoggingEnabledThenShouldApplyAsLogIdentityForGCS.

@Test
public void whenEnvironmentHasLoggingEnabledThenShouldApplyAsLogIdentityForGCS() {
    when(regionAwareInternalCrnGenerator.getInternalCrnForServiceAsString()).thenReturn("crn");
    when(regionAwareInternalCrnGeneratorFactory.iam()).thenReturn(regionAwareInternalCrnGenerator);
    mockFileSystemResponseForCloudbreakClient();
    SdxCluster sdxCluster = new SdxCluster();
    SdxClusterRequest sdxClusterRequest = new SdxClusterRequest();
    sdxCluster.setInitiatorUserCrn(USER_CRN);
    sdxCluster.setClusterName("sdx-cluster");
    SdxCloudStorageRequest cloudStorageRequest = new SdxCloudStorageRequest();
    cloudStorageRequest.setBaseLocation("gs://example-path");
    cloudStorageRequest.setFileSystemType(FileSystemType.GCS);
    GcsCloudStorageV1Parameters gcsCloudStorageV1Parameters = new GcsCloudStorageV1Parameters();
    gcsCloudStorageV1Parameters.setServiceAccountEmail(EMAIL);
    cloudStorageRequest.setGcs(gcsCloudStorageV1Parameters);
    sdxClusterRequest.setCloudStorage(cloudStorageRequest);
    DetailedEnvironmentResponse environment = new DetailedEnvironmentResponse();
    environment.setCloudPlatform("GCP");
    TelemetryResponse telemetryResponse = new TelemetryResponse();
    LoggingResponse loggingResponse = new LoggingResponse();
    loggingResponse.setGcs(gcsCloudStorageV1Parameters);
    telemetryResponse.setLogging(loggingResponse);
    GcpEnvironmentParameters gcpEnvironmentParameters = GcpEnvironmentParameters.builder().build();
    environment.setGcp(gcpEnvironmentParameters);
    environment.setTelemetry(telemetryResponse);
    ClusterV4Request clusterV4Request = new ClusterV4Request();
    clusterV4Request.setBlueprintName(exampleBlueprintName);
    CloudStorageRequest cloudStorageConfigReq = ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.initCloudStorageRequest(environment, clusterV4Request, sdxCluster, sdxClusterRequest));
    StorageLocationBase singleRequest = cloudStorageConfigReq.getLocations().iterator().next();
    assertEquals(2, cloudStorageConfigReq.getIdentities().size());
    assertEquals(1, cloudStorageConfigReq.getIdentities().stream().filter(r -> r.getType().equals(CloudIdentityType.ID_BROKER)).collect(Collectors.toSet()).size());
    assertEquals(1, cloudStorageConfigReq.getIdentities().stream().filter(r -> r.getType().equals(CloudIdentityType.LOG)).collect(Collectors.toSet()).size());
    assertEquals(2, cloudStorageConfigReq.getIdentities().stream().filter(r -> r.getGcs().getServiceAccountEmail().equals(EMAIL)).collect(Collectors.toSet()).size());
    assertEquals(1, cloudStorageConfigReq.getLocations().size());
    assertEquals(CloudStorageCdpService.RANGER_AUDIT, singleRequest.getType());
    assertEquals("ranger/example-path", singleRequest.getValue());
    verify(storageValidationService).validateCloudStorage("GCP", cloudStorageRequest);
}

Example 9 with SdxCloudStorageRequest

use of com.sequenceiq.sdx.api.model.SdxCloudStorageRequest in project cloudbreak by hortonworks.

the class CloudStorageManifesterTest method whenEnvironmentHasLoggingEnabledThenShouldApplyAsLogIdentity.

@Test
public void whenEnvironmentHasLoggingEnabledThenShouldApplyAsLogIdentity() {
    when(regionAwareInternalCrnGenerator.getInternalCrnForServiceAsString()).thenReturn("crn");
    when(regionAwareInternalCrnGeneratorFactory.iam()).thenReturn(regionAwareInternalCrnGenerator);
    mockFileSystemResponseForCloudbreakClient();
    SdxCluster sdxCluster = new SdxCluster();
    SdxClusterRequest sdxClusterRequest = new SdxClusterRequest();
    sdxCluster.setInitiatorUserCrn(USER_CRN);
    sdxCluster.setClusterName("sdx-cluster");
    SdxCloudStorageRequest cloudStorageRequest = new SdxCloudStorageRequest();
    cloudStorageRequest.setBaseLocation("s3a://example-path");
    cloudStorageRequest.setFileSystemType(FileSystemType.S3);
    S3CloudStorageV1Parameters s3Params = new S3CloudStorageV1Parameters();
    s3Params.setInstanceProfile("instance:profile");
    cloudStorageRequest.setS3(s3Params);
    sdxClusterRequest.setCloudStorage(cloudStorageRequest);
    DetailedEnvironmentResponse environment = new DetailedEnvironmentResponse();
    environment.setCloudPlatform("AWS");
    TelemetryResponse telemetryResponse = new TelemetryResponse();
    LoggingResponse loggingResponse = new LoggingResponse();
    S3CloudStorageV1Parameters s3CloudStorageV1Parameters = new S3CloudStorageV1Parameters();
    s3CloudStorageV1Parameters.setInstanceProfile("logprofile");
    loggingResponse.setS3(s3CloudStorageV1Parameters);
    telemetryResponse.setLogging(loggingResponse);
    AwsEnvironmentParameters awsEnvironmentParameters = new AwsEnvironmentParameters();
    S3GuardRequestParameters s3GuardRequestParameters = new S3GuardRequestParameters();
    s3GuardRequestParameters.setDynamoDbTableName("table");
    awsEnvironmentParameters.setS3guard(s3GuardRequestParameters);
    environment.setAws(awsEnvironmentParameters);
    environment.setTelemetry(telemetryResponse);
    ClusterV4Request clusterV4Request = new ClusterV4Request();
    clusterV4Request.setBlueprintName(exampleBlueprintName);
    CloudStorageRequest cloudStorageConfigReq = ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.initCloudStorageRequest(environment, clusterV4Request, sdxCluster, sdxClusterRequest));
    StorageLocationBase singleRequest = cloudStorageConfigReq.getLocations().iterator().next();
    assertEquals(2, cloudStorageConfigReq.getIdentities().size());
    assertEquals(1, cloudStorageConfigReq.getIdentities().stream().filter(r -> r.getType().equals(CloudIdentityType.ID_BROKER)).collect(Collectors.toSet()).size());
    assertEquals(1, cloudStorageConfigReq.getIdentities().stream().filter(r -> r.getType().equals(CloudIdentityType.LOG)).collect(Collectors.toSet()).size());
    assertEquals("table", cloudStorageConfigReq.getAws().getS3Guard().getDynamoTableName());
    assertEquals(1, cloudStorageConfigReq.getLocations().size());
    assertEquals(CloudStorageCdpService.RANGER_AUDIT, singleRequest.getType());
    assertEquals("ranger/example-path", singleRequest.getValue());
    verify(storageValidationService).validateCloudStorage("AWS", cloudStorageRequest);
}

Example 10 with SdxCloudStorageRequest

use of com.sequenceiq.sdx.api.model.SdxCloudStorageRequest in project cloudbreak by hortonworks.

the class CloudStorageManifesterTest method stripWhitespacesWhenBaseLocationHasWhiteSpaces.

@Test
public void stripWhitespacesWhenBaseLocationHasWhiteSpaces() {
    String baseLocationWithWhiteSpaces = "s3a://cloudbreak bucket/something ";
    SdxCloudStorageRequest cloudStorageRequest = new SdxCloudStorageRequest();
    cloudStorageRequest.setBaseLocation(baseLocationWithWhiteSpaces);
    S3CloudStorageV1Parameters params = new S3CloudStorageV1Parameters();
    params.setInstanceProfile("instanceProfile");
    cloudStorageRequest.setS3(params);
    underTest.normalizeCloudStorageRequest(cloudStorageRequest);
    assertEquals("s3a://cloudbreak bucket/something", cloudStorageRequest.getBaseLocation());
}