Search in sources :

Example 21 with CloudStorageRequest

use of com.sequenceiq.common.api.cloudstorage.CloudStorageRequest in project cloudbreak by hortonworks.

the class CloudStorageManifesterTest method whenEnvironmentHasLoggingEnabledThenShouldApplyAsLogIdentity.

@Test
public void whenEnvironmentHasLoggingEnabledThenShouldApplyAsLogIdentity() {
    when(regionAwareInternalCrnGenerator.getInternalCrnForServiceAsString()).thenReturn("crn");
    when(regionAwareInternalCrnGeneratorFactory.iam()).thenReturn(regionAwareInternalCrnGenerator);
    mockFileSystemResponseForCloudbreakClient();
    SdxCluster sdxCluster = new SdxCluster();
    SdxClusterRequest sdxClusterRequest = new SdxClusterRequest();
    sdxCluster.setInitiatorUserCrn(USER_CRN);
    sdxCluster.setClusterName("sdx-cluster");
    SdxCloudStorageRequest cloudStorageRequest = new SdxCloudStorageRequest();
    cloudStorageRequest.setBaseLocation("s3a://example-path");
    cloudStorageRequest.setFileSystemType(FileSystemType.S3);
    S3CloudStorageV1Parameters s3Params = new S3CloudStorageV1Parameters();
    s3Params.setInstanceProfile("instance:profile");
    cloudStorageRequest.setS3(s3Params);
    sdxClusterRequest.setCloudStorage(cloudStorageRequest);
    DetailedEnvironmentResponse environment = new DetailedEnvironmentResponse();
    environment.setCloudPlatform("AWS");
    TelemetryResponse telemetryResponse = new TelemetryResponse();
    LoggingResponse loggingResponse = new LoggingResponse();
    S3CloudStorageV1Parameters s3CloudStorageV1Parameters = new S3CloudStorageV1Parameters();
    s3CloudStorageV1Parameters.setInstanceProfile("logprofile");
    loggingResponse.setS3(s3CloudStorageV1Parameters);
    telemetryResponse.setLogging(loggingResponse);
    AwsEnvironmentParameters awsEnvironmentParameters = new AwsEnvironmentParameters();
    S3GuardRequestParameters s3GuardRequestParameters = new S3GuardRequestParameters();
    s3GuardRequestParameters.setDynamoDbTableName("table");
    awsEnvironmentParameters.setS3guard(s3GuardRequestParameters);
    environment.setAws(awsEnvironmentParameters);
    environment.setTelemetry(telemetryResponse);
    ClusterV4Request clusterV4Request = new ClusterV4Request();
    clusterV4Request.setBlueprintName(exampleBlueprintName);
    CloudStorageRequest cloudStorageConfigReq = ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.initCloudStorageRequest(environment, clusterV4Request, sdxCluster, sdxClusterRequest));
    StorageLocationBase singleRequest = cloudStorageConfigReq.getLocations().iterator().next();
    assertEquals(2, cloudStorageConfigReq.getIdentities().size());
    assertEquals(1, cloudStorageConfigReq.getIdentities().stream().filter(r -> r.getType().equals(CloudIdentityType.ID_BROKER)).collect(Collectors.toSet()).size());
    assertEquals(1, cloudStorageConfigReq.getIdentities().stream().filter(r -> r.getType().equals(CloudIdentityType.LOG)).collect(Collectors.toSet()).size());
    assertEquals("table", cloudStorageConfigReq.getAws().getS3Guard().getDynamoTableName());
    assertEquals(1, cloudStorageConfigReq.getLocations().size());
    assertEquals(CloudStorageCdpService.RANGER_AUDIT, singleRequest.getType());
    assertEquals("ranger/example-path", singleRequest.getValue());
    verify(storageValidationService).validateCloudStorage("AWS", cloudStorageRequest);
}
Also used : SdxCloudStorageRequest(com.sequenceiq.sdx.api.model.SdxCloudStorageRequest) TelemetryResponse(com.sequenceiq.common.api.telemetry.response.TelemetryResponse) SdxCluster(com.sequenceiq.datalake.entity.SdxCluster) GcpEnvironmentParameters(com.sequenceiq.environment.api.v1.environment.model.request.gcp.GcpEnvironmentParameters) ArgumentMatchers.anyLong(org.mockito.ArgumentMatchers.anyLong) Mock(org.mockito.Mock) FileSystemParameterV4Responses(com.sequenceiq.cloudbreak.api.endpoint.v4.filesystems.responses.FileSystemParameterV4Responses) Assertions.assertNull(org.junit.jupiter.api.Assertions.assertNull) ClusterV4Request(com.sequenceiq.cloudbreak.api.endpoint.v4.stacks.request.cluster.ClusterV4Request) GcsCloudStorageV1Parameters(com.sequenceiq.common.api.cloudstorage.old.GcsCloudStorageV1Parameters) ArgumentMatchers.anyBoolean(org.mockito.ArgumentMatchers.anyBoolean) AwsEnvironmentParameters(com.sequenceiq.environment.api.v1.environment.model.request.aws.AwsEnvironmentParameters) ArrayList(java.util.ArrayList) ThreadBasedUserCrnProvider(com.sequenceiq.cloudbreak.auth.ThreadBasedUserCrnProvider) ExtendWith(org.junit.jupiter.api.extension.ExtendWith) FileSystemParameterV4Response(com.sequenceiq.cloudbreak.api.endpoint.v4.filesystems.responses.FileSystemParameterV4Response) DetailedEnvironmentResponse(com.sequenceiq.environment.api.v1.environment.model.response.DetailedEnvironmentResponse) SdxCloudStorageRequest(com.sequenceiq.sdx.api.model.SdxCloudStorageRequest) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) TelemetryResponse(com.sequenceiq.common.api.telemetry.response.TelemetryResponse) InjectMocks(org.mockito.InjectMocks) RegionAwareInternalCrnGenerator(com.sequenceiq.cloudbreak.auth.crn.RegionAwareInternalCrnGenerator) LoggingResponse(com.sequenceiq.common.api.telemetry.response.LoggingResponse) MockitoExtension(org.mockito.junit.jupiter.MockitoExtension) CloudStorageCdpService(com.sequenceiq.common.model.CloudStorageCdpService) StorageLocationBase(com.sequenceiq.common.api.cloudstorage.StorageLocationBase) S3CloudStorageV1Parameters(com.sequenceiq.common.api.cloudstorage.old.S3CloudStorageV1Parameters) Mockito.when(org.mockito.Mockito.when) Collectors(java.util.stream.Collectors) Mockito.verify(org.mockito.Mockito.verify) FileSystemV4Endpoint(com.sequenceiq.cloudbreak.api.endpoint.v4.filesystems.FileSystemV4Endpoint) RegionAwareInternalCrnGeneratorFactory(com.sequenceiq.cloudbreak.auth.crn.RegionAwareInternalCrnGeneratorFactory) Test(org.junit.jupiter.api.Test) SdxClusterRequest(com.sequenceiq.sdx.api.model.SdxClusterRequest) List(java.util.List) S3GuardRequestParameters(com.sequenceiq.environment.api.v1.environment.model.request.aws.S3GuardRequestParameters) CloudStorageRequest(com.sequenceiq.common.api.cloudstorage.CloudStorageRequest) FileSystemType(com.sequenceiq.common.model.FileSystemType) CloudIdentityType(com.sequenceiq.common.model.CloudIdentityType) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) S3CloudStorageV1Parameters(com.sequenceiq.common.api.cloudstorage.old.S3CloudStorageV1Parameters) SdxCloudStorageRequest(com.sequenceiq.sdx.api.model.SdxCloudStorageRequest) CloudStorageRequest(com.sequenceiq.common.api.cloudstorage.CloudStorageRequest) SdxClusterRequest(com.sequenceiq.sdx.api.model.SdxClusterRequest) StorageLocationBase(com.sequenceiq.common.api.cloudstorage.StorageLocationBase) ClusterV4Request(com.sequenceiq.cloudbreak.api.endpoint.v4.stacks.request.cluster.ClusterV4Request) S3GuardRequestParameters(com.sequenceiq.environment.api.v1.environment.model.request.aws.S3GuardRequestParameters) LoggingResponse(com.sequenceiq.common.api.telemetry.response.LoggingResponse) AwsEnvironmentParameters(com.sequenceiq.environment.api.v1.environment.model.request.aws.AwsEnvironmentParameters) SdxCluster(com.sequenceiq.datalake.entity.SdxCluster) DetailedEnvironmentResponse(com.sequenceiq.environment.api.v1.environment.model.response.DetailedEnvironmentResponse) Test(org.junit.jupiter.api.Test)

Example 22 with CloudStorageRequest

use of com.sequenceiq.common.api.cloudstorage.CloudStorageRequest in project cloudbreak by hortonworks.

the class SdxService method prepareCloudStorageForStack.

private void prepareCloudStorageForStack(SdxClusterRequest sdxClusterRequest, StackV4Request stackV4Request, SdxCluster sdxCluster, DetailedEnvironmentResponse environment) {
    CloudStorageRequest cloudStorageRequest = cloudStorageManifester.initCloudStorageRequest(environment, stackV4Request.getCluster(), sdxCluster, sdxClusterRequest);
    stackV4Request.getCluster().setCloudStorage(cloudStorageRequest);
}
Also used : SdxCloudStorageRequest(com.sequenceiq.sdx.api.model.SdxCloudStorageRequest) CloudStorageRequest(com.sequenceiq.common.api.cloudstorage.CloudStorageRequest)

Example 23 with CloudStorageRequest

use of com.sequenceiq.common.api.cloudstorage.CloudStorageRequest in project cloudbreak by hortonworks.

the class StackRequestManifester method setupCloudStorageAccountMapping.

@VisibleForTesting
void setupCloudStorageAccountMapping(StackV4Request stackRequest, String environmentCrn, IdBrokerMappingSource mappingSource, String cloudPlatform) {
    String stackName = stackRequest.getName();
    CloudStorageRequest cloudStorage = stackRequest.getCluster().getCloudStorage();
    if (cloudStorage != null && cloudStorage.getAccountMapping() == null) {
        // getAccountMapping() == null means we need to fetch mappings from IDBMMS.
        if (mappingSource == IdBrokerMappingSource.IDBMMS) {
            LOGGER.info("Fetching account mappings from IDBMMS associated with environment {} for stack {}.", environmentCrn, stackName);
            MappingsConfig mappingsConfig;
            try {
                // Must pass the internal actor here as this operation is internal-use only; requests with other actors will be always rejected.
                mappingsConfig = idbmmsClient.getMappingsConfig(regionAwareInternalCrnGeneratorFactory.iam().getInternalCrnForServiceAsString(), environmentCrn, Optional.empty());
                validateMappingsConfig(mappingsConfig, stackRequest);
            } catch (IdbmmsOperationException e) {
                throw new BadRequestException(String.format("Unable to get mappings: %s", e.getMessage()), e);
            }
            AccountMappingBase accountMapping = new AccountMappingBase();
            accountMapping.setGroupMappings(mappingsConfig.getGroupMappings());
            accountMapping.setUserMappings(mappingsConfig.getActorMappings());
            cloudStorage.setAccountMapping(accountMapping);
            LOGGER.info("Initial account mappings fetched from IDBMMS: {}", JsonUtil.writeValueAsStringSilent(accountMapping));
        } else {
            LOGGER.info("IDBMMS usage is disabled for environment {}. Proceeding with {} mappings for stack {}.", environmentCrn, mappingSource == IdBrokerMappingSource.MOCK && (CloudPlatform.AWS.name().equals(cloudPlatform) || CloudPlatform.AZURE.name().equals(cloudPlatform) || CloudPlatform.GCP.name().equals(cloudPlatform)) ? "mock" : "missing", stackName);
        }
    } else {
        // getAccountMapping() != null is possible only in case of SdxInternalClusterRequest, in which case the user-given values will be honored.
        LOGGER.info("{} for stack {} in environment {}.", cloudStorage == null ? "Cloud storage is disabled" : "Applying user-provided mappings", stackName, environmentCrn);
    }
}
Also used : AccountMappingBase(com.sequenceiq.common.api.cloudstorage.AccountMappingBase) CloudStorageRequest(com.sequenceiq.common.api.cloudstorage.CloudStorageRequest) MappingsConfig(com.sequenceiq.cloudbreak.idbmms.model.MappingsConfig) IdbmmsOperationException(com.sequenceiq.cloudbreak.idbmms.exception.IdbmmsOperationException) BadRequestException(com.sequenceiq.cloudbreak.common.exception.BadRequestException) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Example 24 with CloudStorageRequest

use of com.sequenceiq.common.api.cloudstorage.CloudStorageRequest in project cloudbreak by hortonworks.

the class StorageValidationService method validateObjectStorage.

public ObjectStorageValidateResponse validateObjectStorage(String credentialCrn, SdxCloudStorageRequest sdxCloudStorageRequest, String blueprintName, String clusterName, String dataAccessRole, String rangerAuditRole) {
    CredentialResponse credentialResponse = environmentClientService.getCredentialByCrn(credentialCrn);
    String attributes = secretService.getByResponse(credentialResponse.getAttributes());
    CloudCredential cloudCredential = new CloudCredential(credentialResponse.getCrn(), credentialResponse.getName(), new Json(attributes).getMap(), credentialResponse.getAccountId(), credentialResponse.isVerifyPermissions());
    CloudStorageRequest cloudStorageRequest = cloudStorageManifester.initSdxCloudStorageRequest(credentialResponse.getCloudPlatform(), blueprintName, clusterName, sdxCloudStorageRequest);
    AccountMappingBase accountMapping = new AccountMappingBase();
    Map<String, String> userMapping = getUserMapping(dataAccessRole, rangerAuditRole);
    accountMapping.setUserMappings(userMapping);
    cloudStorageRequest.setAccountMapping(accountMapping);
    ObjectStorageValidateRequest objectStorageValidateRequest = ObjectStorageValidateRequest.builder().withCloudPlatform(credentialResponse.getCloudPlatform()).withCredential(cloudCredential).withCloudStorageRequest(cloudStorageRequest).build();
    return ThreadBasedUserCrnProvider.doAsInternalActor(regionAwareInternalCrnGeneratorFactory.iam().getInternalCrnForServiceAsString(), () -> cloudProviderServicesV4Endpoint.validateObjectStorage(objectStorageValidateRequest));
}
Also used : AccountMappingBase(com.sequenceiq.common.api.cloudstorage.AccountMappingBase) CloudStorageRequest(com.sequenceiq.common.api.cloudstorage.CloudStorageRequest) SdxCloudStorageRequest(com.sequenceiq.sdx.api.model.SdxCloudStorageRequest) CloudCredential(com.sequenceiq.cloudbreak.cloud.model.CloudCredential) CredentialResponse(com.sequenceiq.environment.api.v1.credential.model.response.CredentialResponse) Json(com.sequenceiq.cloudbreak.common.json.Json) ObjectStorageValidateRequest(com.sequenceiq.cloudbreak.cloud.model.objectstorage.ObjectStorageValidateRequest)

Example 25 with CloudStorageRequest

use of com.sequenceiq.common.api.cloudstorage.CloudStorageRequest in project cloudbreak by hortonworks.

the class CloudStorageManifester method setStorageLocations.

private void setStorageLocations(FileSystemParameterV4Responses fileSystemRecommendations, CloudStorageRequest cloudStorageRequest) {
    List<StorageLocationBase> storageLocations = fileSystemRecommendations.getResponses().stream().map(response -> {
        StorageLocationBase storageLocation = new StorageLocationBase();
        storageLocation.setValue(response.getDefaultPath());
        storageLocation.setType(CloudStorageCdpService.valueOf(response.getType()));
        return storageLocation;
    }).collect(Collectors.toList());
    cloudStorageRequest.setLocations(storageLocations);
}
Also used : SdxCluster(com.sequenceiq.datalake.entity.SdxCluster) AwsStorageParameters(com.sequenceiq.common.api.cloudstorage.AwsStorageParameters) FileSystemParameterV4Responses(com.sequenceiq.cloudbreak.api.endpoint.v4.filesystems.responses.FileSystemParameterV4Responses) LoggerFactory(org.slf4j.LoggerFactory) ClusterV4Request(com.sequenceiq.cloudbreak.api.endpoint.v4.stacks.request.cluster.ClusterV4Request) StringUtils(org.apache.commons.lang3.StringUtils) Inject(javax.inject.Inject) Strings(com.google.common.base.Strings) ThreadBasedUserCrnProvider(com.sequenceiq.cloudbreak.auth.ThreadBasedUserCrnProvider) Service(org.springframework.stereotype.Service) DetailedEnvironmentResponse(com.sequenceiq.environment.api.v1.environment.model.response.DetailedEnvironmentResponse) SdxCloudStorageRequest(com.sequenceiq.sdx.api.model.SdxCloudStorageRequest) LoggingResponse(com.sequenceiq.common.api.telemetry.response.LoggingResponse) CloudStorageCdpService(com.sequenceiq.common.model.CloudStorageCdpService) Logger(org.slf4j.Logger) StorageIdentityBase(com.sequenceiq.common.api.cloudstorage.StorageIdentityBase) StorageLocationBase(com.sequenceiq.common.api.cloudstorage.StorageLocationBase) CloudStorageResponse(com.sequenceiq.common.api.cloudstorage.CloudStorageResponse) S3CloudStorageV1Parameters(com.sequenceiq.common.api.cloudstorage.old.S3CloudStorageV1Parameters) Collectors(java.util.stream.Collectors) FileSystemV4Endpoint(com.sequenceiq.cloudbreak.api.endpoint.v4.filesystems.FileSystemV4Endpoint) RegionAwareInternalCrnGeneratorFactory(com.sequenceiq.cloudbreak.auth.crn.RegionAwareInternalCrnGeneratorFactory) SdxClusterRequest(com.sequenceiq.sdx.api.model.SdxClusterRequest) List(java.util.List) CollectionUtils(org.springframework.util.CollectionUtils) CloudStorageRequest(com.sequenceiq.common.api.cloudstorage.CloudStorageRequest) FileSystemType(com.sequenceiq.common.model.FileSystemType) ClusterV4Response(com.sequenceiq.cloudbreak.api.endpoint.v4.stacks.response.cluster.ClusterV4Response) S3Guard(com.sequenceiq.common.api.cloudstorage.S3Guard) CloudIdentityType(com.sequenceiq.common.model.CloudIdentityType) StorageLocationBase(com.sequenceiq.common.api.cloudstorage.StorageLocationBase)

Aggregations

CloudStorageRequest (com.sequenceiq.common.api.cloudstorage.CloudStorageRequest)53 Test (org.junit.jupiter.api.Test)27 StorageLocationBase (com.sequenceiq.common.api.cloudstorage.StorageLocationBase)24 DetailedEnvironmentResponse (com.sequenceiq.environment.api.v1.environment.model.response.DetailedEnvironmentResponse)16 CloudStorageCdpService (com.sequenceiq.common.model.CloudStorageCdpService)15 SdxCloudStorageRequest (com.sequenceiq.sdx.api.model.SdxCloudStorageRequest)15 ClusterV4Request (com.sequenceiq.cloudbreak.api.endpoint.v4.stacks.request.cluster.ClusterV4Request)14 LoggingResponse (com.sequenceiq.common.api.telemetry.response.LoggingResponse)14 ArrayList (java.util.ArrayList)14 TelemetryResponse (com.sequenceiq.common.api.telemetry.response.TelemetryResponse)13 FileSystemType (com.sequenceiq.common.model.FileSystemType)13 List (java.util.List)12 StorageIdentityBase (com.sequenceiq.common.api.cloudstorage.StorageIdentityBase)11 CloudIdentityType (com.sequenceiq.common.model.CloudIdentityType)11 Assertions.assertEquals (org.junit.jupiter.api.Assertions.assertEquals)10 Assertions.assertNull (org.junit.jupiter.api.Assertions.assertNull)10 ExtendWith (org.junit.jupiter.api.extension.ExtendWith)10 InjectMocks (org.mockito.InjectMocks)10 Mock (org.mockito.Mock)10 Mockito.when (org.mockito.Mockito.when)10