use of com.sequenceiq.common.api.cloudstorage.CloudStorageRequest in project cloudbreak by hortonworks.
the class CloudStorageManifesterTest method whenEnvironmentHasLoggingEnabledThenShouldApplyAsLogIdentity.
@Test
public void whenEnvironmentHasLoggingEnabledThenShouldApplyAsLogIdentity() {
when(regionAwareInternalCrnGenerator.getInternalCrnForServiceAsString()).thenReturn("crn");
when(regionAwareInternalCrnGeneratorFactory.iam()).thenReturn(regionAwareInternalCrnGenerator);
mockFileSystemResponseForCloudbreakClient();
SdxCluster sdxCluster = new SdxCluster();
SdxClusterRequest sdxClusterRequest = new SdxClusterRequest();
sdxCluster.setInitiatorUserCrn(USER_CRN);
sdxCluster.setClusterName("sdx-cluster");
SdxCloudStorageRequest cloudStorageRequest = new SdxCloudStorageRequest();
cloudStorageRequest.setBaseLocation("s3a://example-path");
cloudStorageRequest.setFileSystemType(FileSystemType.S3);
S3CloudStorageV1Parameters s3Params = new S3CloudStorageV1Parameters();
s3Params.setInstanceProfile("instance:profile");
cloudStorageRequest.setS3(s3Params);
sdxClusterRequest.setCloudStorage(cloudStorageRequest);
DetailedEnvironmentResponse environment = new DetailedEnvironmentResponse();
environment.setCloudPlatform("AWS");
TelemetryResponse telemetryResponse = new TelemetryResponse();
LoggingResponse loggingResponse = new LoggingResponse();
S3CloudStorageV1Parameters s3CloudStorageV1Parameters = new S3CloudStorageV1Parameters();
s3CloudStorageV1Parameters.setInstanceProfile("logprofile");
loggingResponse.setS3(s3CloudStorageV1Parameters);
telemetryResponse.setLogging(loggingResponse);
AwsEnvironmentParameters awsEnvironmentParameters = new AwsEnvironmentParameters();
S3GuardRequestParameters s3GuardRequestParameters = new S3GuardRequestParameters();
s3GuardRequestParameters.setDynamoDbTableName("table");
awsEnvironmentParameters.setS3guard(s3GuardRequestParameters);
environment.setAws(awsEnvironmentParameters);
environment.setTelemetry(telemetryResponse);
ClusterV4Request clusterV4Request = new ClusterV4Request();
clusterV4Request.setBlueprintName(exampleBlueprintName);
CloudStorageRequest cloudStorageConfigReq = ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.initCloudStorageRequest(environment, clusterV4Request, sdxCluster, sdxClusterRequest));
StorageLocationBase singleRequest = cloudStorageConfigReq.getLocations().iterator().next();
assertEquals(2, cloudStorageConfigReq.getIdentities().size());
assertEquals(1, cloudStorageConfigReq.getIdentities().stream().filter(r -> r.getType().equals(CloudIdentityType.ID_BROKER)).collect(Collectors.toSet()).size());
assertEquals(1, cloudStorageConfigReq.getIdentities().stream().filter(r -> r.getType().equals(CloudIdentityType.LOG)).collect(Collectors.toSet()).size());
assertEquals("table", cloudStorageConfigReq.getAws().getS3Guard().getDynamoTableName());
assertEquals(1, cloudStorageConfigReq.getLocations().size());
assertEquals(CloudStorageCdpService.RANGER_AUDIT, singleRequest.getType());
assertEquals("ranger/example-path", singleRequest.getValue());
verify(storageValidationService).validateCloudStorage("AWS", cloudStorageRequest);
}
use of com.sequenceiq.common.api.cloudstorage.CloudStorageRequest in project cloudbreak by hortonworks.
the class SdxService method prepareCloudStorageForStack.
private void prepareCloudStorageForStack(SdxClusterRequest sdxClusterRequest, StackV4Request stackV4Request, SdxCluster sdxCluster, DetailedEnvironmentResponse environment) {
CloudStorageRequest cloudStorageRequest = cloudStorageManifester.initCloudStorageRequest(environment, stackV4Request.getCluster(), sdxCluster, sdxClusterRequest);
stackV4Request.getCluster().setCloudStorage(cloudStorageRequest);
}
use of com.sequenceiq.common.api.cloudstorage.CloudStorageRequest in project cloudbreak by hortonworks.
the class StackRequestManifester method setupCloudStorageAccountMapping.
@VisibleForTesting
void setupCloudStorageAccountMapping(StackV4Request stackRequest, String environmentCrn, IdBrokerMappingSource mappingSource, String cloudPlatform) {
String stackName = stackRequest.getName();
CloudStorageRequest cloudStorage = stackRequest.getCluster().getCloudStorage();
if (cloudStorage != null && cloudStorage.getAccountMapping() == null) {
// getAccountMapping() == null means we need to fetch mappings from IDBMMS.
if (mappingSource == IdBrokerMappingSource.IDBMMS) {
LOGGER.info("Fetching account mappings from IDBMMS associated with environment {} for stack {}.", environmentCrn, stackName);
MappingsConfig mappingsConfig;
try {
// Must pass the internal actor here as this operation is internal-use only; requests with other actors will be always rejected.
mappingsConfig = idbmmsClient.getMappingsConfig(regionAwareInternalCrnGeneratorFactory.iam().getInternalCrnForServiceAsString(), environmentCrn, Optional.empty());
validateMappingsConfig(mappingsConfig, stackRequest);
} catch (IdbmmsOperationException e) {
throw new BadRequestException(String.format("Unable to get mappings: %s", e.getMessage()), e);
}
AccountMappingBase accountMapping = new AccountMappingBase();
accountMapping.setGroupMappings(mappingsConfig.getGroupMappings());
accountMapping.setUserMappings(mappingsConfig.getActorMappings());
cloudStorage.setAccountMapping(accountMapping);
LOGGER.info("Initial account mappings fetched from IDBMMS: {}", JsonUtil.writeValueAsStringSilent(accountMapping));
} else {
LOGGER.info("IDBMMS usage is disabled for environment {}. Proceeding with {} mappings for stack {}.", environmentCrn, mappingSource == IdBrokerMappingSource.MOCK && (CloudPlatform.AWS.name().equals(cloudPlatform) || CloudPlatform.AZURE.name().equals(cloudPlatform) || CloudPlatform.GCP.name().equals(cloudPlatform)) ? "mock" : "missing", stackName);
}
} else {
// getAccountMapping() != null is possible only in case of SdxInternalClusterRequest, in which case the user-given values will be honored.
LOGGER.info("{} for stack {} in environment {}.", cloudStorage == null ? "Cloud storage is disabled" : "Applying user-provided mappings", stackName, environmentCrn);
}
}
use of com.sequenceiq.common.api.cloudstorage.CloudStorageRequest in project cloudbreak by hortonworks.
the class StorageValidationService method validateObjectStorage.
public ObjectStorageValidateResponse validateObjectStorage(String credentialCrn, SdxCloudStorageRequest sdxCloudStorageRequest, String blueprintName, String clusterName, String dataAccessRole, String rangerAuditRole) {
CredentialResponse credentialResponse = environmentClientService.getCredentialByCrn(credentialCrn);
String attributes = secretService.getByResponse(credentialResponse.getAttributes());
CloudCredential cloudCredential = new CloudCredential(credentialResponse.getCrn(), credentialResponse.getName(), new Json(attributes).getMap(), credentialResponse.getAccountId(), credentialResponse.isVerifyPermissions());
CloudStorageRequest cloudStorageRequest = cloudStorageManifester.initSdxCloudStorageRequest(credentialResponse.getCloudPlatform(), blueprintName, clusterName, sdxCloudStorageRequest);
AccountMappingBase accountMapping = new AccountMappingBase();
Map<String, String> userMapping = getUserMapping(dataAccessRole, rangerAuditRole);
accountMapping.setUserMappings(userMapping);
cloudStorageRequest.setAccountMapping(accountMapping);
ObjectStorageValidateRequest objectStorageValidateRequest = ObjectStorageValidateRequest.builder().withCloudPlatform(credentialResponse.getCloudPlatform()).withCredential(cloudCredential).withCloudStorageRequest(cloudStorageRequest).build();
return ThreadBasedUserCrnProvider.doAsInternalActor(regionAwareInternalCrnGeneratorFactory.iam().getInternalCrnForServiceAsString(), () -> cloudProviderServicesV4Endpoint.validateObjectStorage(objectStorageValidateRequest));
}
use of com.sequenceiq.common.api.cloudstorage.CloudStorageRequest in project cloudbreak by hortonworks.
the class CloudStorageManifester method setStorageLocations.
private void setStorageLocations(FileSystemParameterV4Responses fileSystemRecommendations, CloudStorageRequest cloudStorageRequest) {
List<StorageLocationBase> storageLocations = fileSystemRecommendations.getResponses().stream().map(response -> {
StorageLocationBase storageLocation = new StorageLocationBase();
storageLocation.setValue(response.getDefaultPath());
storageLocation.setType(CloudStorageCdpService.valueOf(response.getType()));
return storageLocation;
}).collect(Collectors.toList());
cloudStorageRequest.setLocations(storageLocations);
}
Aggregations