Examples with AwsParamsDto org.finra.herd.model.dto.AwsParamsDto used on opensource projects

Example 6 with AwsParamsDto

use of org.finra.herd.model.dto.AwsParamsDto in project herd by FINRAOS.

the class EmrDaoImplTest method testGetActiveEmrClusterByName.

@Test
public void testGetActiveEmrClusterByName() {
    // Create an AWS parameters DTO.
    AwsParamsDto awsParamsDto = new AwsParamsDto(AWS_ASSUMED_ROLE_ACCESS_KEY, AWS_ASSUMED_ROLE_SECRET_KEY, AWS_ASSUMED_ROLE_SESSION_TOKEN, HTTP_PROXY_HOST, HTTP_PROXY_PORT);
    // Create a mock AmazonElasticMapReduceClient.
    AmazonElasticMapReduceClient amazonElasticMapReduceClient = mock(AmazonElasticMapReduceClient.class);
    // Create a list cluster request.
    ListClustersRequest listClustersRequest = new ListClustersRequest().withClusterStates(EMR_VALID_STATE);
    // Create a list cluster result with a non-matching cluster and a marker.
    ListClustersResult listClusterResultWithMarker = new ListClustersResult().withClusters(new ClusterSummary().withName(INVALID_VALUE)).withMarker(MARKER);
    // Create a list cluster request with marker.
    ListClustersRequest listClustersRequestWithMarker = new ListClustersRequest().withClusterStates(EMR_VALID_STATE).withMarker(MARKER);
    // Create a cluster summary.
    ClusterSummary clusterSummary = new ClusterSummary().withName(EMR_CLUSTER_NAME);
    // Create a list cluster result with the matching cluster.
    ListClustersResult listClusterResult = new ListClustersResult().withClusters(clusterSummary);
    // Mock the external calls.
    when(configurationHelper.getProperty(ConfigurationValue.EMR_VALID_STATES)).thenReturn(EMR_VALID_STATE);
    when(configurationHelper.getProperty(ConfigurationValue.FIELD_DATA_DELIMITER)).thenReturn((String) ConfigurationValue.FIELD_DATA_DELIMITER.getDefaultValue());
    when(awsClientFactory.getEmrClient(awsParamsDto)).thenReturn(amazonElasticMapReduceClient);
    when(emrOperations.listEmrClusters(amazonElasticMapReduceClient, listClustersRequest)).thenReturn(listClusterResultWithMarker);
    when(emrOperations.listEmrClusters(amazonElasticMapReduceClient, listClustersRequestWithMarker)).thenReturn(listClusterResult);
    // Call the method under test.
    ClusterSummary result = emrDaoImpl.getActiveEmrClusterByName(EMR_CLUSTER_NAME, awsParamsDto);
    // Verify the external calls.
    verify(configurationHelper).getProperty(ConfigurationValue.EMR_VALID_STATES);
    verify(configurationHelper).getProperty(ConfigurationValue.FIELD_DATA_DELIMITER);
    verify(awsClientFactory, times(2)).getEmrClient(awsParamsDto);
    verify(emrOperations, times(2)).listEmrClusters(eq(amazonElasticMapReduceClient), any(ListClustersRequest.class));
    verifyNoMoreInteractionsHelper();
    // Validate the results.
    assertEquals(clusterSummary, result);
}

Example 7 with AwsParamsDto

use of org.finra.herd.model.dto.AwsParamsDto in project herd by FINRAOS.

the class StsDaoTest method testGetTemporarySecurityCredentialsMissingOptionalParameters.

@Test
public void testGetTemporarySecurityCredentialsMissingOptionalParameters() {
    // Create an AWS parameters DTO without proxy settings.
    AwsParamsDto awsParamsDto = new AwsParamsDto();
    // Specify the duration, in seconds, of the role session.
    int awsRoleDurationSeconds = INTEGER_VALUE;
    // Create a retry policy.
    RetryPolicy retryPolicy = new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, INTEGER_VALUE, true);
    // Create the expected assume role request.
    AssumeRoleRequest assumeRoleRequest = new AssumeRoleRequest().withRoleArn(AWS_ROLE_ARN).withRoleSessionName(SESSION_NAME).withDurationSeconds(awsRoleDurationSeconds);
    // Create AWS credentials for API authentication.
    Credentials credentials = new Credentials();
    credentials.setAccessKeyId(AWS_ASSUMED_ROLE_ACCESS_KEY);
    credentials.setSecretAccessKey(AWS_ASSUMED_ROLE_SECRET_KEY);
    credentials.setSessionToken(AWS_ASSUMED_ROLE_SESSION_TOKEN);
    // Create an assume role result.
    AssumeRoleResult assumeRoleResult = new AssumeRoleResult();
    assumeRoleResult.setCredentials(credentials);
    // Mock the external calls.
    when(retryPolicyFactory.getRetryPolicy()).thenReturn(retryPolicy);
    when(stsOperations.assumeRole(any(AWSSecurityTokenServiceClient.class), eq(assumeRoleRequest))).thenReturn(assumeRoleResult);
    // Call the method under test. Please note that we do not specify an IAM policy.
    Credentials result = stsDaoImpl.getTemporarySecurityCredentials(awsParamsDto, SESSION_NAME, AWS_ROLE_ARN, awsRoleDurationSeconds, null);
    // Verify the external calls.
    verify(retryPolicyFactory).getRetryPolicy();
    verify(stsOperations).assumeRole(any(AWSSecurityTokenServiceClient.class), eq(assumeRoleRequest));
    verifyNoMoreInteractionsHelper();
    // Validate the returned object.
    assertEquals(credentials, result);
}

Example 8 with AwsParamsDto

use of org.finra.herd.model.dto.AwsParamsDto in project herd by FINRAOS.

the class CredStashHelperTest method testGetCredentialFromCredStashEmptyPasswordValue.

@Test
public void testGetCredentialFromCredStashEmptyPasswordValue() throws Exception {
    // Build AWS parameters.
    AwsParamsDto awsParamsDto = new AwsParamsDto(NO_AWS_ACCESS_KEY, NO_AWS_SECRET_KEY, NO_SESSION_TOKEN, HTTP_PROXY_HOST, HTTP_PROXY_PORT);
    // Build AWS client configuration.
    ClientConfiguration clientConfiguration = new ClientConfiguration();
    // Create CredStash encryption context map.
    Map<String, String> credStashEncryptionContextMap = new HashMap<>();
    credStashEncryptionContextMap.put(KEY, VALUE);
    // Mock the CredStash.
    CredStash credStash = mock(CredStash.class);
    when(credStash.getCredential(USER_CREDENTIAL_NAME, credStashEncryptionContextMap)).thenReturn(EMPTY_STRING);
    // Mock the external calls.
    when(configurationHelper.getProperty(ConfigurationValue.CREDSTASH_AWS_REGION_NAME)).thenReturn(AWS_REGION_NAME);
    when(configurationHelper.getProperty(ConfigurationValue.CREDSTASH_TABLE_NAME)).thenReturn(TABLE_NAME);
    when(awsHelper.getAwsParamsDto()).thenReturn(awsParamsDto);
    when(awsHelper.getClientConfiguration(awsParamsDto)).thenReturn(clientConfiguration);
    when(credStashFactory.getCredStash(AWS_REGION_NAME, TABLE_NAME, clientConfiguration)).thenReturn(credStash);
    when(jsonHelper.unmarshallJsonToObject(Map.class, CREDSTASH_ENCRYPTION_CONTEXT)).thenReturn(credStashEncryptionContextMap);
    // Try to call the method under test.
    try {
        credStashHelper.getCredentialFromCredStash(CREDSTASH_ENCRYPTION_CONTEXT, USER_CREDENTIAL_NAME);
        fail();
    } catch (CredStashGetCredentialFailedException e) {
        assertEquals(String.format("Failed to obtain the keystore or truststore credential from credstash. " + "credStashAwsRegion=%s credStashTableName=%s credStashEncryptionContext=%s credentialName=%s", AWS_REGION_NAME, TABLE_NAME, CREDSTASH_ENCRYPTION_CONTEXT, USER_CREDENTIAL_NAME), e.getMessage());
    }
    // Verify the external calls.
    verify(configurationHelper).getProperty(ConfigurationValue.CREDSTASH_AWS_REGION_NAME);
    verify(configurationHelper).getProperty(ConfigurationValue.CREDSTASH_TABLE_NAME);
    verify(awsHelper).getAwsParamsDto();
    verify(awsHelper).getClientConfiguration(awsParamsDto);
    verify(credStashFactory).getCredStash(AWS_REGION_NAME, TABLE_NAME, clientConfiguration);
    verify(jsonHelper).unmarshallJsonToObject(Map.class, CREDSTASH_ENCRYPTION_CONTEXT);
    verify(credStash).getCredential(USER_CREDENTIAL_NAME, credStashEncryptionContextMap);
    verifyNoMoreInteractions(credStash);
    verifyNoMoreInteractionsHelper();
}

Example 9 with AwsParamsDto

use of org.finra.herd.model.dto.AwsParamsDto in project herd by FINRAOS.

the class CredStashHelperTest method testGetCredentialFromCredStashException.

@Test
public void testGetCredentialFromCredStashException() throws Exception {
    // Build AWS parameters.
    AwsParamsDto awsParamsDto = new AwsParamsDto(NO_AWS_ACCESS_KEY, NO_AWS_SECRET_KEY, NO_SESSION_TOKEN, HTTP_PROXY_HOST, HTTP_PROXY_PORT);
    // Build AWS client configuration.
    ClientConfiguration clientConfiguration = new ClientConfiguration();
    // Create CredStash encryption context map.
    Map<String, String> credStashEncryptionContextMap = new HashMap<>();
    credStashEncryptionContextMap.put(KEY, VALUE);
    // Mock the CredStash.
    CredStash credStash = mock(CredStash.class);
    when(credStash.getCredential(USER_CREDENTIAL_NAME, credStashEncryptionContextMap)).thenThrow(new Exception(ERROR_MESSAGE));
    // Mock the external calls.
    when(configurationHelper.getProperty(ConfigurationValue.CREDSTASH_AWS_REGION_NAME)).thenReturn(AWS_REGION_NAME);
    when(configurationHelper.getProperty(ConfigurationValue.CREDSTASH_TABLE_NAME)).thenReturn(TABLE_NAME);
    when(awsHelper.getAwsParamsDto()).thenReturn(awsParamsDto);
    when(awsHelper.getClientConfiguration(awsParamsDto)).thenReturn(clientConfiguration);
    when(credStashFactory.getCredStash(AWS_REGION_NAME, TABLE_NAME, clientConfiguration)).thenReturn(credStash);
    when(jsonHelper.unmarshallJsonToObject(Map.class, CREDSTASH_ENCRYPTION_CONTEXT)).thenReturn(credStashEncryptionContextMap);
    // Try to call the method under test.
    try {
        credStashHelper.getCredentialFromCredStash(CREDSTASH_ENCRYPTION_CONTEXT, USER_CREDENTIAL_NAME);
        fail();
    } catch (CredStashGetCredentialFailedException e) {
        assertEquals(String.format("Failed to obtain the keystore or truststore credential from credstash. Reason: %s " + "credStashAwsRegion=%s credStashTableName=%s credStashEncryptionContext=%s credentialName=%s", ERROR_MESSAGE, AWS_REGION_NAME, TABLE_NAME, CREDSTASH_ENCRYPTION_CONTEXT, USER_CREDENTIAL_NAME), e.getMessage());
    }
    // Verify the external calls.
    verify(configurationHelper).getProperty(ConfigurationValue.CREDSTASH_AWS_REGION_NAME);
    verify(configurationHelper).getProperty(ConfigurationValue.CREDSTASH_TABLE_NAME);
    verify(awsHelper).getAwsParamsDto();
    verify(awsHelper).getClientConfiguration(awsParamsDto);
    verify(credStashFactory).getCredStash(AWS_REGION_NAME, TABLE_NAME, clientConfiguration);
    verify(jsonHelper).unmarshallJsonToObject(Map.class, CREDSTASH_ENCRYPTION_CONTEXT);
    verify(credStash).getCredential(USER_CREDENTIAL_NAME, credStashEncryptionContextMap);
    verifyNoMoreInteractions(credStash);
    verifyNoMoreInteractionsHelper();
}

Example 10 with AwsParamsDto

use of org.finra.herd.model.dto.AwsParamsDto in project herd by FINRAOS.

the class UploadDownloadHelperServiceImpl method prepareForFileMoveImpl.

/**
 * Prepares to move an S3 file from the source bucket to the target bucket. On success, both the target and source business object data statuses are set to
 * "RE-ENCRYPTING" and the DTO is updated accordingly.
 *
 * @param objectKey the object key (i.e. filename)
 * @param completeUploadSingleParamsDto the DTO to be initialized with parameters required for complete upload single message processing
 */
protected void prepareForFileMoveImpl(String objectKey, CompleteUploadSingleParamsDto completeUploadSingleParamsDto) {
    try {
        // Obtain the source business object data entity.
        BusinessObjectDataEntity sourceBusinessObjectDataEntity = storageFileDaoHelper.getStorageFileEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, objectKey).getStorageUnit().getBusinessObjectData();
        // Get the status and key of the source business object data entity.
        completeUploadSingleParamsDto.setSourceOldStatus(sourceBusinessObjectDataEntity.getStatus().getCode());
        completeUploadSingleParamsDto.setSourceBusinessObjectDataKey(businessObjectDataHelper.getBusinessObjectDataKey(sourceBusinessObjectDataEntity));
        // Find the target business object data by the source business object data's partition value, which should have been an UUID.
        // This is assuming that the target has the same partition value as the source, and that there exist one and only one target
        // business object data for this UUID.
        BusinessObjectDataEntity targetBusinessObjectDataEntity = getTargetBusinessObjectDataEntity(sourceBusinessObjectDataEntity);
        // Get the status and key of the target business object data entity.
        completeUploadSingleParamsDto.setTargetOldStatus(targetBusinessObjectDataEntity.getStatus().getCode());
        completeUploadSingleParamsDto.setTargetBusinessObjectDataKey(businessObjectDataHelper.getBusinessObjectDataKey(targetBusinessObjectDataEntity));
        // This check effectively discards any duplicate SQS messages coming from S3 for the same uploaded file.
        for (BusinessObjectDataEntity businessObjectDataEntity : Arrays.asList(sourceBusinessObjectDataEntity, targetBusinessObjectDataEntity)) {
            if (!BusinessObjectDataStatusEntity.UPLOADING.equals(businessObjectDataEntity.getStatus().getCode())) {
                LOGGER.info("Ignoring S3 notification since business object data status \"{}\" does not match the expected status \"{}\". " + "businessObjectDataKey={}", businessObjectDataEntity.getStatus().getCode(), BusinessObjectDataStatusEntity.UPLOADING, jsonHelper.objectToJson(businessObjectDataHelper.getBusinessObjectDataKey(businessObjectDataEntity)));
                // method skip the rest of the steps required to complete the upload single message processing.
                return;
            }
        }
        // Get the S3 managed "loading dock" storage entity and make sure it exists.
        StorageEntity s3ManagedLoadingDockStorageEntity = storageDaoHelper.getStorageEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE);
        // Get bucket name for S3 managed "loading dock" storage. Please note that this attribute value is required.
        completeUploadSingleParamsDto.setSourceBucketName(storageHelper.getStorageBucketName(s3ManagedLoadingDockStorageEntity));
        // Get the storage unit entity for this business object data in the S3 managed "loading dock" storage and make sure it exists.
        StorageUnitEntity sourceStorageUnitEntity = storageUnitDaoHelper.getStorageUnitEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, sourceBusinessObjectDataEntity);
        // Get the storage file entity.
        StorageFileEntity sourceStorageFileEntity = IterableUtils.get(sourceStorageUnitEntity.getStorageFiles(), 0);
        // Get the source storage file path.
        completeUploadSingleParamsDto.setSourceFilePath(sourceStorageFileEntity.getPath());
        // Get the AWS parameters.
        AwsParamsDto awsParamsDto = awsHelper.getAwsParamsDto();
        completeUploadSingleParamsDto.setAwsParams(awsParamsDto);
        // Validate the source S3 file.
        S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = S3FileTransferRequestParamsDto.builder().withS3BucketName(completeUploadSingleParamsDto.getSourceBucketName()).withS3KeyPrefix(completeUploadSingleParamsDto.getSourceFilePath()).withHttpProxyHost(awsParamsDto.getHttpProxyHost()).withHttpProxyPort(awsParamsDto.getHttpProxyPort()).build();
        s3Dao.validateS3File(s3FileTransferRequestParamsDto, sourceStorageFileEntity.getFileSizeBytes());
        // Get the S3 managed "external" storage entity and make sure it exists.
        StorageEntity s3ManagedExternalStorageEntity = getUniqueStorage(targetBusinessObjectDataEntity);
        // Get bucket name for S3 managed "external" storage. Please note that this attribute value is required.
        completeUploadSingleParamsDto.setTargetBucketName(storageHelper.getStorageBucketName(s3ManagedExternalStorageEntity));
        // Get AWS KMS External Key ID.
        completeUploadSingleParamsDto.setKmsKeyId(storageHelper.getStorageKmsKeyId(s3ManagedExternalStorageEntity));
        // Make sure the target does not already contain the file.
        completeUploadSingleParamsDto.setTargetFilePath(IterableUtils.get(IterableUtils.get(targetBusinessObjectDataEntity.getStorageUnits(), 0).getStorageFiles(), 0).getPath());
        assertS3ObjectKeyDoesNotExist(completeUploadSingleParamsDto.getTargetBucketName(), completeUploadSingleParamsDto.getTargetFilePath());
        try {
            // Change the status of the source and target business object data to RE-ENCRYPTING.
            businessObjectDataDaoHelper.updateBusinessObjectDataStatus(sourceBusinessObjectDataEntity, BusinessObjectDataStatusEntity.RE_ENCRYPTING);
            businessObjectDataDaoHelper.updateBusinessObjectDataStatus(targetBusinessObjectDataEntity, BusinessObjectDataStatusEntity.RE_ENCRYPTING);
        }// caught by a business object data status check that occurs inside the prepareForFileMove() helper method.
         catch (OptimisticLockException e) {
            LOGGER.info("Ignoring S3 notification due to an optimistic lock exception caused by duplicate S3 event notifications. " + "sourceBusinessObjectDataKey={} targetBusinessObjectDataKey={}", jsonHelper.objectToJson(completeUploadSingleParamsDto.getSourceBusinessObjectDataKey()), jsonHelper.objectToJson(completeUploadSingleParamsDto.getTargetBusinessObjectDataKey()));
            // method skip the rest of the steps required to complete the upload single message processing.
            return;
        }
        // Set new status for the source and target business object data in the DTO.
        completeUploadSingleParamsDto.setSourceNewStatus(BusinessObjectDataStatusEntity.RE_ENCRYPTING);
        completeUploadSingleParamsDto.setTargetNewStatus(BusinessObjectDataStatusEntity.RE_ENCRYPTING);
    } catch (RuntimeException e) {
        // Update statuses for both the source and target business object data instances.
        completeUploadSingleParamsDto.setSourceNewStatus(setAndReturnNewSourceBusinessObjectDataStatusAfterError(completeUploadSingleParamsDto.getSourceBusinessObjectDataKey()));
        // Update statuses for both the source and target business object data instances.
        completeUploadSingleParamsDto.setTargetNewStatus(setAndReturnNewTargetBusinessObjectDataStatusAfterError(completeUploadSingleParamsDto.getTargetBusinessObjectDataKey()));
        // Delete the source S3 file. Please note that the method below only logs runtime exceptions without re-throwing them.
        deleteSourceS3ObjectAfterError(completeUploadSingleParamsDto.getSourceBucketName(), completeUploadSingleParamsDto.getSourceFilePath(), completeUploadSingleParamsDto.getSourceBusinessObjectDataKey());
        // Log the error.
        LOGGER.error("Failed to process upload single completion request for file. s3Key=\"{}\"", objectKey, e);
    }
    // If a status update occurred for the source business object data, create a business object data notification for this event.
    if (completeUploadSingleParamsDto.getSourceNewStatus() != null) {
        notificationEventService.processBusinessObjectDataNotificationEventAsync(NotificationEventTypeEntity.EventTypesBdata.BUS_OBJCT_DATA_STTS_CHG, completeUploadSingleParamsDto.getSourceBusinessObjectDataKey(), completeUploadSingleParamsDto.getSourceNewStatus(), completeUploadSingleParamsDto.getSourceOldStatus());
    }
    // If a status update occurred for the target business object data, create a business object data notification for this event.
    if (completeUploadSingleParamsDto.getTargetNewStatus() != null) {
        notificationEventService.processBusinessObjectDataNotificationEventAsync(NotificationEventTypeEntity.EventTypesBdata.BUS_OBJCT_DATA_STTS_CHG, completeUploadSingleParamsDto.getTargetBusinessObjectDataKey(), completeUploadSingleParamsDto.getTargetNewStatus(), completeUploadSingleParamsDto.getTargetOldStatus());
    }
}