Search in sources :

Example 66 with AwsParamsDto

use of org.finra.herd.model.dto.AwsParamsDto in project herd by FINRAOS.

the class StoragePolicySelectorServiceImpl method sendStoragePolicySelectionToSqsQueue.

/**
 * Sends storage policy selections to the specified AWS SQS queue.
 *
 * @param sqsQueueName the SQS queue name to send storage policy selections to
 * @param storagePolicySelections the list of storage policy selections
 */
private void sendStoragePolicySelectionToSqsQueue(String sqsQueueName, List<StoragePolicySelection> storagePolicySelections) {
    if (CollectionUtils.isNotEmpty(storagePolicySelections)) {
        AwsParamsDto awsParamsDto = awsHelper.getAwsParamsDto();
        for (StoragePolicySelection storagePolicySelection : storagePolicySelections) {
            String messageText = null;
            try {
                messageText = jsonHelper.objectToJson(storagePolicySelection);
                sqsDao.sendMessage(awsParamsDto, sqsQueueName, messageText, null);
            } catch (Exception e) {
                // Log the error and throw the exception up.
                LOGGER.error("Failed to publish message to the JMS queue. jmsQueueName=\"{}\" jmsMessagePayload={}", sqsQueueName, messageText);
                throw new IllegalStateException(e.getMessage(), e);
            }
        }
    }
}
Also used : AwsParamsDto(org.finra.herd.model.dto.AwsParamsDto) StoragePolicySelection(org.finra.herd.model.dto.StoragePolicySelection)

Example 67 with AwsParamsDto

use of org.finra.herd.model.dto.AwsParamsDto in project herd by FINRAOS.

the class UploadDownloadHelperServiceImpl method deleteSourceS3ObjectAfterError.

/**
 * Deletes a source S3 object based on the given bucket name and file path.
 *
 * @param s3BucketName the S3 bucket name
 * @param storageFilePath the storage file path
 * @param businessObjectDataKey the business object key
 */
private void deleteSourceS3ObjectAfterError(String s3BucketName, String storageFilePath, BusinessObjectDataKey businessObjectDataKey) {
    // Delete the file from S3 if storage file information exists.
    if (!StringUtils.isEmpty(storageFilePath)) {
        try {
            // Delete the source file from S3.
            AwsParamsDto awsParams = awsHelper.getAwsParamsDto();
            S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = S3FileTransferRequestParamsDto.builder().withS3BucketName(s3BucketName).withS3KeyPrefix(storageFilePath).withHttpProxyHost(awsParams.getHttpProxyHost()).withHttpProxyPort(awsParams.getHttpProxyPort()).build();
            s3Dao.deleteDirectory(s3FileTransferRequestParamsDto);
        } catch (Exception e) {
            LOGGER.error("Failed to delete source business object data file. s3Key=\"{}\" sourceS3BucketName=\"{}\" sourceBusinessObjectDataKey={}", storageFilePath, s3BucketName, jsonHelper.objectToJson(businessObjectDataKey), e);
        }
    }
}
Also used : AwsParamsDto(org.finra.herd.model.dto.AwsParamsDto) S3FileTransferRequestParamsDto(org.finra.herd.model.dto.S3FileTransferRequestParamsDto) OptimisticLockException(javax.persistence.OptimisticLockException)

Aggregations

AwsParamsDto (org.finra.herd.model.dto.AwsParamsDto)67 Test (org.junit.Test)52 InvocationOnMock (org.mockito.invocation.InvocationOnMock)13 ClusterSummary (com.amazonaws.services.elasticmapreduce.model.ClusterSummary)11 ClientConfiguration (com.amazonaws.ClientConfiguration)9 ArrayList (java.util.ArrayList)9 ListClustersResult (com.amazonaws.services.elasticmapreduce.model.ListClustersResult)8 AbstractDaoTest (org.finra.herd.dao.AbstractDaoTest)8 EmrClusterDefinition (org.finra.herd.model.api.xml.EmrClusterDefinition)7 AmazonElasticMapReduceClient (com.amazonaws.services.elasticmapreduce.AmazonElasticMapReduceClient)6 AmazonServiceException (com.amazonaws.AmazonServiceException)5 RunJobFlowRequest (com.amazonaws.services.elasticmapreduce.model.RunJobFlowRequest)5 NodeTag (org.finra.herd.model.api.xml.NodeTag)5 EmrClusterDefinitionEntity (org.finra.herd.model.jpa.EmrClusterDefinitionEntity)5 NamespaceEntity (org.finra.herd.model.jpa.NamespaceEntity)5 Cluster (com.amazonaws.services.elasticmapreduce.model.Cluster)4 DescribeClusterResult (com.amazonaws.services.elasticmapreduce.model.DescribeClusterResult)4 ListInstancesResult (com.amazonaws.services.elasticmapreduce.model.ListInstancesResult)4 List (java.util.List)4 InstanceDefinitions (org.finra.herd.model.api.xml.InstanceDefinitions)4