Examples with AwsParamsDto org.finra.herd.model.dto.AwsParamsDto used on opensource projects

Example 1 with AwsParamsDto

use of org.finra.herd.model.dto.AwsParamsDto in project herd by FINRAOS.

the class ServiceSpringModuleConfig method jmsConnectionFactory.

/**
 * Gets a JMS connection factory.
 *
 * @return the JMS connection factory.
 */
@Bean
public ConnectionFactory jmsConnectionFactory() {
    AwsParamsDto awsParamsDto = awsHelper.getAwsParamsDto();
    ClientConfiguration clientConfiguration = new ClientConfiguration();
    // Only set the proxy hostname and/or port if they're configured.
    if (StringUtils.isNotBlank(awsParamsDto.getHttpProxyHost())) {
        clientConfiguration.setProxyHost(awsParamsDto.getHttpProxyHost());
    }
    if (awsParamsDto.getHttpProxyPort() != null) {
        clientConfiguration.setProxyPort(awsParamsDto.getHttpProxyPort());
    }
    return SQSConnectionFactory.builder().withClientConfiguration(clientConfiguration).build();
}

Example 2 with AwsParamsDto

use of org.finra.herd.model.dto.AwsParamsDto in project herd by FINRAOS.

the class UploadDownloadHelperServiceImplTest method testPrepareForFileMove.

@Test
public void testPrepareForFileMove() {
    // Create an object key.
    String objectKey = UUID_VALUE;
    // Create a complete upload single parameters DTO.
    CompleteUploadSingleParamsDto completeUploadSingleParamsDto = new CompleteUploadSingleParamsDto();
    // Create a business object data status entity.
    BusinessObjectDataStatusEntity businessObjectDataStatusEntity = new BusinessObjectDataStatusEntity();
    businessObjectDataStatusEntity.setCode(BusinessObjectDataStatusEntity.UPLOADING);
    // Create a source storage entity.
    StorageEntity sourceStorageEntity = new StorageEntity();
    // Create a source business object data key.
    BusinessObjectDataKey sourceBusinessObjectDataKey = new BusinessObjectDataKey();
    sourceBusinessObjectDataKey.setBusinessObjectFormatUsage(FORMAT_USAGE_CODE);
    // Create a source business object data entity.
    BusinessObjectDataEntity sourceBusinessObjectDataEntity = new BusinessObjectDataEntity();
    sourceBusinessObjectDataEntity.setId(ID);
    sourceBusinessObjectDataEntity.setPartitionValue(objectKey);
    sourceBusinessObjectDataEntity.setStatus(businessObjectDataStatusEntity);
    // Create a list of source storage files.
    List<StorageFileEntity> sourceStorageFileEntities = new ArrayList<>();
    // Create a source storage unit.
    StorageUnitEntity sourceStorageUnitEntity = new StorageUnitEntity();
    sourceStorageUnitEntity.setBusinessObjectData(sourceBusinessObjectDataEntity);
    sourceStorageUnitEntity.setStorage(sourceStorageEntity);
    sourceStorageUnitEntity.setStorageFiles(sourceStorageFileEntities);
    // Create a source storage file entity.
    StorageFileEntity sourceStorageFileEntity = new StorageFileEntity();
    sourceStorageFileEntities.add(sourceStorageFileEntity);
    sourceStorageFileEntity.setStorageUnit(sourceStorageUnitEntity);
    sourceStorageFileEntity.setPath(S3_KEY);
    sourceStorageFileEntity.setFileSizeBytes(FILE_SIZE);
    // Create a target storage entity.
    StorageEntity targetStorageEntity = new StorageEntity();
    // Create a target business object data key.
    BusinessObjectDataKey targetBusinessObjectDataKey = new BusinessObjectDataKey();
    targetBusinessObjectDataKey.setBusinessObjectFormatUsage(FORMAT_USAGE_CODE_2);
    // Create a list of source storage files.
    List<StorageFileEntity> targetStorageFileEntities = new ArrayList<>();
    // Create a target storage unit.
    StorageUnitEntity targetStorageUnitEntity = new StorageUnitEntity();
    targetStorageUnitEntity.setStorage(targetStorageEntity);
    targetStorageUnitEntity.setStorageFiles(targetStorageFileEntities);
    // Create a source storage file entity.
    StorageFileEntity targetStorageFileEntity = new StorageFileEntity();
    targetStorageFileEntities.add(targetStorageFileEntity);
    targetStorageFileEntity.setPath(S3_KEY_2);
    // Create a target business object data entity.
    BusinessObjectDataEntity targetBusinessObjectDataEntity = new BusinessObjectDataEntity();
    targetBusinessObjectDataEntity.setId(ID_2);
    targetBusinessObjectDataEntity.setStatus(businessObjectDataStatusEntity);
    targetBusinessObjectDataEntity.setStorageUnits(Collections.singletonList(targetStorageUnitEntity));
    // Create an AWS parameters DTO.
    AwsParamsDto awsParamsDto = new AwsParamsDto(NO_AWS_ACCESS_KEY, NO_AWS_SECRET_KEY, NO_SESSION_TOKEN, HTTP_PROXY_HOST, HTTP_PROXY_PORT);
    // Mock the external calls.
    when(storageFileDaoHelper.getStorageFileEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, objectKey)).thenReturn(sourceStorageFileEntity);
    when(businessObjectDataHelper.getBusinessObjectDataKey(sourceBusinessObjectDataEntity)).thenReturn(sourceBusinessObjectDataKey);
    when(businessObjectDataDao.getBusinessObjectDataEntitiesByPartitionValue(objectKey)).thenReturn(Arrays.asList(sourceBusinessObjectDataEntity, targetBusinessObjectDataEntity));
    when(businessObjectDataHelper.getBusinessObjectDataKey(targetBusinessObjectDataEntity)).thenReturn(targetBusinessObjectDataKey);
    when(storageDaoHelper.getStorageEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE)).thenReturn(sourceStorageEntity);
    when(storageHelper.getStorageBucketName(sourceStorageEntity)).thenReturn(S3_BUCKET_NAME);
    when(storageUnitDaoHelper.getStorageUnitEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, sourceBusinessObjectDataEntity)).thenReturn(sourceStorageUnitEntity);
    when(awsHelper.getAwsParamsDto()).thenReturn(awsParamsDto);
    when(storageHelper.getStorageBucketName(targetStorageEntity)).thenReturn(S3_BUCKET_NAME_2);
    when(storageHelper.getStorageKmsKeyId(targetStorageEntity)).thenReturn(AWS_KMS_KEY_ID);
    // Call the method under test.
    uploadDownloadHelperService.prepareForFileMoveImpl(objectKey, completeUploadSingleParamsDto);
    // Verify the external calls.
    verify(storageFileDaoHelper).getStorageFileEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, objectKey);
    verify(storageFileDaoHelper).getStorageFileEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, objectKey);
    verify(businessObjectDataHelper).getBusinessObjectDataKey(sourceBusinessObjectDataEntity);
    verify(businessObjectDataDao).getBusinessObjectDataEntitiesByPartitionValue(objectKey);
    verify(businessObjectDataHelper).getBusinessObjectDataKey(targetBusinessObjectDataEntity);
    verify(storageDaoHelper).getStorageEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE);
    verify(storageHelper).getStorageBucketName(sourceStorageEntity);
    verify(storageUnitDaoHelper).getStorageUnitEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, sourceBusinessObjectDataEntity);
    verify(awsHelper, times(2)).getAwsParamsDto();
    verify(s3Dao).validateS3File(any(S3FileTransferRequestParamsDto.class), eq(FILE_SIZE));
    verify(storageHelper).getStorageBucketName(targetStorageEntity);
    verify(storageHelper).getStorageKmsKeyId(targetStorageEntity);
    verify(s3Dao).s3FileExists(any(S3FileTransferRequestParamsDto.class));
    verify(businessObjectDataDaoHelper).updateBusinessObjectDataStatus(sourceBusinessObjectDataEntity, BusinessObjectDataStatusEntity.RE_ENCRYPTING);
    verify(businessObjectDataDaoHelper).updateBusinessObjectDataStatus(targetBusinessObjectDataEntity, BusinessObjectDataStatusEntity.RE_ENCRYPTING);
    verify(notificationEventService).processBusinessObjectDataNotificationEventAsync(NotificationEventTypeEntity.EventTypesBdata.BUS_OBJCT_DATA_STTS_CHG, sourceBusinessObjectDataKey, BusinessObjectDataStatusEntity.RE_ENCRYPTING, BusinessObjectDataStatusEntity.UPLOADING);
    verify(notificationEventService).processBusinessObjectDataNotificationEventAsync(NotificationEventTypeEntity.EventTypesBdata.BUS_OBJCT_DATA_STTS_CHG, targetBusinessObjectDataKey, BusinessObjectDataStatusEntity.RE_ENCRYPTING, BusinessObjectDataStatusEntity.UPLOADING);
    verifyNoMoreInteractionsHelper();
    // Validate the results.
    assertEquals(new CompleteUploadSingleParamsDto(sourceBusinessObjectDataKey, S3_BUCKET_NAME, S3_KEY, BusinessObjectDataStatusEntity.UPLOADING, BusinessObjectDataStatusEntity.RE_ENCRYPTING, targetBusinessObjectDataKey, S3_BUCKET_NAME_2, S3_KEY_2, BusinessObjectDataStatusEntity.UPLOADING, BusinessObjectDataStatusEntity.RE_ENCRYPTING, AWS_KMS_KEY_ID, awsParamsDto), completeUploadSingleParamsDto);
}

Example 3 with AwsParamsDto

use of org.finra.herd.model.dto.AwsParamsDto in project herd by FINRAOS.

the class UploadDownloadHelperServiceImplTest method testPrepareForFileMoveImplOptimisticLockException.

@Test
public void testPrepareForFileMoveImplOptimisticLockException() {
    // Create an object key.
    String objectKey = UUID_VALUE;
    // Create a complete upload single parameters DTO.
    CompleteUploadSingleParamsDto completeUploadSingleParamsDto = new CompleteUploadSingleParamsDto();
    // Create a business object data status entity.
    BusinessObjectDataStatusEntity businessObjectDataStatusEntity = new BusinessObjectDataStatusEntity();
    businessObjectDataStatusEntity.setCode(BusinessObjectDataStatusEntity.UPLOADING);
    // Create a source storage entity.
    StorageEntity sourceStorageEntity = new StorageEntity();
    // Create a source business object data key.
    BusinessObjectDataKey sourceBusinessObjectDataKey = new BusinessObjectDataKey();
    sourceBusinessObjectDataKey.setBusinessObjectFormatUsage(FORMAT_USAGE_CODE);
    // Create a source business object data entity.
    BusinessObjectDataEntity sourceBusinessObjectDataEntity = new BusinessObjectDataEntity();
    sourceBusinessObjectDataEntity.setId(ID);
    sourceBusinessObjectDataEntity.setPartitionValue(objectKey);
    sourceBusinessObjectDataEntity.setStatus(businessObjectDataStatusEntity);
    // Create a list of source storage files.
    List<StorageFileEntity> sourceStorageFileEntities = new ArrayList<>();
    // Create a source storage unit.
    StorageUnitEntity sourceStorageUnitEntity = new StorageUnitEntity();
    sourceStorageUnitEntity.setBusinessObjectData(sourceBusinessObjectDataEntity);
    sourceStorageUnitEntity.setStorage(sourceStorageEntity);
    sourceStorageUnitEntity.setStorageFiles(sourceStorageFileEntities);
    // Create a source storage file entity.
    StorageFileEntity sourceStorageFileEntity = new StorageFileEntity();
    sourceStorageFileEntities.add(sourceStorageFileEntity);
    sourceStorageFileEntity.setStorageUnit(sourceStorageUnitEntity);
    sourceStorageFileEntity.setPath(S3_KEY);
    sourceStorageFileEntity.setFileSizeBytes(FILE_SIZE);
    // Create a target storage entity.
    StorageEntity targetStorageEntity = new StorageEntity();
    // Create a target business object data key.
    BusinessObjectDataKey targetBusinessObjectDataKey = new BusinessObjectDataKey();
    targetBusinessObjectDataKey.setBusinessObjectFormatUsage(FORMAT_USAGE_CODE_2);
    // Create a list of source storage files.
    List<StorageFileEntity> targetStorageFileEntities = new ArrayList<>();
    // Create a target storage unit.
    StorageUnitEntity targetStorageUnitEntity = new StorageUnitEntity();
    targetStorageUnitEntity.setStorage(targetStorageEntity);
    targetStorageUnitEntity.setStorageFiles(targetStorageFileEntities);
    // Create a source storage file entity.
    StorageFileEntity targetStorageFileEntity = new StorageFileEntity();
    targetStorageFileEntities.add(targetStorageFileEntity);
    targetStorageFileEntity.setPath(S3_KEY_2);
    // Create a target business object data entity.
    BusinessObjectDataEntity targetBusinessObjectDataEntity = new BusinessObjectDataEntity();
    targetBusinessObjectDataEntity.setId(ID_2);
    targetBusinessObjectDataEntity.setStatus(businessObjectDataStatusEntity);
    targetBusinessObjectDataEntity.setStorageUnits(Collections.singletonList(targetStorageUnitEntity));
    // Create an AWS parameters DTO.
    AwsParamsDto awsParamsDto = new AwsParamsDto(NO_AWS_ACCESS_KEY, NO_AWS_SECRET_KEY, NO_SESSION_TOKEN, HTTP_PROXY_HOST, HTTP_PROXY_PORT);
    // Mock the external calls.
    when(storageFileDaoHelper.getStorageFileEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, objectKey)).thenReturn(sourceStorageFileEntity);
    when(businessObjectDataHelper.getBusinessObjectDataKey(sourceBusinessObjectDataEntity)).thenReturn(sourceBusinessObjectDataKey);
    when(businessObjectDataDao.getBusinessObjectDataEntitiesByPartitionValue(objectKey)).thenReturn(Arrays.asList(sourceBusinessObjectDataEntity, targetBusinessObjectDataEntity));
    when(businessObjectDataHelper.getBusinessObjectDataKey(targetBusinessObjectDataEntity)).thenReturn(targetBusinessObjectDataKey);
    when(storageDaoHelper.getStorageEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE)).thenReturn(sourceStorageEntity);
    when(storageHelper.getStorageBucketName(sourceStorageEntity)).thenReturn(S3_BUCKET_NAME);
    when(storageUnitDaoHelper.getStorageUnitEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, sourceBusinessObjectDataEntity)).thenReturn(sourceStorageUnitEntity);
    when(awsHelper.getAwsParamsDto()).thenReturn(awsParamsDto);
    when(storageHelper.getStorageBucketName(targetStorageEntity)).thenReturn(S3_BUCKET_NAME_2);
    when(storageHelper.getStorageKmsKeyId(targetStorageEntity)).thenReturn(AWS_KMS_KEY_ID);
    doThrow(new OptimisticLockException(ERROR_MESSAGE)).when(businessObjectDataDaoHelper).updateBusinessObjectDataStatus(sourceBusinessObjectDataEntity, BusinessObjectDataStatusEntity.RE_ENCRYPTING);
    when(jsonHelper.objectToJson(sourceBusinessObjectDataKey)).thenReturn(BUSINESS_OBJECT_DATA_KEY_AS_STRING);
    when(jsonHelper.objectToJson(targetBusinessObjectDataKey)).thenReturn(BUSINESS_OBJECT_DATA_KEY_AS_STRING_2);
    // Try to call the method under test.
    try {
        uploadDownloadHelperService.prepareForFileMoveImpl(objectKey, completeUploadSingleParamsDto);
    } catch (OptimisticLockException e) {
        assertEquals(String.format("Ignoring S3 notification due to an optimistic lock exception caused by duplicate S3 event notifications. " + "sourceBusinessObjectDataKey=%s targetBusinessObjectDataKey=%s", BUSINESS_OBJECT_DATA_KEY_AS_STRING, BUSINESS_OBJECT_DATA_KEY_AS_STRING_2), e.getMessage());
    }
    // Verify the external calls.
    verify(storageFileDaoHelper).getStorageFileEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, objectKey);
    verify(storageFileDaoHelper).getStorageFileEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, objectKey);
    verify(businessObjectDataHelper).getBusinessObjectDataKey(sourceBusinessObjectDataEntity);
    verify(businessObjectDataDao).getBusinessObjectDataEntitiesByPartitionValue(objectKey);
    verify(businessObjectDataHelper).getBusinessObjectDataKey(targetBusinessObjectDataEntity);
    verify(storageDaoHelper).getStorageEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE);
    verify(storageHelper).getStorageBucketName(sourceStorageEntity);
    verify(storageUnitDaoHelper).getStorageUnitEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, sourceBusinessObjectDataEntity);
    verify(awsHelper, times(2)).getAwsParamsDto();
    verify(s3Dao).validateS3File(any(S3FileTransferRequestParamsDto.class), eq(FILE_SIZE));
    verify(storageHelper).getStorageBucketName(targetStorageEntity);
    verify(storageHelper).getStorageKmsKeyId(targetStorageEntity);
    verify(s3Dao).s3FileExists(any(S3FileTransferRequestParamsDto.class));
    verify(businessObjectDataDaoHelper).updateBusinessObjectDataStatus(sourceBusinessObjectDataEntity, BusinessObjectDataStatusEntity.RE_ENCRYPTING);
    verify(jsonHelper).objectToJson(sourceBusinessObjectDataKey);
    verify(jsonHelper).objectToJson(targetBusinessObjectDataKey);
    verifyNoMoreInteractionsHelper();
    // Validate the results.
    assertEquals(new CompleteUploadSingleParamsDto(sourceBusinessObjectDataKey, S3_BUCKET_NAME, S3_KEY, BusinessObjectDataStatusEntity.UPLOADING, NO_BDATA_STATUS, targetBusinessObjectDataKey, S3_BUCKET_NAME_2, S3_KEY_2, BusinessObjectDataStatusEntity.UPLOADING, NO_BDATA_STATUS, AWS_KMS_KEY_ID, awsParamsDto), completeUploadSingleParamsDto);
}

Example 4 with AwsParamsDto

use of org.finra.herd.model.dto.AwsParamsDto in project herd by FINRAOS.

the class AwsHelperTest method testGetClientConfiguration.

@Test
public void testGetClientConfiguration() throws Exception {
    // Try to get AWS parameters using all possible permutations of HTTP proxy settings.
    for (String testHttpProxyHost : Arrays.asList(STRING_VALUE, BLANK_TEXT, null)) {
        for (Integer testHttpProxyPort : Arrays.asList(INTEGER_VALUE, null)) {
            // Create AWS parameters DTO.
            AwsParamsDto testAwsParamsDto = awsHelper.getAwsParamsDto();
            testAwsParamsDto.setHttpProxyHost(testHttpProxyHost);
            testAwsParamsDto.setHttpProxyPort(testHttpProxyPort);
            // Get client configuration.
            ClientConfiguration resultClientConfiguration = awsHelper.getClientConfiguration(testAwsParamsDto);
            // Validate the results.
            assertNotNull(resultClientConfiguration);
            // The proxy settings are set only when both host and port are specified in the AWS parameters DTO.
            if (STRING_VALUE.equals(testHttpProxyHost) && INTEGER_VALUE.equals(testHttpProxyPort)) {
                assertEquals(testHttpProxyHost, resultClientConfiguration.getProxyHost());
                assertEquals(testHttpProxyPort, Integer.valueOf(resultClientConfiguration.getProxyPort()));
            } else {
                assertNull(resultClientConfiguration.getProxyHost());
                assertEquals(-1, resultClientConfiguration.getProxyPort());
            }
        }
    }
}

Example 5 with AwsParamsDto

use of org.finra.herd.model.dto.AwsParamsDto in project herd by FINRAOS.

the class EmrDaoImplTest method testGetListInstanceFleetsResult.

@Test
public void testGetListInstanceFleetsResult() {
    // Create an AWS parameters DTO.
    AwsParamsDto awsParamsDto = new AwsParamsDto(AWS_ASSUMED_ROLE_ACCESS_KEY, AWS_ASSUMED_ROLE_SECRET_KEY, AWS_ASSUMED_ROLE_SESSION_TOKEN, HTTP_PROXY_HOST, HTTP_PROXY_PORT);
    // Create a mock AmazonElasticMapReduceClient.
    AmazonElasticMapReduceClient amazonElasticMapReduceClient = mock(AmazonElasticMapReduceClient.class);
    // Create a list instance fleets request.
    ListInstanceFleetsRequest listInstanceFleetsRequest = new ListInstanceFleetsRequest().withClusterId(EMR_CLUSTER_ID);
    // Create a list instance fleets result.
    ListInstanceFleetsResult listInstanceFleetsResult = new ListInstanceFleetsResult().withMarker(MARKER);
    // Mock the external calls.
    when(awsClientFactory.getEmrClient(awsParamsDto)).thenReturn(amazonElasticMapReduceClient);
    when(emrOperations.listInstanceFleets(amazonElasticMapReduceClient, listInstanceFleetsRequest)).thenReturn(listInstanceFleetsResult);
    // Call the method under test.
    ListInstanceFleetsResult result = emrDaoImpl.getListInstanceFleetsResult(EMR_CLUSTER_ID, awsParamsDto);
    // Verify the external calls.
    verify(awsClientFactory).getEmrClient(awsParamsDto);
    verify(emrOperations).listInstanceFleets(amazonElasticMapReduceClient, listInstanceFleetsRequest);
    verifyNoMoreInteractionsHelper();
    // Validate the results.
    assertEquals(listInstanceFleetsResult, result);
}