Examples with S3FileTransferRequestParamsDto org.finra.herd.model.dto.S3FileTransferRequestParamsDto used on opensource projects

Example 1 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class BusinessObjectDataDaoHelper method discoverStorageFiles.

/**
 * Discovers storage files in the specified S3 storage that match the S3 key prefix.
 *
 * @param storageEntity the storage entity
 * @param s3KeyPrefix the S3 key prefix
 *
 * @return the list of discovered storage files
 */
private List<StorageFile> discoverStorageFiles(StorageEntity storageEntity, String s3KeyPrefix) {
    // Only S3 storage platform is currently supported for storage file discovery.
    Assert.isTrue(storageEntity.getStoragePlatform().getName().equals(StoragePlatformEntity.S3), String.format("Cannot discover storage files at \"%s\" storage platform.", storageEntity.getStoragePlatform().getName()));
    // Get S3 bucket access parameters.
    S3FileTransferRequestParamsDto params = storageHelper.getS3BucketAccessParams(storageEntity);
    // Retrieve a list of all keys/objects from the S3 bucket matching the specified S3 key prefix.
    // Since S3 key prefix represents the directory, we add a trailing '/' character to it, unless it is already present.
    params.setS3KeyPrefix(StringUtils.appendIfMissing(s3KeyPrefix, "/"));
    // When listing S3 files, we ignore 0 byte objects that represent S3 directories.
    List<S3ObjectSummary> s3ObjectSummaries = s3Service.listDirectory(params, true);
    // Fail registration if no storage files were discovered.
    if (CollectionUtils.isEmpty(s3ObjectSummaries)) {
        throw new ObjectNotFoundException(String.format("Found no files at \"s3://%s/%s\" location.", params.getS3BucketName(), params.getS3KeyPrefix()));
    }
    return storageFileHelper.createStorageFilesFromS3ObjectSummaries(s3ObjectSummaries);
}

Example 2 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class BusinessObjectDataInitiateRestoreHelperServiceImplTest method testExecuteS3SpecificSteps.

@Test
public void testExecuteS3SpecificSteps() {
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION);
    // Create a list of storage files to be passed as an input.
    List<StorageFile> storageFiles = Arrays.asList(new StorageFile(S3_KEY, FILE_SIZE, ROW_COUNT));
    // Create a DTO for business object data restore parameters.
    BusinessObjectDataRestoreDto businessObjectDataRestoreDto = new BusinessObjectDataRestoreDto(businessObjectDataKey, STORAGE_NAME, S3_ENDPOINT, S3_BUCKET_NAME, S3_KEY_PREFIX, NO_STORAGE_UNIT_STATUS, NO_STORAGE_UNIT_STATUS, storageFiles, NO_EXCEPTION);
    // Create an S3 file transfer parameters DTO to access the S3 bucket.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    // Create an updated version of the S3 file transfer request parameters DTO.
    S3FileTransferRequestParamsDto updatedS3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    updatedS3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
    updatedS3FileTransferRequestParamsDto.setS3Endpoint(S3_ENDPOINT);
    updatedS3FileTransferRequestParamsDto.setS3KeyPrefix(S3_KEY_PREFIX + "/");
    // Create a mock S3 object summary for S3 object that does not belong to Glacier storage class.
    S3ObjectSummary glacierS3ObjectSummary = mock(S3ObjectSummary.class);
    when(glacierS3ObjectSummary.getStorageClass()).thenReturn(StorageClass.Glacier.toString());
    // Create a mock S3 object summary for S3 object that does not belong to Glacier storage class.
    S3ObjectSummary standardS3ObjectSummary = mock(S3ObjectSummary.class);
    when(standardS3ObjectSummary.getStorageClass()).thenReturn(StorageClass.Standard.toString());
    // Create a list of S3 files.
    List<S3ObjectSummary> s3Files = Arrays.asList(glacierS3ObjectSummary, standardS3ObjectSummary);
    // Create a list of S3 objects that belong to Glacier storage class.
    List<S3ObjectSummary> glacierS3Files = Arrays.asList(glacierS3ObjectSummary);
    // Create a list of storage files that represent S3 objects of Glacier storage class.
    List<StorageFile> glacierStorageFiles = Arrays.asList(new StorageFile(S3_KEY, FILE_SIZE, ROW_COUNT));
    // Create a list of storage files selected for S3 object tagging.
    List<File> filesSelectedForRestore = Arrays.asList(new File(S3_KEY));
    // Create a final version of DTO for business object data restore parameters.
    S3FileTransferRequestParamsDto finalS3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    finalS3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
    finalS3FileTransferRequestParamsDto.setS3Endpoint(S3_ENDPOINT);
    finalS3FileTransferRequestParamsDto.setS3KeyPrefix(S3_KEY_PREFIX + "/");
    finalS3FileTransferRequestParamsDto.setFiles(filesSelectedForRestore);
    // Mock the external calls.
    when(storageHelper.getS3FileTransferRequestParamsDto()).thenReturn(s3FileTransferRequestParamsDto);
    when(s3Service.listDirectory(updatedS3FileTransferRequestParamsDto, true)).thenReturn(s3Files);
    when(storageFileHelper.createStorageFilesFromS3ObjectSummaries(glacierS3Files)).thenReturn(glacierStorageFiles);
    when(storageFileHelper.getFiles(glacierStorageFiles)).thenReturn(filesSelectedForRestore);
    // Call the method under test.
    businessObjectDataInitiateRestoreHelperServiceImpl.executeS3SpecificSteps(businessObjectDataRestoreDto);
    // Verify the external calls.
    verify(storageHelper).getS3FileTransferRequestParamsDto();
    verify(s3Service).listDirectory(any(S3FileTransferRequestParamsDto.class), eq(true));
    verify(storageFileHelper).validateRegisteredS3Files(storageFiles, s3Files, STORAGE_NAME, businessObjectDataKey);
    verify(storageFileHelper).createStorageFilesFromS3ObjectSummaries(glacierS3Files);
    verify(storageFileHelper).getFiles(glacierStorageFiles);
    verify(s3Service).restoreObjects(finalS3FileTransferRequestParamsDto, 36135);
    verifyNoMoreInteractionsHelper();
}

Example 3 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class BusinessObjectDataFinalizeRestoreHelperServiceImplTest method testExecuteS3SpecificSteps.

@Test
public void testExecuteS3SpecificSteps() {
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION);
    // Create a list of storage files.
    List<StorageFile> storageFiles = Arrays.asList(new StorageFile(S3_KEY, FILE_SIZE, ROW_COUNT), new StorageFile(S3_KEY_2, FILE_SIZE_2, ROW_COUNT_2));
    // Create a DTO for business object data restore parameters.
    BusinessObjectDataRestoreDto businessObjectDataRestoreDto = new BusinessObjectDataRestoreDto(businessObjectDataKey, STORAGE_NAME, S3_ENDPOINT, S3_BUCKET_NAME, S3_KEY_PREFIX, STORAGE_UNIT_STATUS_2, STORAGE_UNIT_STATUS, storageFiles, NO_EXCEPTION);
    // Create an initial instance of S3 file transfer parameters DTO.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    // Create an updated version of S3 file transfer parameters DTO.
    S3FileTransferRequestParamsDto updatedS3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    updatedS3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
    updatedS3FileTransferRequestParamsDto.setS3Endpoint(S3_ENDPOINT);
    updatedS3FileTransferRequestParamsDto.setS3KeyPrefix(S3_KEY_PREFIX + "/");
    // Create a mock S3 object summary for S3 object that belongs to Glacier storage class.
    S3ObjectSummary glacierS3ObjectSummary = mock(S3ObjectSummary.class);
    when(glacierS3ObjectSummary.getStorageClass()).thenReturn(StorageClass.Glacier.toString());
    // Create a mock S3 object summary for S3 object that does not belong to Glacier storage class.
    S3ObjectSummary standardS3ObjectSummary = mock(S3ObjectSummary.class);
    when(standardS3ObjectSummary.getStorageClass()).thenReturn(StorageClass.Standard.toString());
    // Create a list of S3 files.
    List<S3ObjectSummary> s3Files = Arrays.asList(glacierS3ObjectSummary, standardS3ObjectSummary);
    // Create a list of S3 objects that belong to Glacier storage class.
    List<S3ObjectSummary> glacierS3Files = Arrays.asList(glacierS3ObjectSummary);
    // Create a list of storage files that represent S3 objects of Glacier storage class.
    List<StorageFile> glacierStorageFiles = Arrays.asList(new StorageFile(S3_KEY, FILE_SIZE, ROW_COUNT));
    // Create a list of files.
    List<File> files = Arrays.asList(new File(S3_KEY));
    // Create a final version of DTO for business object data restore parameters.
    S3FileTransferRequestParamsDto finalS3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    finalS3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
    finalS3FileTransferRequestParamsDto.setS3Endpoint(S3_ENDPOINT);
    finalS3FileTransferRequestParamsDto.setS3KeyPrefix(S3_KEY_PREFIX + "/");
    finalS3FileTransferRequestParamsDto.setFiles(files);
    // Mock the external calls.
    when(storageHelper.getS3FileTransferRequestParamsDto()).thenReturn(s3FileTransferRequestParamsDto);
    when(s3Service.listDirectory(updatedS3FileTransferRequestParamsDto, true)).thenReturn(s3Files);
    when(storageFileHelper.createStorageFilesFromS3ObjectSummaries(glacierS3Files)).thenReturn(glacierStorageFiles);
    when(storageFileHelper.getFiles(glacierStorageFiles)).thenReturn(files);
    // Call the method under test.
    businessObjectDataFinalizeRestoreHelperServiceImpl.executeS3SpecificSteps(businessObjectDataRestoreDto);
    // Verify the external calls.
    verify(storageHelper).getS3FileTransferRequestParamsDto();
    verify(s3Service).listDirectory(any(S3FileTransferRequestParamsDto.class), eq(true));
    verify(storageFileHelper).validateRegisteredS3Files(storageFiles, s3Files, STORAGE_NAME, businessObjectDataKey);
    verify(storageFileHelper).createStorageFilesFromS3ObjectSummaries(glacierS3Files);
    verify(storageFileHelper).getFiles(glacierStorageFiles);
    verify(s3Service).validateGlacierS3FilesRestored(finalS3FileTransferRequestParamsDto);
    verifyNoMoreInteractionsHelper();
}

Example 4 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class BusinessObjectDataInitiateDestroyHelperServiceImplTest method runExecuteS3SpecificStepsTest.

private void runExecuteS3SpecificStepsTest() {
    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, DATA_VERSION);
    // Create a storage file path.
    String storageFilePath = TEST_S3_KEY_PREFIX + "/" + LOCAL_FILE;
    // Create a business object data destroy parameters DTO.
    BusinessObjectDataDestroyDto businessObjectDataDestroyDto = new BusinessObjectDataDestroyDto(businessObjectDataKey, STORAGE_NAME, BusinessObjectDataStatusEntity.DELETED, BusinessObjectDataStatusEntity.VALID, StorageUnitStatusEntity.DISABLING, StorageUnitStatusEntity.ENABLED, S3_ENDPOINT, S3_BUCKET_NAME, TEST_S3_KEY_PREFIX, S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE, S3_OBJECT_TAGGER_ROLE_ARN, S3_OBJECT_TAGGER_ROLE_SESSION_NAME, BDATA_FINAL_DESTROY_DELAY_IN_DAYS);
    // Create an S3 file transfer parameters DTO to access the S3 bucket.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    // Create an S3 file transfer parameters DTO to be used for S3 object tagging operation.
    S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto = new S3FileTransferRequestParamsDto();
    s3ObjectTaggerParamsDto.setAwsAccessKeyId(AWS_ASSUMED_ROLE_ACCESS_KEY);
    s3ObjectTaggerParamsDto.setAwsSecretKey(AWS_ASSUMED_ROLE_SECRET_KEY);
    s3ObjectTaggerParamsDto.setSessionToken(AWS_ASSUMED_ROLE_SESSION_TOKEN);
    // Create a list of all S3 files matching the S3 key prefix form the S3 bucket.
    List<S3ObjectSummary> actualS3Files = Arrays.asList(new S3ObjectSummary());
    // Create a list of storage files selected for S3 object tagging.
    List<StorageFile> storageFilesSelectedForTagging = Arrays.asList(new StorageFile());
    // Create a list of storage files selected for S3 object tagging.
    List<File> filesSelectedForTagging = Arrays.asList(new File(storageFilePath));
    // Create an updated S3 file transfer parameters DTO to access the S3 bucket.
    S3FileTransferRequestParamsDto updatedS3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    updatedS3FileTransferRequestParamsDto.setS3Endpoint(S3_ENDPOINT);
    updatedS3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
    updatedS3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX + "/");
    updatedS3FileTransferRequestParamsDto.setFiles(filesSelectedForTagging);
    // Create an updated S3 file transfer parameters DTO to be used for S3 object tagging operation.
    S3FileTransferRequestParamsDto updatedS3ObjectTaggerParamsDto = new S3FileTransferRequestParamsDto();
    updatedS3ObjectTaggerParamsDto.setAwsAccessKeyId(AWS_ASSUMED_ROLE_ACCESS_KEY);
    updatedS3ObjectTaggerParamsDto.setAwsSecretKey(AWS_ASSUMED_ROLE_SECRET_KEY);
    updatedS3ObjectTaggerParamsDto.setSessionToken(AWS_ASSUMED_ROLE_SESSION_TOKEN);
    updatedS3ObjectTaggerParamsDto.setS3Endpoint(S3_ENDPOINT);
    // Mock the external calls.
    when(storageHelper.getS3FileTransferRequestParamsDto()).thenReturn(s3FileTransferRequestParamsDto);
    when(storageHelper.getS3FileTransferRequestParamsDtoByRole(S3_OBJECT_TAGGER_ROLE_ARN, S3_OBJECT_TAGGER_ROLE_SESSION_NAME)).thenReturn(s3ObjectTaggerParamsDto);
    when(s3Service.listDirectory(s3FileTransferRequestParamsDto, false)).thenReturn(actualS3Files);
    when(storageFileHelper.createStorageFilesFromS3ObjectSummaries(actualS3Files)).thenReturn(storageFilesSelectedForTagging);
    when(storageFileHelper.getFiles(storageFilesSelectedForTagging)).thenReturn(filesSelectedForTagging);
    // Call the method under test.
    businessObjectDataInitiateDestroyHelperServiceImpl.executeS3SpecificSteps(businessObjectDataDestroyDto);
    // Verify the external calls.
    verify(storageHelper).getS3FileTransferRequestParamsDto();
    verify(storageHelper).getS3FileTransferRequestParamsDtoByRole(S3_OBJECT_TAGGER_ROLE_ARN, S3_OBJECT_TAGGER_ROLE_SESSION_NAME);
    verify(s3Service).listDirectory(s3FileTransferRequestParamsDto, false);
    verify(storageFileHelper).createStorageFilesFromS3ObjectSummaries(actualS3Files);
    verify(storageFileHelper).getFiles(storageFilesSelectedForTagging);
    verify(s3Service).tagObjects(updatedS3FileTransferRequestParamsDto, updatedS3ObjectTaggerParamsDto, new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE));
    verifyNoMoreInteractionsHelper();
    // Validate the results.
    assertEquals(new BusinessObjectDataDestroyDto(businessObjectDataKey, STORAGE_NAME, BusinessObjectDataStatusEntity.DELETED, BusinessObjectDataStatusEntity.VALID, StorageUnitStatusEntity.DISABLING, StorageUnitStatusEntity.ENABLED, S3_ENDPOINT, S3_BUCKET_NAME, TEST_S3_KEY_PREFIX, S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE, S3_OBJECT_TAGGER_ROLE_ARN, S3_OBJECT_TAGGER_ROLE_SESSION_NAME, BDATA_FINAL_DESTROY_DELAY_IN_DAYS), businessObjectDataDestroyDto);
}

Example 5 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class DataBridgeWebClientTest method testAddStorageFilesResponse200BadContentReturnsNull.

@Test
public void testAddStorageFilesResponse200BadContentReturnsNull() throws Exception {
    dataBridgeWebClient.regServerAccessParamsDto.setRegServerHost(MockHttpClientOperationsImpl.HOSTNAME_RESPOND_WITH_STATUS_CODE_200_AND_INVALID_CONTENT);
    dataBridgeWebClient.regServerAccessParamsDto.setUseSsl(false);
    BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey();
    UploaderInputManifestDto manifest = getUploaderInputManifestDto();
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    String storageName = "testStorage";
    BusinessObjectDataStorageFilesCreateResponse businessObjectDataStorageFilesCreateResponse = dataBridgeWebClient.addStorageFiles(businessObjectDataKey, manifest, s3FileTransferRequestParamsDto, storageName);
    assertNull("businessObjectDataStorageFilesCreateResponse", businessObjectDataStorageFilesCreateResponse);
}