Examples with S3FileTransferRequestParamsDto org.finra.herd.model.dto.S3FileTransferRequestParamsDto used on opensource projects

Example 51 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class S3DaoTest method testGetObjectMetadataS3KeyNoExists.

/**
 * Test "key not found" scenario for the getObjectMetadata S3Dao operation.
 */
@Test
public void testGetObjectMetadataS3KeyNoExists() {
    // Try to retrieve S3 object metadata for a non-existing S3 key.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
    s3FileTransferRequestParamsDto.setS3KeyPrefix(TARGET_S3_KEY);
    assertNull(s3Dao.getObjectMetadata(s3FileTransferRequestParamsDto));
}

Example 52 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class S3DaoTest method testValidateGlacierS3FilesRestoredAmazonServiceException.

@Test
public void testValidateGlacierS3FilesRestoredAmazonServiceException() {
    // Build a mock file path that triggers an Amazon service exception when we request S3 metadata for the object.
    String testKey = String.format("%s/%s", TEST_S3_KEY_PREFIX, MockS3OperationsImpl.MOCK_S3_FILE_NAME_SERVICE_EXCEPTION);
    // Put a 1 byte Glacier storage class file in S3.
    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setHeader(Headers.STORAGE_CLASS, StorageClass.Glacier);
    metadata.setOngoingRestore(false);
    s3Operations.putObject(new PutObjectRequest(storageDaoTestHelper.getS3ManagedBucketName(), testKey, new ByteArrayInputStream(new byte[1]), metadata), null);
    // that triggers an Amazon service exception when we request S3 metadata for the object.
    try {
        S3FileTransferRequestParamsDto params = new S3FileTransferRequestParamsDto();
        params.setS3BucketName(storageDaoTestHelper.getS3ManagedBucketName());
        params.setFiles(Arrays.asList(new File(testKey)));
        s3Dao.validateGlacierS3FilesRestored(params);
        fail("Should throw an IllegalStateException when Glacier S3 object validation fails due to an Amazon service exception.");
    } catch (IllegalStateException e) {
        assertEquals(String.format("Fail to check restore status for \"%s\" key in \"%s\" bucket. " + "Reason: InternalError (Service: null; Status Code: 0; Error Code: InternalError; Request ID: null)", testKey, storageDaoTestHelper.getS3ManagedBucketName()), e.getMessage());
    }
}

Example 53 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class S3DaoTest method testAbortMultipartUploadsAssertTruncatedResult.

@Test
public void testAbortMultipartUploadsAssertTruncatedResult() {
    S3Operations originalS3Operations = (S3Operations) ReflectionTestUtils.getField(s3Dao, "s3Operations");
    S3Operations mockS3Operations = mock(S3Operations.class);
    ReflectionTestUtils.setField(s3Dao, "s3Operations", mockS3Operations);
    try {
        String s3BucketName = "s3BucketName";
        String uploadKey = "uploadKey";
        String uploadId = "uploadId";
        Date uploadInitiated = new Date(0);
        S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
        s3FileTransferRequestParamsDto.setS3BucketName(s3BucketName);
        Date thresholdDate = new Date(1);
        when(mockS3Operations.listMultipartUploads(any(), any())).then(new Answer<MultipartUploadListing>() {

            @Override
            public MultipartUploadListing answer(InvocationOnMock invocation) throws Throwable {
                ListMultipartUploadsRequest listMultipartUploadsRequest = invocation.getArgument(0);
                String keyMarker = listMultipartUploadsRequest.getKeyMarker();
                String uploadIdMarker = listMultipartUploadsRequest.getUploadIdMarker();
                MultipartUploadListing multipartUploadListing = new MultipartUploadListing();
                if (keyMarker == null || uploadIdMarker == null) {
                    multipartUploadListing.setNextKeyMarker("nextKeyMarker");
                    multipartUploadListing.setNextUploadIdMarker("nextUploadIdMarker");
                    multipartUploadListing.setTruncated(true);
                } else {
                    assertEquals("nextKeyMarker", keyMarker);
                    assertEquals("nextUploadIdMarker", uploadIdMarker);
                    MultipartUpload multipartUpload = new MultipartUpload();
                    multipartUpload.setUploadId(uploadId);
                    multipartUpload.setKey(uploadKey);
                    multipartUpload.setInitiated(uploadInitiated);
                    multipartUploadListing.getMultipartUploads().add(multipartUpload);
                }
                return multipartUploadListing;
            }
        });
        assertEquals(1, s3Dao.abortMultipartUploads(s3FileTransferRequestParamsDto, thresholdDate));
        // Assert listMultipartUploads() is called twice due to truncation
        verify(mockS3Operations, times(2)).listMultipartUploads(any(), any());
        /*
             * Assert that S3Operations.abortMultipartUpload is called exactly ONCE with arguments matching the given ArgumentMatcher
             */
        verify(mockS3Operations).abortMultipartUpload(argThat(argument -> Objects.equal(s3BucketName, argument.getBucketName()) && Objects.equal(uploadKey, argument.getKey()) && Objects.equal(uploadId, argument.getUploadId())), any());
        // Assert that no other interactions occur with the mock
        verifyNoMoreInteractions(mockS3Operations);
    } finally {
        ReflectionTestUtils.setField(s3Dao, "s3Operations", originalS3Operations);
    }
}

Example 54 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class S3DaoTest method testDownloadDirectory.

/**
 * Test that we are able to perform the uploadDirectory S3Dao operation on S3 using our DAO tier.
 */
@Test
public void testDownloadDirectory() throws IOException, InterruptedException {
    // Upload local directory to s3Dao.
    testUploadDirectory();
    // Clean up the local directory, so we can test the download.
    FileUtils.deleteDirectory(localTempPath.toFile());
    // Create local temp directory - this also validates that clean up was really executed.
    Assert.assertTrue(localTempPath.toFile().mkdir());
    // Execute download.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX + "/");
    s3FileTransferRequestParamsDto.setLocalPath(localTempPath.toString());
    s3FileTransferRequestParamsDto.setRecursive(true);
    S3FileTransferResultsDto results = s3Dao.downloadDirectory(s3FileTransferRequestParamsDto);
    // Validate results.
    Assert.assertTrue(results.getTotalFilesTransferred() == LOCAL_FILES.size());
    // Validate that we have the directory downloaded from S3.
    for (String file : LOCAL_FILES) {
        Assert.assertTrue(Paths.get(localTempPath.toString(), TEST_S3_KEY_PREFIX, file).toFile().isFile());
    }
}

Example 55 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class S3DaoTest method testValidateGlacierS3FilesRestoredGlacierObjectRestoreInProgress.

@Test
public void testValidateGlacierS3FilesRestoredGlacierObjectRestoreInProgress() {
    // Put a 1 byte Glacier storage class file in S3 that is still being restored (OngoingRestore flag is true).
    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setHeader(Headers.STORAGE_CLASS, StorageClass.Glacier);
    metadata.setOngoingRestore(true);
    s3Operations.putObject(new PutObjectRequest(storageDaoTestHelper.getS3ManagedBucketName(), TARGET_S3_KEY, new ByteArrayInputStream(new byte[1]), metadata), null);
    // Try to validate if the Glacier S3 file is already restored.
    try {
        S3FileTransferRequestParamsDto params = new S3FileTransferRequestParamsDto();
        params.setS3BucketName(storageDaoTestHelper.getS3ManagedBucketName());
        params.setFiles(Arrays.asList(new File(TARGET_S3_KEY)));
        s3Dao.validateGlacierS3FilesRestored(params);
        fail("Should throw an IllegalArgumentException when Glacier S3 file is not restored.");
    } catch (IllegalArgumentException e) {
        assertEquals(String.format("Archived Glacier S3 file \"%s\" is not restored. StorageClass {GLACIER}, OngoingRestore flag {true}, S3 bucket name {%s}", TARGET_S3_KEY, storageDaoTestHelper.getS3ManagedBucketName()), e.getMessage());
    }
}