Examples with S3FileTransferRequestParamsDto org.finra.herd.model.dto.S3FileTransferRequestParamsDto used on opensource projects

Example 46 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class S3DaoTest method testTagObjectsTargetTagKeyAlreadyExists.

@Test
public void testTagObjectsTargetTagKeyAlreadyExists() {
    // Create two S3 object tags having the same tag key.
    List<Tag> tags = Arrays.asList(new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE), new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE_2));
    // Put a file in S3 that is already tagged with the first S3 object tag.
    PutObjectRequest putObjectRequest = new PutObjectRequest(S3_BUCKET_NAME, TARGET_S3_KEY, new ByteArrayInputStream(new byte[1]), new ObjectMetadata());
    putObjectRequest.setTagging(new ObjectTagging(Arrays.asList(tags.get(0))));
    s3Operations.putObject(putObjectRequest, null);
    // Validate that the S3 object is tagged with the first tag.
    GetObjectTaggingResult getObjectTaggingResult = s3Operations.getObjectTagging(new GetObjectTaggingRequest(S3_BUCKET_NAME, TARGET_S3_KEY), null);
    assertEquals(Arrays.asList(tags.get(0)), getObjectTaggingResult.getTagSet());
    // Tag the S3 file with the second S3 object tag.
    S3FileTransferRequestParamsDto params = new S3FileTransferRequestParamsDto();
    params.setS3BucketName(S3_BUCKET_NAME);
    params.setFiles(Arrays.asList(new File(TARGET_S3_KEY)));
    s3Dao.tagObjects(params, new S3FileTransferRequestParamsDto(), tags.get(1));
    // Validate that the S3 object is tagged with the second tag now.
    getObjectTaggingResult = s3Operations.getObjectTagging(new GetObjectTaggingRequest(S3_BUCKET_NAME, TARGET_S3_KEY), null);
    assertEquals(Arrays.asList(tags.get(1)), getObjectTaggingResult.getTagSet());
}

Example 47 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class S3DaoTest method testAbortMultipartUploadsAssertHandleGenericException.

/**
 * When any exception is thrown during the flow, the S3 client should be shut down gracefully. The exception is thrown as-is, without any wrapping or
 * special handling.
 */
@Test
public void testAbortMultipartUploadsAssertHandleGenericException() {
    S3Operations originalS3Operations = (S3Operations) ReflectionTestUtils.getField(s3Dao, "s3Operations");
    S3Operations mockS3Operations = mock(S3Operations.class);
    ReflectionTestUtils.setField(s3Dao, "s3Operations", mockS3Operations);
    try {
        String s3BucketName = "s3BucketName";
        S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
        s3FileTransferRequestParamsDto.setS3BucketName(s3BucketName);
        Date thresholdDate = new Date(1);
        when(mockS3Operations.listMultipartUploads(any(), any())).thenThrow(new AmazonServiceException("message"));
        try {
            s3Dao.abortMultipartUploads(s3FileTransferRequestParamsDto, thresholdDate);
            fail();
        } catch (Exception e) {
            assertEquals(AmazonServiceException.class, e.getClass());
            assertEquals("message (Service: null; Status Code: 0; Error Code: null; Request ID: null)", e.getMessage());
        }
    } finally {
        ReflectionTestUtils.setField(s3Dao, "s3Operations", originalS3Operations);
    }
}

Example 48 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class S3DaoTest method testUploadDirectory.

/**
 * Test that we are able to perform the uploadDirectory S3Dao operation on S3 using our DAO tier.
 */
@Test
public void testUploadDirectory() throws IOException, InterruptedException {
    // Create local test files.
    for (String file : LOCAL_FILES) {
        createLocalFile(localTempPath.toString(), file, FILE_SIZE_1_KB);
    }
    // Upload test file to s3Dao.
    // Since the S3 key prefix represents a directory, we add a trailing '/' character to it.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX + "/");
    s3FileTransferRequestParamsDto.setLocalPath(localTempPath.toString());
    s3FileTransferRequestParamsDto.setRecursive(true);
    S3FileTransferResultsDto results = s3Dao.uploadDirectory(s3FileTransferRequestParamsDto);
    // Validate results.
    Assert.assertTrue(results.getTotalFilesTransferred() == LOCAL_FILES.size());
    // Build a list of expected S3 key values.
    List<String> expectedKeys = new ArrayList<>();
    for (String file : LOCAL_FILES) {
        expectedKeys.add(TEST_S3_KEY_PREFIX + "/" + file.replaceAll("\\\\", "/"));
    }
    // Validate the file upload.
    s3DaoTestHelper.validateS3FileUpload(s3FileTransferRequestParamsDto, expectedKeys);
}

Example 49 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class S3DaoTest method testUploadFileZeroBytes.

/**
 * Test that we are able to upload a zero byte file to S3 using our DAO tier.
 */
@Test
public void testUploadFileZeroBytes() throws IOException, InterruptedException {
    // Create an empty local file.
    File targetFile = createLocalFile(localTempPath.toString(), LOCAL_FILE, FILE_SIZE_0_BYTE);
    Assert.assertTrue(targetFile.isFile());
    Assert.assertTrue(targetFile.length() == FILE_SIZE_0_BYTE);
    // Upload test file to s3Dao.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setS3KeyPrefix(TARGET_S3_KEY);
    s3FileTransferRequestParamsDto.setLocalPath(targetFile.getPath());
    S3FileTransferResultsDto results = s3Dao.uploadFile(s3FileTransferRequestParamsDto);
    // Validate results.
    Assert.assertTrue(results.getTotalFilesTransferred() == 1L);
    // Validate the file upload.
    s3DaoTestHelper.validateS3FileUpload(s3FileTransferRequestParamsDto, Arrays.asList(TARGET_S3_KEY));
}

Example 50 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class S3DaoTest method testRestoreObjectsAmazonServiceException.

@Test
public void testRestoreObjectsAmazonServiceException() {
    // Build a mock file path that triggers an Amazon service exception when we request to restore an object.
    String testKey = String.format("%s/%s", TEST_S3_KEY_PREFIX, MockS3OperationsImpl.MOCK_S3_FILE_NAME_SERVICE_EXCEPTION);
    // Put a 1 byte Glacier storage class file in S3.
    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setHeader(Headers.STORAGE_CLASS, StorageClass.Glacier);
    metadata.setOngoingRestore(false);
    s3Operations.putObject(new PutObjectRequest(storageDaoTestHelper.getS3ManagedBucketName(), testKey, new ByteArrayInputStream(new byte[1]), metadata), null);
    // Try to initiate a restore request for a mocked S3 file that would trigger an Amazon service exception when we request to restore an object.
    try {
        S3FileTransferRequestParamsDto params = new S3FileTransferRequestParamsDto();
        params.setS3BucketName(storageDaoTestHelper.getS3ManagedBucketName());
        params.setFiles(Arrays.asList(new File(testKey)));
        s3Dao.restoreObjects(params, S3_RESTORE_OBJECT_EXPIRATION_IN_DAYS);
        fail("Should throw an IllegalStateException when an S3 restore object operation fails.");
    } catch (IllegalStateException e) {
        assertEquals(String.format("Failed to initiate a restore request for \"%s\" key in \"%s\" bucket. " + "Reason: InternalError (Service: null; Status Code: 0; Error Code: InternalError; Request ID: null)", testKey, storageDaoTestHelper.getS3ManagedBucketName()), e.getMessage());
    }
}