Examples with S3FileTransferRequestParamsDto org.finra.herd.model.dto.S3FileTransferRequestParamsDto used on opensource projects

Example 96 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class S3DaoTest method testRestoreObjectsGlacierObjectAlreadyBeingRestored.

@Test
public void testRestoreObjectsGlacierObjectAlreadyBeingRestored() {
    // Put a 1 byte Glacier storage class file in S3 flagged as already being restored.
    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setHeader(Headers.STORAGE_CLASS, StorageClass.Glacier);
    metadata.setOngoingRestore(true);
    s3Operations.putObject(new PutObjectRequest(storageDaoTestHelper.getS3ManagedBucketName(), TARGET_S3_KEY, new ByteArrayInputStream(new byte[1]), metadata), null);
    // Initiate a restore request for the test S3 file.
    S3FileTransferRequestParamsDto params = new S3FileTransferRequestParamsDto();
    params.setS3BucketName(storageDaoTestHelper.getS3ManagedBucketName());
    params.setFiles(Arrays.asList(new File(TARGET_S3_KEY)));
    s3Dao.restoreObjects(params, S3_RESTORE_OBJECT_EXPIRATION_IN_DAYS);
    // Validate that there is still an ongoing restore request for this object.
    ObjectMetadata objectMetadata = s3Operations.getObjectMetadata(storageDaoTestHelper.getS3ManagedBucketName(), TARGET_S3_KEY, null);
    assertTrue(objectMetadata.getOngoingRestore());
}

Example 97 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class S3DaoTest method testDeleteDirectoryBucketVersioningEnabled.

/**
 * Test that we are able to perform the deleteDirectory S3Dao operation on S3 with enabled versioning using our DAO tier.
 */
@Test
public void testDeleteDirectoryBucketVersioningEnabled() throws IOException, InterruptedException {
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = null;
    try {
        // Create local test files.
        for (String file : LOCAL_FILES) {
            createLocalFile(localTempPath.toString(), file, FILE_SIZE_1_KB);
        }
        // Upload twice the same set of test files to an S3 bucket with enabled versioning.
        // Since the S3 key prefix represents a directory, we add a trailing '/' character to it.
        s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto();
        s3FileTransferRequestParamsDto.setS3BucketName(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED);
        s3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX + "/");
        s3FileTransferRequestParamsDto.setLocalPath(localTempPath.toString());
        s3FileTransferRequestParamsDto.setRecursive(true);
        for (int i = 0; i < 2; i++) {
            S3FileTransferResultsDto results = s3Dao.uploadDirectory(s3FileTransferRequestParamsDto);
            Assert.assertEquals(Long.valueOf(LOCAL_FILES.size()), results.getTotalFilesTransferred());
        }
        // Validate the existence of keys and key versions for the uploaded test files in S3.
        Assert.assertEquals(LOCAL_FILES.size(), s3Dao.listDirectory(s3FileTransferRequestParamsDto).size());
        Assert.assertEquals(LOCAL_FILES.size() * 2, s3Dao.listVersions(s3FileTransferRequestParamsDto).size());
        // Delete directory from S3 using s3Dao.
        s3Dao.deleteDirectory(s3FileTransferRequestParamsDto);
        // Validate that S3 directory got deleted.
        Assert.assertEquals(0, s3Dao.listDirectory(s3FileTransferRequestParamsDto).size());
        Assert.assertEquals(0, s3Dao.listVersions(s3FileTransferRequestParamsDto).size());
    } catch (Exception e) {
        // Clean up the S3 on failure.
        if (s3FileTransferRequestParamsDto != null) {
            s3Dao.deleteDirectory(s3FileTransferRequestParamsDto);
        }
    }
}

Example 98 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class S3DaoTest method testUploadDirectoryZeroBytes.

/**
 * Test that we are able to perform the uploadDirectory S3Dao operation on a folder with 0 bytes of data using our DAO tier.
 */
@Test
public void testUploadDirectoryZeroBytes() throws IOException, InterruptedException {
    // Create a zero size local file.
    File targetFile = createLocalFile(localTempPath.toString(), LOCAL_FILE, FILE_SIZE_0_BYTE);
    Assert.assertTrue(targetFile.isFile());
    // Upload empty folder to s3Dao.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX);
    s3FileTransferRequestParamsDto.setLocalPath(localTempPath.toString());
    s3FileTransferRequestParamsDto.setRecursive(true);
    S3FileTransferResultsDto results = s3Dao.uploadDirectory(s3FileTransferRequestParamsDto);
    // Validate results.
    Assert.assertTrue(results.getTotalFilesTransferred() == 1L);
    // Validate the zero bytes upload.
    Assert.assertEquals(1, s3Dao.listDirectory(s3FileTransferRequestParamsDto).size());
}

Example 99 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class S3DaoTest method testValidateGlacierS3FilesRestoredEmptyList.

@Test
public void testValidateGlacierS3FilesRestoredEmptyList() {
    // Make a call to validate Glacier S3 files being restored for an empty list of S3 files.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setFiles(new ArrayList<>());
    s3Dao.validateGlacierS3FilesRestored(s3FileTransferRequestParamsDto);
}

Example 100 with S3FileTransferRequestParamsDto

use of org.finra.herd.model.dto.S3FileTransferRequestParamsDto in project herd by FINRAOS.

the class S3DaoTest method testDeleteDirectoryAssertMultiObjectDeleteExceptionLogged.

/**
 * Asserts that when delete directory is called but S3 throws MultiObjectDeleteException, the exception is logged properly.
 */
@Test
// TODO: Log4J2 - This test works within an IDE, but not from Maven. We need to figure out why.
@Ignore
public void testDeleteDirectoryAssertMultiObjectDeleteExceptionLogged() throws Exception {
    // Inject mock
    S3Operations mockS3Operations = mock(S3Operations.class);
    S3Operations originalS3Operations = (S3Operations) ReflectionTestUtils.getField(s3Dao, "s3Operations");
    ReflectionTestUtils.setField(s3Dao, "s3Operations", mockS3Operations);
    // Override logger with my own appender to inspect the output
    String loggerName = S3DaoImpl.class.getName();
    LogLevel originalLoggerLevel = getLogLevel(loggerName);
    setLogLevel(loggerName, LogLevel.ERROR);
    String appenderName = "TestWriterAppender";
    StringWriter stringWriter = addLoggingWriterAppender(appenderName);
    try {
        // Set up mocked behavior
        // Return a list of mock version listing
        VersionListing versionListing = new VersionListing();
        S3VersionSummary s3VersionSummary = new S3VersionSummary();
        s3VersionSummary.setKey("s3VersionSummaryKey");
        s3VersionSummary.setVersionId("s3VersionSummaryVersionId");
        versionListing.setVersionSummaries(Arrays.asList(s3VersionSummary));
        when(mockS3Operations.listVersions(any(), any())).thenReturn(versionListing);
        // Have mock implementation throw exception
        List<DeleteError> errors = new ArrayList<>();
        {
            DeleteError deleteError = new DeleteError();
            deleteError.setCode("deleteError1Code");
            deleteError.setKey("deleteError1Key");
            deleteError.setMessage("deleteError1Message");
            deleteError.setVersionId("deleteError1VersionId");
            errors.add(deleteError);
        }
        {
            DeleteError deleteError = new DeleteError();
            deleteError.setCode("deleteError2Code");
            deleteError.setKey("deleteError2Key");
            deleteError.setMessage("deleteError2Message");
            deleteError.setVersionId("deleteError2VersionId");
            errors.add(deleteError);
        }
        List<DeletedObject> deletedObjects = new ArrayList<>();
        MultiObjectDeleteException multiObjectDeleteException = new MultiObjectDeleteException(errors, deletedObjects);
        when(mockS3Operations.deleteObjects(any(), any())).thenThrow(multiObjectDeleteException);
        // try the operation and catch exception
        try {
            S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
            s3FileTransferRequestParamsDto.setS3KeyPrefix("/test/prefix");
            s3Dao.deleteDirectory(s3FileTransferRequestParamsDto);
            fail();
        } catch (Exception e) {
            // Inspect and assert exception
            assertEquals(IllegalStateException.class, e.getClass());
            assertEquals(multiObjectDeleteException, e.getCause());
        }
        assertEquals(String.format("Error deleting multiple objects. Below are the list of objects which failed to delete.%n" + "s3Key=\"deleteError1Key\" s3VersionId=\"deleteError1VersionId\" " + "s3DeleteErrorCode=\"deleteError1Code\" s3DeleteErrorMessage=\"deleteError1Message\"%n" + "s3Key=\"deleteError2Key\" s3VersionId=\"deleteError2VersionId\" " + "s3DeleteErrorCode=\"deleteError2Code\" s3DeleteErrorMessage=\"deleteError2Message\"%n%n"), stringWriter.toString());
    } finally {
        // Restore original resources
        ReflectionTestUtils.setField(s3Dao, "s3Operations", originalS3Operations);
        setLogLevel(loggerName, originalLoggerLevel);
        removeLoggingAppender(appenderName);
    }
}