Examples with MultiObjectDeleteException com.amazonaws.services.s3.model.MultiObjectDeleteException used on opensource projects

Example 1 with MultiObjectDeleteException

use of com.amazonaws.services.s3.model.MultiObjectDeleteException in project herd by FINRAOS.

the class S3DaoImpl method logMultiObjectDeleteException.

/**
 * Logs the given MultiObjectDeleteException.
 *
 * @param multiObjectDeleteException The exception to log
 */
private void logMultiObjectDeleteException(MultiObjectDeleteException multiObjectDeleteException) {
    // Create and initialize a string buffer. The initialization is required here in order to avoid an InsufficientStringBufferDeclaration PMD violation.
    StringBuilder builder = new StringBuilder(128);
    builder.append(String.format("Error deleting multiple objects. Below are the list of objects which failed to delete.%n"));
    List<DeleteError> deleteErrors = multiObjectDeleteException.getErrors();
    for (DeleteError deleteError : deleteErrors) {
        String key = deleteError.getKey();
        String versionId = deleteError.getVersionId();
        String code = deleteError.getCode();
        String message = deleteError.getMessage();
        builder.append(String.format("s3Key=\"%s\" s3VersionId=\"%s\" s3DeleteErrorCode=\"%s\" s3DeleteErrorMessage=\"%s\"%n", key, versionId, code, message));
    }
    LOGGER.error(builder.toString());
}

Example 2 with MultiObjectDeleteException

use of com.amazonaws.services.s3.model.MultiObjectDeleteException in project dataverse by IQSS.

the class S3AccessIO method deleteAllAuxObjects.

@Override
public void deleteAllAuxObjects() throws IOException {
    if (!this.canWrite()) {
        open(DataAccessOption.WRITE_ACCESS);
    }
    String prefix = getDestinationKey("");
    List<S3ObjectSummary> storedAuxFilesSummary = null;
    try {
        ListObjectsRequest req = new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix);
        ObjectListing storedAuxFilesList = s3.listObjects(req);
        storedAuxFilesSummary = storedAuxFilesList.getObjectSummaries();
        while (storedAuxFilesList.isTruncated()) {
            storedAuxFilesList = s3.listNextBatchOfObjects(storedAuxFilesList);
            storedAuxFilesSummary.addAll(storedAuxFilesList.getObjectSummaries());
        }
    } catch (AmazonClientException ase) {
        logger.warning("Caught an AmazonServiceException:    " + ase.getMessage());
        throw new IOException("S3AccessIO: Failed to get aux objects for listing to delete.");
    }
    DeleteObjectsRequest multiObjectDeleteRequest = new DeleteObjectsRequest(bucketName);
    List<KeyVersion> keys = new ArrayList<>();
    for (S3ObjectSummary item : storedAuxFilesSummary) {
        String destinationKey = item.getKey();
        keys.add(new KeyVersion(destinationKey));
    }
    // Check if the list of auxiliary files for a data file is empty
    if (keys.isEmpty()) {
        logger.fine("S3AccessIO: No auxiliary objects to delete.");
        return;
    }
    multiObjectDeleteRequest.setKeys(keys);
    logger.fine("Trying to delete auxiliary files...");
    try {
        s3.deleteObjects(multiObjectDeleteRequest);
    } catch (MultiObjectDeleteException e) {
        logger.warning("S3AccessIO: Unable to delete auxilary objects" + e.getMessage());
        throw new IOException("S3AccessIO: Failed to delete one or more auxiliary objects.");
    }
}

Example 3 with MultiObjectDeleteException

use of com.amazonaws.services.s3.model.MultiObjectDeleteException in project herd by FINRAOS.

the class S3DaoImplTest method testDeleteDirectoryMultiObjectDeleteException.

@Test
public void testDeleteDirectoryMultiObjectDeleteException() {
    // Create an S3 file transfer request parameters DTO to access S3 objects.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);
    s3FileTransferRequestParamsDto.setS3KeyPrefix(S3_KEY_PREFIX);
    // Create a retry policy.
    RetryPolicy retryPolicy = new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, INTEGER_VALUE, true);
    // Create an S3 version summary.
    S3VersionSummary s3VersionSummary = new S3VersionSummary();
    s3VersionSummary.setKey(S3_KEY);
    s3VersionSummary.setVersionId(S3_VERSION_ID);
    // Create a version listing.
    VersionListing versionListing = new VersionListing();
    versionListing.setVersionSummaries(Arrays.asList(s3VersionSummary));
    // Create a delete error.
    MultiObjectDeleteException.DeleteError deleteError = new MultiObjectDeleteException.DeleteError();
    deleteError.setKey(S3_KEY);
    deleteError.setVersionId(S3_VERSION_ID);
    deleteError.setCode(ERROR_CODE);
    deleteError.setMessage(ERROR_MESSAGE);
    // Create a multi object delete exception.
    MultiObjectDeleteException multiObjectDeleteException = new MultiObjectDeleteException(Arrays.asList(deleteError), new ArrayList<>());
    // Mock the external calls.
    when(retryPolicyFactory.getRetryPolicy()).thenReturn(retryPolicy);
    when(s3Operations.listVersions(any(ListVersionsRequest.class), any(AmazonS3Client.class))).thenReturn(versionListing);
    when(s3Operations.deleteObjects(any(DeleteObjectsRequest.class), any(AmazonS3Client.class))).thenThrow(multiObjectDeleteException);
    // Try to call the method under test.
    try {
        s3DaoImpl.deleteDirectory(s3FileTransferRequestParamsDto);
    } catch (IllegalStateException e) {
        assertEquals(String.format("Failed to delete keys/key versions with prefix \"%s\" from bucket \"%s\". " + "Reason: One or more objects could not be deleted (Service: null; Status Code: 0; Error Code: null; Request ID: null; S3 Extended Request ID: null)", S3_KEY_PREFIX, S3_BUCKET_NAME), e.getMessage());
    }
    // Verify the external calls.
    verify(retryPolicyFactory, times(2)).getRetryPolicy();
    verify(s3Operations).listVersions(any(ListVersionsRequest.class), any(AmazonS3Client.class));
    verify(s3Operations).deleteObjects(any(DeleteObjectsRequest.class), any(AmazonS3Client.class));
    verifyNoMoreInteractionsHelper();
}

Example 4 with MultiObjectDeleteException

use of com.amazonaws.services.s3.model.MultiObjectDeleteException in project herd by FINRAOS.

the class S3DaoImpl method deleteKeyVersions.

/**
 * Deletes a list of keys/key versions from the specified S3 bucket.
 *
 * @param s3Client the S3 client
 * @param s3BucketName the S3 bucket name
 * @param keyVersions the list of S3 keys/key versions
 */
private void deleteKeyVersions(AmazonS3Client s3Client, String s3BucketName, List<DeleteObjectsRequest.KeyVersion> keyVersions) {
    // Create a request to delete multiple objects in the specified bucket.
    DeleteObjectsRequest multiObjectDeleteRequest = new DeleteObjectsRequest(s3BucketName);
    // The Multi-Object Delete request can contain a list of up to 1000 keys.
    for (int i = 0; i < keyVersions.size() / MAX_KEYS_PER_DELETE_REQUEST + 1; i++) {
        List<DeleteObjectsRequest.KeyVersion> keysSubList = keyVersions.subList(i * MAX_KEYS_PER_DELETE_REQUEST, Math.min(keyVersions.size(), (i + 1) * MAX_KEYS_PER_DELETE_REQUEST));
        multiObjectDeleteRequest.setKeys(keysSubList);
        try {
            s3Operations.deleteObjects(multiObjectDeleteRequest, s3Client);
        } catch (MultiObjectDeleteException multiObjectDeleteException) {
            logMultiObjectDeleteException(multiObjectDeleteException);
            throw multiObjectDeleteException;
        }
        LOGGER.info("Successfully requested the deletion of the listed below keys/key versions from the S3 bucket. s3KeyCount={} s3BucketName=\"{}\"", keysSubList.size(), s3BucketName);
        for (DeleteObjectsRequest.KeyVersion keyVersion : keysSubList) {
            LOGGER.info("s3Key=\"{}\" s3VersionId=\"{}\"", keyVersion.getKey(), keyVersion.getVersion());
        }
    }
}

Example 5 with MultiObjectDeleteException

use of com.amazonaws.services.s3.model.MultiObjectDeleteException in project herd by FINRAOS.

the class S3DaoTest method testDeleteDirectoryAssertMultiObjectDeleteExceptionLogged.

/**
 * Asserts that when delete directory is called but S3 throws MultiObjectDeleteException, the exception is logged properly.
 */
@Test
// TODO: Log4J2 - This test works within an IDE, but not from Maven. We need to figure out why.
@Ignore
public void testDeleteDirectoryAssertMultiObjectDeleteExceptionLogged() throws Exception {
    // Inject mock
    S3Operations mockS3Operations = mock(S3Operations.class);
    S3Operations originalS3Operations = (S3Operations) ReflectionTestUtils.getField(s3Dao, "s3Operations");
    ReflectionTestUtils.setField(s3Dao, "s3Operations", mockS3Operations);
    // Override logger with my own appender to inspect the output
    String loggerName = S3DaoImpl.class.getName();
    LogLevel originalLoggerLevel = getLogLevel(loggerName);
    setLogLevel(loggerName, LogLevel.ERROR);
    String appenderName = "TestWriterAppender";
    StringWriter stringWriter = addLoggingWriterAppender(appenderName);
    try {
        // Set up mocked behavior
        // Return a list of mock version listing
        VersionListing versionListing = new VersionListing();
        S3VersionSummary s3VersionSummary = new S3VersionSummary();
        s3VersionSummary.setKey("s3VersionSummaryKey");
        s3VersionSummary.setVersionId("s3VersionSummaryVersionId");
        versionListing.setVersionSummaries(Arrays.asList(s3VersionSummary));
        when(mockS3Operations.listVersions(any(), any())).thenReturn(versionListing);
        // Have mock implementation throw exception
        List<DeleteError> errors = new ArrayList<>();
        {
            DeleteError deleteError = new DeleteError();
            deleteError.setCode("deleteError1Code");
            deleteError.setKey("deleteError1Key");
            deleteError.setMessage("deleteError1Message");
            deleteError.setVersionId("deleteError1VersionId");
            errors.add(deleteError);
        }
        {
            DeleteError deleteError = new DeleteError();
            deleteError.setCode("deleteError2Code");
            deleteError.setKey("deleteError2Key");
            deleteError.setMessage("deleteError2Message");
            deleteError.setVersionId("deleteError2VersionId");
            errors.add(deleteError);
        }
        List<DeletedObject> deletedObjects = new ArrayList<>();
        MultiObjectDeleteException multiObjectDeleteException = new MultiObjectDeleteException(errors, deletedObjects);
        when(mockS3Operations.deleteObjects(any(), any())).thenThrow(multiObjectDeleteException);
        // try the operation and catch exception
        try {
            S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
            s3FileTransferRequestParamsDto.setS3KeyPrefix("/test/prefix");
            s3Dao.deleteDirectory(s3FileTransferRequestParamsDto);
            fail();
        } catch (Exception e) {
            // Inspect and assert exception
            assertEquals(IllegalStateException.class, e.getClass());
            assertEquals(multiObjectDeleteException, e.getCause());
        }
        assertEquals(String.format("Error deleting multiple objects. Below are the list of objects which failed to delete.%n" + "s3Key=\"deleteError1Key\" s3VersionId=\"deleteError1VersionId\" " + "s3DeleteErrorCode=\"deleteError1Code\" s3DeleteErrorMessage=\"deleteError1Message\"%n" + "s3Key=\"deleteError2Key\" s3VersionId=\"deleteError2VersionId\" " + "s3DeleteErrorCode=\"deleteError2Code\" s3DeleteErrorMessage=\"deleteError2Message\"%n%n"), stringWriter.toString());
    } finally {
        // Restore original resources
        ReflectionTestUtils.setField(s3Dao, "s3Operations", originalS3Operations);
        setLogLevel(loggerName, originalLoggerLevel);
        removeLoggingAppender(appenderName);
    }
}