Examples with DeleteObjectsRequest software.amazon.awssdk.services.s3.model.DeleteObjectsRequest used on opensource projects

Example 36 with DeleteObjectsRequest

use of software.amazon.awssdk.services.s3.model.DeleteObjectsRequest in project beam by apache.

the class S3FileSystem method delete.

private void delete(String bucket, Collection<String> keys) throws IOException {
    checkArgument(keys.size() <= MAX_DELETE_OBJECTS_PER_REQUEST, "only %s keys can be deleted per request, but got %s", MAX_DELETE_OBJECTS_PER_REQUEST, keys.size());
    List<KeyVersion> deleteKeyVersions = keys.stream().map(KeyVersion::new).collect(Collectors.toList());
    DeleteObjectsRequest request = new DeleteObjectsRequest(bucket).withKeys(deleteKeyVersions).withQuiet(true);
    try {
        amazonS3.get().deleteObjects(request);
    } catch (AmazonClientException e) {
        throw new IOException(e);
    }
}

Example 37 with DeleteObjectsRequest

use of software.amazon.awssdk.services.s3.model.DeleteObjectsRequest in project jackrabbit by apache.

the class S3Backend method deleteAllOlderThan.

@Override
public Set<DataIdentifier> deleteAllOlderThan(long min) throws DataStoreException {
    long start = System.currentTimeMillis();
    // S3 stores lastModified to lower boundary of timestamp in ms.
    // and hence min is reduced by 1000ms.
    min = min - 1000;
    Set<DataIdentifier> deleteIdSet = new HashSet<DataIdentifier>(30);
    ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
    try {
        Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
        ObjectListing prevObjectListing = s3service.listObjects(bucket);
        while (true) {
            List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>();
            for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
                DataIdentifier identifier = new DataIdentifier(getIdentifierName(s3ObjSumm.getKey()));
                long lastModified = s3ObjSumm.getLastModified().getTime();
                LOG.debug("Identifier [{}]'s lastModified = [{}]", identifier, lastModified);
                if (lastModified < min && getDataStore().confirmDelete(identifier) && // order is important here
                s3service.getObjectMetadata(bucket, s3ObjSumm.getKey()).getLastModified().getTime() < min) {
                    getDataStore().deleteFromCache(identifier);
                    LOG.debug("add id [{}] to delete lists", s3ObjSumm.getKey());
                    deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey()));
                    deleteIdSet.add(identifier);
                }
            }
            if (deleteList.size() > 0) {
                DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket);
                delObjsReq.setKeys(deleteList);
                DeleteObjectsResult dobjs = s3service.deleteObjects(delObjsReq);
                if (dobjs.getDeletedObjects().size() != deleteList.size()) {
                    throw new DataStoreException("Incomplete delete object request. only  " + dobjs.getDeletedObjects().size() + " out of " + deleteList.size() + " are deleted");
                } else {
                    LOG.debug("[{}] records deleted from datastore", deleteList);
                }
            }
            if (!prevObjectListing.isTruncated()) {
                break;
            }
            prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
        }
    } finally {
        if (contextClassLoader != null) {
            Thread.currentThread().setContextClassLoader(contextClassLoader);
        }
    }
    LOG.info("deleteAllOlderThan: min=[{}] exit. Deleted[{}] records. Number of records deleted [{}] took [{}]ms", new Object[] { min, deleteIdSet, deleteIdSet.size(), (System.currentTimeMillis() - start) });
    return deleteIdSet;
}

Example 38 with DeleteObjectsRequest

use of software.amazon.awssdk.services.s3.model.DeleteObjectsRequest in project ats-framework by Axway.

the class S3Operations method deleteObjects.

/**
 * Delete all objects matching given prefix. This method is preferred for efficient deletion of many files
 *
 * @param folderPrefix empty path is expected for objects in the "root" of the bucket
 * @param searchString what pattern to be matched. This pattern will be matched against "short file name", i.e.
 *                     the object's ID after last path separator (&quot;/&quot;).<br />
 *                     If null it means all ( string &quot;.*&quot;).
 * @param recursive if true searches recursively for matching in nested path levels (&quot;/&quot;)
 *
 * @return list of deleted objects
 * @throws S3OperationException in case of an error from server
 */
@PublicAtsApi
public void deleteObjects(String folderPrefix, String searchString, boolean recursive) {
    // Alternative but not documented in S3 API: getClient().listObjectsV2(bucket, "prefix")
    ListObjectsRequest request = new ListObjectsRequest(bucketName, folderPrefix, null, recursive ? null : "/", null);
    int totallyDeleted = 0;
    try {
        ObjectListing objectListing = s3Client.listObjects(request);
        int i = 0;
        if (searchString == null) {
            // any string
            searchString = ".*";
        }
        List<KeyVersion> keysForDelete = new ArrayList<KeyVersion>(100);
        Pattern searchStringPattern = Pattern.compile(searchString);
        while (true) {
            keysForDelete.clear();
            for (Iterator<?> iterator = objectListing.getObjectSummaries().iterator(); iterator.hasNext(); ) {
                S3ObjectSummary objectSummary = (S3ObjectSummary) iterator.next();
                if (LOG.isTraceEnabled()) {
                    LOG.trace("listObject[" + (++i) + "]: " + objectSummary.toString());
                }
                String[] fileTokens = objectSummary.getKey().split("/");
                String s3Object = fileTokens[fileTokens.length - 1];
                Matcher matcher = searchStringPattern.matcher(s3Object);
                if (matcher.find()) {
                    keysForDelete.add(new KeyVersion(objectSummary.getKey()));
                // allListElements.add(new S3ObjectInfo(objectSummary));
                }
            }
            if (keysForDelete.size() > 0) {
                // delete current set / batch size
                DeleteObjectsRequest multiObjectDeleteRequest = new DeleteObjectsRequest(bucketName).withKeys(keysForDelete).withQuiet(false);
                DeleteObjectsResult delObjRes = s3Client.deleteObjects(multiObjectDeleteRequest);
                int currentlyDeletedCount = delObjRes.getDeletedObjects().size();
                totallyDeleted = totallyDeleted + currentlyDeletedCount;
                // verify size of deleted objects
                if (keysForDelete.size() != currentlyDeletedCount) {
                    LOG.warn("The number of actually deleted objects " + currentlyDeletedCount + " does not match the expected size of " + keysForDelete.size());
                } else {
                    LOG.debug("Number of deleted S3 objects in current batch is " + currentlyDeletedCount);
                }
            }
            // more objects to retrieve (1K batch size of objects)
            if (objectListing.isTruncated()) {
                objectListing = s3Client.listNextBatchOfObjects(objectListing);
            } else {
                break;
            }
        }
        LOG.info("Successfully deleted " + totallyDeleted + " objects");
    } catch (AmazonClientException e) {
        throw new S3OperationException("Error deleting multiple objects matching pattern " + searchString + ". Number of deleted objects is " + totallyDeleted, e);
    }
}

Example 39 with DeleteObjectsRequest

use of software.amazon.awssdk.services.s3.model.DeleteObjectsRequest in project ats-framework by Axway.

the class S3Operations method deleteObjects.

/**
 * Delete multiple objects
 * @param object list of names/keys of the objects to be deleted
 */
@PublicAtsApi
public void deleteObjects(List<String> objectsList) {
    if (objectsList == null || objectsList.isEmpty()) {
        return;
    }
    List<KeyVersion> keys = new ArrayList<KeyVersion>(objectsList.size());
    for (String key : objectsList) {
        keys.add(new KeyVersion(key));
    }
    DeleteObjectsRequest request = new DeleteObjectsRequest(bucketName);
    request.withKeys(keys);
    try {
        s3Client.deleteObjects(request);
    } catch (MultiObjectDeleteException e) {
        handleMultiDeleteExceptionDetails(e);
    } catch (AmazonClientException e) {
        handleExeption(e, "Error deleting multiple objects");
    }
    LOG.info("Deleted " + objectsList.size() + " objects from bucket '" + bucketName + "'");
}

Example 40 with DeleteObjectsRequest

use of software.amazon.awssdk.services.s3.model.DeleteObjectsRequest in project tutorials by eugenp.

the class AWSS3ServiceIntegrationTest method whenVerifyingDeleteObjects_thenCorrect.

@Test
public void whenVerifyingDeleteObjects_thenCorrect() {
    DeleteObjectsRequest request = mock(DeleteObjectsRequest.class);
    DeleteObjectsResult result = mock(DeleteObjectsResult.class);
    when(s3.deleteObjects((DeleteObjectsRequest) any())).thenReturn(result);
    assertThat(service.deleteObjects(request)).isEqualTo(result);
    verify(s3).deleteObjects(request);
}