Examples with DeleteObjectsRequest software.amazon.awssdk.services.s3.model.DeleteObjectsRequest used on opensource projects

Example 6 with DeleteObjectsRequest

use of software.amazon.awssdk.services.s3.model.DeleteObjectsRequest in project beam by apache.

the class S3FileSystem method delete.

private void delete(String bucket, Collection<String> keys) throws IOException {
    checkArgument(keys.size() <= MAX_DELETE_OBJECTS_PER_REQUEST, "only %s keys can be deleted per request, but got %s", MAX_DELETE_OBJECTS_PER_REQUEST, keys.size());
    List<ObjectIdentifier> deleteKeyVersions = keys.stream().map((key) -> ObjectIdentifier.builder().key(key).build()).collect(Collectors.toList());
    Delete delete = Delete.builder().objects(deleteKeyVersions).quiet(true).build();
    DeleteObjectsRequest deleteObjectsRequest = DeleteObjectsRequest.builder().bucket(bucket).delete(delete).build();
    try {
        s3Client.get().deleteObjects(deleteObjectsRequest);
    } catch (SdkServiceException e) {
        throw new IOException(e);
    }
}

Example 7 with DeleteObjectsRequest

use of software.amazon.awssdk.services.s3.model.DeleteObjectsRequest in project jackrabbit by apache.

the class TestS3Ds method deleteBucket.

public void deleteBucket(String bucket) throws Exception {
    LOG.info("deleting bucket [" + bucket + "]");
    Properties props = Utils.readConfig(config);
    AmazonS3Client s3service = Utils.openService(props);
    TransferManager tmx = new TransferManager(s3service);
    if (s3service.doesBucketExist(bucket)) {
        for (int i = 0; i < 4; i++) {
            tmx.abortMultipartUploads(bucket, startTime);
            ObjectListing prevObjectListing = s3service.listObjects(bucket);
            while (prevObjectListing != null) {
                List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>();
                for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
                    deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey()));
                }
                if (deleteList.size() > 0) {
                    DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket);
                    delObjsReq.setKeys(deleteList);
                    s3service.deleteObjects(delObjsReq);
                }
                if (!prevObjectListing.isTruncated())
                    break;
                prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
            }
        }
        s3service.deleteBucket(bucket);
        LOG.info("bucket [ " + bucket + "] deleted");
    } else {
        LOG.info("bucket [" + bucket + "] doesn't exists");
    }
    tmx.shutdownNow();
    s3service.shutdown();
}

Example 8 with DeleteObjectsRequest

use of software.amazon.awssdk.services.s3.model.DeleteObjectsRequest in project jackrabbit by apache.

the class S3Backend method renameKeys.

/**
 * This method rename object keys in S3 concurrently. The number of
 * concurrent threads is defined by 'maxConnections' property in
 * aws.properties. As S3 doesn't have "move" command, this method simulate
 * move as copy object object to new key and then delete older key.
 */
private void renameKeys() throws DataStoreException {
    long startTime = System.currentTimeMillis();
    ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
    long count = 0;
    try {
        Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
        ObjectListing prevObjectListing = s3service.listObjects(bucket);
        List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>();
        int nThreads = Integer.parseInt(properties.getProperty("maxConnections"));
        ExecutorService executor = Executors.newFixedThreadPool(nThreads, new NamedThreadFactory("s3-object-rename-worker"));
        boolean taskAdded = false;
        while (true) {
            for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
                executor.execute(new KeyRenameThread(s3ObjSumm.getKey()));
                taskAdded = true;
                count++;
                // delete the object if it follows old key name format
                if (s3ObjSumm.getKey().startsWith(KEY_PREFIX)) {
                    deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey()));
                }
            }
            if (!prevObjectListing.isTruncated())
                break;
            prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
        }
        // This will make the executor accept no new threads
        // and finish all existing threads in the queue
        executor.shutdown();
        try {
            // Wait until all threads are finish
            while (taskAdded && !executor.awaitTermination(10, TimeUnit.SECONDS)) {
                LOG.info("Rename S3 keys tasks timedout. Waiting again");
            }
        } catch (InterruptedException ie) {
        }
        LOG.info("Renamed [{}] keys, time taken [{}]sec", count, ((System.currentTimeMillis() - startTime) / 1000));
        // Delete older keys.
        if (deleteList.size() > 0) {
            DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket);
            int batchSize = 500, startIndex = 0, size = deleteList.size();
            int endIndex = batchSize < size ? batchSize : size;
            while (endIndex <= size) {
                delObjsReq.setKeys(Collections.unmodifiableList(deleteList.subList(startIndex, endIndex)));
                DeleteObjectsResult dobjs = s3service.deleteObjects(delObjsReq);
                LOG.info("Records[{}] deleted in datastore from index [{}] to [{}]", new Object[] { dobjs.getDeletedObjects().size(), startIndex, (endIndex - 1) });
                if (endIndex == size) {
                    break;
                } else {
                    startIndex = endIndex;
                    endIndex = (startIndex + batchSize) < size ? (startIndex + batchSize) : size;
                }
            }
        }
    } finally {
        if (contextClassLoader != null) {
            Thread.currentThread().setContextClassLoader(contextClassLoader);
        }
    }
}

Example 9 with DeleteObjectsRequest

use of software.amazon.awssdk.services.s3.model.DeleteObjectsRequest in project druid by druid-io.

the class S3DataSegmentKillerTest method test_killAll_recoverableExceptionWhenListingObjects_deletesAllSegments.

@Test
public void test_killAll_recoverableExceptionWhenListingObjects_deletesAllSegments() throws IOException {
    S3ObjectSummary objectSummary1 = S3TestUtils.newS3ObjectSummary(TEST_BUCKET, KEY_1, TIME_0);
    S3TestUtils.expectListObjects(s3Client, PREFIX_URI, ImmutableList.of(objectSummary1));
    DeleteObjectsRequest deleteRequest1 = new DeleteObjectsRequest(TEST_BUCKET).withBucketName(TEST_BUCKET).withKeys(ImmutableList.of(new DeleteObjectsRequest.KeyVersion(KEY_1)));
    S3TestUtils.mockS3ClientDeleteObjects(s3Client, ImmutableList.of(deleteRequest1), ImmutableMap.of(deleteRequest1, RECOVERABLE_EXCEPTION));
    EasyMock.expect(segmentPusherConfig.getBucket()).andReturn(TEST_BUCKET);
    EasyMock.expectLastCall().anyTimes();
    EasyMock.expect(segmentPusherConfig.getBaseKey()).andReturn(TEST_PREFIX);
    EasyMock.expectLastCall().anyTimes();
    EasyMock.expect(inputDataConfig.getMaxListingLength()).andReturn(MAX_KEYS);
    EasyMock.expectLastCall().anyTimes();
    EasyMock.replay(s3Client, segmentPusherConfig, inputDataConfig);
    segmentKiller = new S3DataSegmentKiller(Suppliers.ofInstance(s3Client), segmentPusherConfig, inputDataConfig);
    segmentKiller.killAll();
    EasyMock.verify(s3Client, segmentPusherConfig, inputDataConfig);
}

Example 10 with DeleteObjectsRequest

use of software.amazon.awssdk.services.s3.model.DeleteObjectsRequest in project druid by druid-io.

the class S3DataSegmentKillerTest method test_killAll_nonrecoverableExceptionWhenListingObjects_deletesAllSegments.

@Test
public void test_killAll_nonrecoverableExceptionWhenListingObjects_deletesAllSegments() {
    boolean ioExceptionThrown = false;
    try {
        S3ObjectSummary objectSummary1 = S3TestUtils.newS3ObjectSummary(TEST_BUCKET, KEY_1, TIME_0);
        S3TestUtils.expectListObjects(s3Client, PREFIX_URI, ImmutableList.of(objectSummary1));
        DeleteObjectsRequest deleteRequest1 = new DeleteObjectsRequest(TEST_BUCKET).withBucketName(TEST_BUCKET).withKeys(ImmutableList.of(new DeleteObjectsRequest.KeyVersion(KEY_1)));
        S3TestUtils.mockS3ClientDeleteObjects(s3Client, ImmutableList.of(), ImmutableMap.of(deleteRequest1, NON_RECOVERABLE_EXCEPTION));
        EasyMock.expect(segmentPusherConfig.getBucket()).andReturn(TEST_BUCKET);
        EasyMock.expectLastCall().anyTimes();
        EasyMock.expect(segmentPusherConfig.getBaseKey()).andReturn(TEST_PREFIX);
        EasyMock.expectLastCall().anyTimes();
        EasyMock.expect(inputDataConfig.getMaxListingLength()).andReturn(MAX_KEYS);
        EasyMock.expectLastCall().anyTimes();
        EasyMock.replay(s3Client, segmentPusherConfig, inputDataConfig);
        segmentKiller = new S3DataSegmentKiller(Suppliers.ofInstance(s3Client), segmentPusherConfig, inputDataConfig);
        segmentKiller.killAll();
    } catch (IOException e) {
        ioExceptionThrown = true;
    }
    Assert.assertTrue(ioExceptionThrown);
    EasyMock.verify(s3Client, segmentPusherConfig, inputDataConfig);
}