Search in sources :

Example 1 with Delete

use of software.amazon.awssdk.services.s3.model.Delete in project beam by apache.

the class S3FileSystem method delete.

private void delete(String bucket, Collection<String> keys) throws IOException {
    checkArgument(keys.size() <= MAX_DELETE_OBJECTS_PER_REQUEST, "only %s keys can be deleted per request, but got %s", MAX_DELETE_OBJECTS_PER_REQUEST, keys.size());
    List<ObjectIdentifier> deleteKeyVersions = keys.stream().map((key) -> ObjectIdentifier.builder().key(key).build()).collect(Collectors.toList());
    Delete delete = Delete.builder().objects(deleteKeyVersions).quiet(true).build();
    DeleteObjectsRequest deleteObjectsRequest = DeleteObjectsRequest.builder().bucket(bucket).delete(delete).build();
    try {
        s3Client.get().deleteObjects(deleteObjectsRequest);
    } catch (SdkServiceException e) {
        throw new IOException(e);
    }
}
Also used : CompletedMultipartUpload(software.amazon.awssdk.services.s3.model.CompletedMultipartUpload) Date(java.util.Date) LoggerFactory(org.slf4j.LoggerFactory) CreateMultipartUploadResponse(software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse) S3Exception(software.amazon.awssdk.services.s3.model.S3Exception) ImmutableSet(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet) Future(java.util.concurrent.Future) Supplier(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Supplier) Strings(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Strings) Map(java.util.Map) Iterables(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables) ArrayListMultimap(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ArrayListMultimap) ObjectIdentifier(software.amazon.awssdk.services.s3.model.ObjectIdentifier) CompleteMultipartUploadResponse(software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse) CopyObjectResponse(software.amazon.awssdk.services.s3.model.CopyObjectResponse) Suppliers(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Suppliers) MoreExecutors(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors) Collection(java.util.Collection) CompleteMultipartUploadRequest(software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest) HeadObjectRequest(software.amazon.awssdk.services.s3.model.HeadObjectRequest) ListObjectsV2Request(software.amazon.awssdk.services.s3.model.ListObjectsV2Request) Collectors(java.util.stream.Collectors) S3Options(org.apache.beam.sdk.io.aws2.options.S3Options) FileNotFoundException(java.io.FileNotFoundException) Executors(java.util.concurrent.Executors) ThreadFactoryBuilder(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ThreadFactoryBuilder) Objects(java.util.Objects) List(java.util.List) DeleteObjectsRequest(software.amazon.awssdk.services.s3.model.DeleteObjectsRequest) Stream(java.util.stream.Stream) MoveOptions(org.apache.beam.sdk.io.fs.MoveOptions) VisibleForTesting(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting) HeadObjectResponse(software.amazon.awssdk.services.s3.model.HeadObjectResponse) AutoValue(com.google.auto.value.AutoValue) ListeningExecutorService(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ListeningExecutorService) ImmutableList(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList) Pattern(java.util.regex.Pattern) SdkServiceException(software.amazon.awssdk.core.exception.SdkServiceException) Delete(software.amazon.awssdk.services.s3.model.Delete) FileSystem(org.apache.beam.sdk.io.FileSystem) Preconditions.checkNotNull(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull) MatchResult(org.apache.beam.sdk.io.fs.MatchResult) S3Object(software.amazon.awssdk.services.s3.model.S3Object) CopyObjectRequest(software.amazon.awssdk.services.s3.model.CopyObjectRequest) FileSystemUtils.wildcardToRegexp(org.apache.beam.sdk.io.FileSystemUtils.wildcardToRegexp) Callable(java.util.concurrent.Callable) UploadPartCopyRequest(software.amazon.awssdk.services.s3.model.UploadPartCopyRequest) ListObjectsV2Response(software.amazon.awssdk.services.s3.model.ListObjectsV2Response) ArrayList(java.util.ArrayList) Multimap(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Multimap) Preconditions.checkArgument(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument) Nullable(org.checkerframework.checker.nullness.qual.Nullable) ReadableByteChannel(java.nio.channels.ReadableByteChannel) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) CreateOptions(org.apache.beam.sdk.io.fs.CreateOptions) S3Client(software.amazon.awssdk.services.s3.S3Client) IOException(java.io.IOException) CopyPartResult(software.amazon.awssdk.services.s3.model.CopyPartResult) CompletedPart(software.amazon.awssdk.services.s3.model.CompletedPart) ExecutionException(java.util.concurrent.ExecutionException) Preconditions.checkState(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState) MoreFutures(org.apache.beam.sdk.util.MoreFutures) WritableByteChannel(java.nio.channels.WritableByteChannel) CreateMultipartUploadRequest(software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest) Delete(software.amazon.awssdk.services.s3.model.Delete) SdkServiceException(software.amazon.awssdk.core.exception.SdkServiceException) IOException(java.io.IOException) ObjectIdentifier(software.amazon.awssdk.services.s3.model.ObjectIdentifier) DeleteObjectsRequest(software.amazon.awssdk.services.s3.model.DeleteObjectsRequest)

Example 2 with Delete

use of software.amazon.awssdk.services.s3.model.Delete in project beam by apache.

the class S3FileSystemTest method deleteThousandsOfObjectsInMultipleBucketsWithS3Options.

@Test
public void deleteThousandsOfObjectsInMultipleBucketsWithS3Options() throws IOException {
    S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options());
    List<String> buckets = ImmutableList.of("bucket1", "bucket2");
    List<String> keys = new ArrayList<>();
    for (int i = 0; i < 2500; i++) {
        keys.add(String.format("key-%d", i));
    }
    List<S3ResourceId> paths = new ArrayList<>();
    for (String bucket : buckets) {
        for (String key : keys) {
            paths.add(S3ResourceId.fromComponents("s3", bucket, key));
        }
    }
    s3FileSystem.delete(paths);
    // Should require 6 calls to delete 2500 objects in each of 2 buckets.
    verify(s3FileSystem.getS3Client(), times(6)).deleteObjects(any(DeleteObjectsRequest.class));
}
Also used : ArrayList(java.util.ArrayList) S3TestUtils.buildMockedS3FileSystem(org.apache.beam.sdk.io.aws2.s3.S3TestUtils.buildMockedS3FileSystem) DeleteObjectsRequest(software.amazon.awssdk.services.s3.model.DeleteObjectsRequest) Test(org.junit.Test)

Example 3 with Delete

use of software.amazon.awssdk.services.s3.model.Delete in project aws-doc-sdk-examples by awsdocs.

the class S3BucketDeletion method main.

public static void main(String[] args) throws Exception {
    final String USAGE = "\n" + "Usage:\n" + "    <bucket>\n\n" + "Where:\n" + "    bucket - the bucket to delete (for example, bucket1). \n\n";
    if (args.length != 1) {
        System.out.println(USAGE);
        System.exit(1);
    }
    String bucket = args[0];
    Region region = Region.US_WEST_2;
    S3Client s3 = S3Client.builder().region(region).build();
    listAllObjects(s3, bucket);
    s3.close();
}
Also used : Region(software.amazon.awssdk.regions.Region) S3Client(software.amazon.awssdk.services.s3.S3Client)

Example 4 with Delete

use of software.amazon.awssdk.services.s3.model.Delete in project aws-doc-sdk-examples by awsdocs.

the class S3BucketDeletion method listAllObjects.

// snippet-start:[s3.java2.s3_bucket_ops.delete_bucket]
public static void listAllObjects(S3Client s3, String bucket) {
    try {
        // To delete a bucket, all the objects in the bucket must be deleted first
        ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder().bucket(bucket).build();
        ListObjectsV2Response listObjectsV2Response;
        do {
            listObjectsV2Response = s3.listObjectsV2(listObjectsV2Request);
            for (S3Object s3Object : listObjectsV2Response.contents()) {
                s3.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(s3Object.key()).build());
            }
            listObjectsV2Request = ListObjectsV2Request.builder().bucket(bucket).continuationToken(listObjectsV2Response.nextContinuationToken()).build();
        } while (listObjectsV2Response.isTruncated());
        // snippet-end:[s3.java2.s3_bucket_ops.delete_bucket]
        DeleteBucketRequest deleteBucketRequest = DeleteBucketRequest.builder().bucket(bucket).build();
        s3.deleteBucket(deleteBucketRequest);
    } catch (S3Exception e) {
        System.err.println(e.awsErrorDetails().errorMessage());
        System.exit(1);
    }
}
Also used : ListObjectsV2Request(software.amazon.awssdk.services.s3.model.ListObjectsV2Request) DeleteBucketRequest(software.amazon.awssdk.services.s3.model.DeleteBucketRequest) S3Exception(software.amazon.awssdk.services.s3.model.S3Exception) S3Object(software.amazon.awssdk.services.s3.model.S3Object) ListObjectsV2Response(software.amazon.awssdk.services.s3.model.ListObjectsV2Response)

Example 5 with Delete

use of software.amazon.awssdk.services.s3.model.Delete in project aws-doc-sdk-examples by awsdocs.

the class S3Cors method setCorsInformation.

public static void setCorsInformation(S3Client s3, String bucketName, String accountId) {
    List<String> allowMethods = new ArrayList();
    allowMethods.add("PUT");
    allowMethods.add("POST");
    allowMethods.add("DELETE");
    List<String> allowOrigins = new ArrayList();
    allowOrigins.add("http://example.com");
    try {
        // Define CORS rules.
        CORSRule corsRule = CORSRule.builder().allowedMethods(allowMethods).allowedOrigins(allowOrigins).build();
        List<CORSRule> corsRules = new ArrayList<>();
        corsRules.add(corsRule);
        CORSConfiguration configuration = CORSConfiguration.builder().corsRules(corsRules).build();
        PutBucketCorsRequest putBucketCorsRequest = PutBucketCorsRequest.builder().bucket(bucketName).corsConfiguration(configuration).expectedBucketOwner(accountId).build();
        s3.putBucketCors(putBucketCorsRequest);
    } catch (S3Exception e) {
        System.err.println(e.awsErrorDetails().errorMessage());
        System.exit(1);
    }
}
Also used : CORSConfiguration(software.amazon.awssdk.services.s3.model.CORSConfiguration) PutBucketCorsRequest(software.amazon.awssdk.services.s3.model.PutBucketCorsRequest) S3Exception(software.amazon.awssdk.services.s3.model.S3Exception) ArrayList(java.util.ArrayList) CORSRule(software.amazon.awssdk.services.s3.model.CORSRule)

Aggregations

S3Exception (software.amazon.awssdk.services.s3.model.S3Exception)10 S3Client (software.amazon.awssdk.services.s3.S3Client)9 ArrayList (java.util.ArrayList)8 Region (software.amazon.awssdk.regions.Region)7 DeleteObjectsRequest (software.amazon.awssdk.services.s3.model.DeleteObjectsRequest)6 ObjectIdentifier (software.amazon.awssdk.services.s3.model.ObjectIdentifier)5 ListObjectsV2Request (software.amazon.awssdk.services.s3.model.ListObjectsV2Request)4 ListObjectsV2Response (software.amazon.awssdk.services.s3.model.ListObjectsV2Response)4 S3Object (software.amazon.awssdk.services.s3.model.S3Object)4 IOException (java.io.IOException)2 List (java.util.List)2 Collectors (java.util.stream.Collectors)2 Test (org.junit.Test)2 CompleteMultipartUploadRequest (software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest)2 CompletedMultipartUpload (software.amazon.awssdk.services.s3.model.CompletedMultipartUpload)2 CompletedPart (software.amazon.awssdk.services.s3.model.CompletedPart)2 CreateMultipartUploadRequest (software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest)2 CreateMultipartUploadResponse (software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse)2 Delete (software.amazon.awssdk.services.s3.model.Delete)2 DeleteBucketRequest (software.amazon.awssdk.services.s3.model.DeleteBucketRequest)2