Examples with ListObjectsV2Response software.amazon.awssdk.services.s3.model.ListObjectsV2Response used on opensource projects

Example 1 with ListObjectsV2Response

use of software.amazon.awssdk.services.s3.model.ListObjectsV2Response in project beam by apache.

the class S3FileSystem method expandGlob.

private ExpandedGlob expandGlob(S3ResourceId glob) {
    // The S3 API can list objects, filtered by prefix, but not by wildcard.
    // Here, we find the longest prefix without wildcard "*",
    // then filter the results with a regex.
    checkArgument(glob.isWildcard(), "isWildcard");
    String keyPrefix = glob.getKeyNonWildcardPrefix();
    Pattern wildcardRegexp = Pattern.compile(wildcardToRegexp(glob.getKey()));
    LOG.debug("expanding bucket {}, prefix {}, against pattern {}", glob.getBucket(), keyPrefix, wildcardRegexp);
    ImmutableList.Builder<S3ResourceId> expandedPaths = ImmutableList.builder();
    String continuationToken = null;
    do {
        ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(glob.getBucket()).prefix(keyPrefix).continuationToken(continuationToken).build();
        ListObjectsV2Response response;
        try {
            response = s3Client.get().listObjectsV2(request);
        } catch (SdkServiceException e) {
            return ExpandedGlob.create(glob, new IOException(e));
        }
        continuationToken = response.nextContinuationToken();
        List<S3Object> contents = response.contents();
        contents.stream().filter(s3Object -> wildcardRegexp.matcher(s3Object.key()).matches()).forEach(s3Object -> {
            S3ResourceId expandedPath = S3ResourceId.fromComponents(glob.getScheme(), glob.getBucket(), s3Object.key()).withSize(s3Object.size()).withLastModified(Date.from(s3Object.lastModified()));
            LOG.debug("Expanded S3 object path {}", expandedPath);
            expandedPaths.add(expandedPath);
        });
    } while (continuationToken != null);
    return ExpandedGlob.create(glob, expandedPaths.build());
}

Example 2 with ListObjectsV2Response

use of software.amazon.awssdk.services.s3.model.ListObjectsV2Response in project aws-doc-sdk-examples by awsdocs.

the class S3BucketDeletion method listAllObjects.

// snippet-start:[s3.java2.s3_bucket_ops.delete_bucket]
public static void listAllObjects(S3Client s3, String bucket) {
    try {
        // To delete a bucket, all the objects in the bucket must be deleted first
        ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder().bucket(bucket).build();
        ListObjectsV2Response listObjectsV2Response;
        do {
            listObjectsV2Response = s3.listObjectsV2(listObjectsV2Request);
            for (S3Object s3Object : listObjectsV2Response.contents()) {
                s3.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(s3Object.key()).build());
            }
            listObjectsV2Request = ListObjectsV2Request.builder().bucket(bucket).continuationToken(listObjectsV2Response.nextContinuationToken()).build();
        } while (listObjectsV2Response.isTruncated());
        // snippet-end:[s3.java2.s3_bucket_ops.delete_bucket]
        DeleteBucketRequest deleteBucketRequest = DeleteBucketRequest.builder().bucket(bucket).build();
        s3.deleteBucket(deleteBucketRequest);
    } catch (S3Exception e) {
        System.err.println(e.awsErrorDetails().errorMessage());
        System.exit(1);
    }
}

Example 3 with ListObjectsV2Response

use of software.amazon.awssdk.services.s3.model.ListObjectsV2Response in project aws-doc-sdk-examples by awsdocs.

the class S3ObjectOperations method main.

public static void main(String[] args) throws IOException {
    final String USAGE = "\n" + "Usage:\n" + "    <bucketName> <key>\n\n" + "Where:\n" + "    bucketName - the Amazon S3 bucket to create.\n\n" + "    key - the key to use.\n\n";
    if (args.length != 2) {
        System.out.println(USAGE);
        System.exit(1);
    }
    String bucketName = args[0];
    String key = args[1];
    // snippet-start:[s3.java2.s3_object_operations.upload]
    Region region = Region.US_WEST_2;
    s3 = S3Client.builder().region(region).build();
    createBucket(s3, bucketName, region);
    PutObjectRequest objectRequest = PutObjectRequest.builder().bucket(bucketName).key(key).build();
    s3.putObject(objectRequest, RequestBody.fromByteBuffer(getRandomByteBuffer(10_000)));
    // snippet-end:[s3.java2.s3_object_operations.upload]
    // Multipart upload example
    String multipartKey = "multiPartKey";
    multipartUpload(bucketName, multipartKey);
    // snippet-start:[s3.java2.s3_object_operations.pagination]
    ListObjectsV2Request listObjectsReqManual = ListObjectsV2Request.builder().bucket(bucketName).maxKeys(1).build();
    boolean done = false;
    while (!done) {
        ListObjectsV2Response listObjResponse = s3.listObjectsV2(listObjectsReqManual);
        for (S3Object content : listObjResponse.contents()) {
            System.out.println(content.key());
        }
        if (listObjResponse.nextContinuationToken() == null) {
            done = true;
        }
        listObjectsReqManual = listObjectsReqManual.toBuilder().continuationToken(listObjResponse.nextContinuationToken()).build();
    }
    // snippet-end:[s3.java2.s3_object_operations.pagination]
    // snippet-start:[s3.java2.s3_object_operations.iterative]
    ListObjectsV2Request listReq = ListObjectsV2Request.builder().bucket(bucketName).maxKeys(1).build();
    ListObjectsV2Iterable listRes = s3.listObjectsV2Paginator(listReq);
    // Process response pages
    listRes.stream().flatMap(r -> r.contents().stream()).forEach(content -> System.out.println(" Key: " + content.key() + " size = " + content.size()));
    // snippet-end:[s3.java2.s3_object_operations.iterative]
    // snippet-start:[s3.java2.s3_object_operations.stream]
    // Helper method to work with paginated collection of items directly
    listRes.contents().stream().forEach(content -> System.out.println(" Key: " + content.key() + " size = " + content.size()));
    // snippet-start:[s3.java2.s3_object_operations.forloop]
    for (S3Object content : listRes.contents()) {
        System.out.println(" Key: " + content.key() + " size = " + content.size());
    }
    // snippet-end:[s3.java2.s3_object_operations.forloop]
    // snippet-start:[s3.java2.s3_object_operations.download]
    GetObjectRequest getObjectRequest = GetObjectRequest.builder().bucket(bucketName).key(key).build();
    s3.getObject(getObjectRequest);
    // snippet-end:[s3.java2.s3_object_operations.download]
    // snippet-start:[s3.java2.s3_object_operations.delete]
    DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder().bucket(bucketName).key(key).build();
    s3.deleteObject(deleteObjectRequest);
    // snippet-end:[s3.java2.s3_object_operations.delete]
    // Delete an object
    deleteObjectRequest = DeleteObjectRequest.builder().bucket(bucketName).key(multipartKey).build();
    s3.deleteObject(deleteObjectRequest);
    deleteBucket(s3, bucketName);
    System.out.println("Done");
}

Example 4 with ListObjectsV2Response

use of software.amazon.awssdk.services.s3.model.ListObjectsV2Response in project aws-doc-sdk-examples by awsdocs.

the class ExportEndpoints method exportEndpointsToS3.

public static List<String> exportEndpointsToS3(PinpointClient pinpoint, S3Client s3Client, String s3BucketName, String iamExportRoleArn, String applicationId) {
    SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd-HH_mm:ss.SSS_z");
    String endpointsKeyPrefix = "exports/" + applicationId + "_" + dateFormat.format(new Date());
    String s3UrlPrefix = "s3://" + s3BucketName + "/" + endpointsKeyPrefix + "/";
    List<String> objectKeys = new ArrayList<>();
    String key = "";
    try {
        // Defines the export job that Amazon Pinpoint runs
        ExportJobRequest jobRequest = ExportJobRequest.builder().roleArn(iamExportRoleArn).s3UrlPrefix(s3UrlPrefix).build();
        CreateExportJobRequest exportJobRequest = CreateExportJobRequest.builder().applicationId(applicationId).exportJobRequest(jobRequest).build();
        System.out.format("Exporting endpoints from Amazon Pinpoint application %s to Amazon S3 " + "bucket %s . . .\n", applicationId, s3BucketName);
        CreateExportJobResponse exportResult = pinpoint.createExportJob(exportJobRequest);
        String jobId = exportResult.exportJobResponse().id();
        System.out.println(jobId);
        printExportJobStatus(pinpoint, applicationId, jobId);
        ListObjectsV2Request v2Request = ListObjectsV2Request.builder().bucket(s3BucketName).prefix(endpointsKeyPrefix).build();
        // Create a list of object keys
        ListObjectsV2Response v2Response = s3Client.listObjectsV2(v2Request);
        List<S3Object> objects = v2Response.contents();
        for (S3Object object : objects) {
            key = object.key();
            objectKeys.add(key);
        }
        return objectKeys;
    } catch (PinpointException e) {
        System.err.println(e.awsErrorDetails().errorMessage());
        System.exit(1);
    }
    return null;
}

Example 5 with ListObjectsV2Response

use of software.amazon.awssdk.services.s3.model.ListObjectsV2Response in project beam by apache.

the class S3FileSystemTest method matchVariousInvokeThreadPool.

@Test
public void matchVariousInvokeThreadPool() throws IOException {
    S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Config("mys3"));
    SdkServiceException notFoundException = S3Exception.builder().message("mock exception").statusCode(404).build();
    S3ResourceId pathNotExist = S3ResourceId.fromUri("mys3://testbucket/testdirectory/nonexistentfile");
    HeadObjectRequest headObjectRequestNotExist = HeadObjectRequest.builder().bucket(pathNotExist.getBucket()).key(pathNotExist.getKey()).build();
    when(s3FileSystem.getS3Client().headObject(argThat(new GetHeadObjectRequestMatcher(headObjectRequestNotExist)))).thenThrow(notFoundException);
    SdkServiceException forbiddenException = SdkServiceException.builder().message("mock exception").statusCode(403).build();
    S3ResourceId pathForbidden = S3ResourceId.fromUri("mys3://testbucket/testdirectory/forbiddenfile");
    HeadObjectRequest headObjectRequestForbidden = HeadObjectRequest.builder().bucket(pathForbidden.getBucket()).key(pathForbidden.getKey()).build();
    when(s3FileSystem.getS3Client().headObject(argThat(new GetHeadObjectRequestMatcher(headObjectRequestForbidden)))).thenThrow(forbiddenException);
    S3ResourceId pathExist = S3ResourceId.fromUri("mys3://testbucket/testdirectory/filethatexists");
    HeadObjectRequest headObjectRequestExist = HeadObjectRequest.builder().bucket(pathExist.getBucket()).key(pathExist.getKey()).build();
    HeadObjectResponse s3ObjectMetadata = HeadObjectResponse.builder().contentLength(100L).contentEncoding("not-gzip").lastModified(Instant.ofEpochMilli(1540000000000L)).build();
    when(s3FileSystem.getS3Client().headObject(argThat(new GetHeadObjectRequestMatcher(headObjectRequestExist)))).thenReturn(s3ObjectMetadata);
    S3ResourceId pathGlob = S3ResourceId.fromUri("mys3://testbucket/path/part*");
    S3Object foundListObject = S3Object.builder().key("path/part-0").size(200L).lastModified(Instant.ofEpochMilli(1541000000000L)).build();
    ListObjectsV2Response listObjectsResponse = ListObjectsV2Response.builder().continuationToken(null).contents(foundListObject).build();
    when(s3FileSystem.getS3Client().listObjectsV2((ListObjectsV2Request) notNull())).thenReturn(listObjectsResponse);
    HeadObjectResponse headObjectResponse = HeadObjectResponse.builder().contentEncoding("").build();
    when(s3FileSystem.getS3Client().headObject(argThat(new GetHeadObjectRequestMatcher(HeadObjectRequest.builder().bucket(pathGlob.getBucket()).key("path/part-0").build())))).thenReturn(headObjectResponse);
    assertThat(s3FileSystem.match(ImmutableList.of(pathNotExist.toString(), pathForbidden.toString(), pathExist.toString(), pathGlob.toString())), contains(MatchResultMatcher.create(MatchResult.Status.NOT_FOUND, new FileNotFoundException()), MatchResultMatcher.create(MatchResult.Status.ERROR, new IOException(forbiddenException)), MatchResultMatcher.create(100, 1540000000000L, pathExist, true), MatchResultMatcher.create(200, 1541000000000L, S3ResourceId.fromComponents("mys3", pathGlob.getBucket(), foundListObject.key()), true)));
}