Examples with ListObjectsV2Response software.amazon.awssdk.services.s3.model.ListObjectsV2Response used on opensource projects

Example 6 with ListObjectsV2Response

use of software.amazon.awssdk.services.s3.model.ListObjectsV2Response in project beam by apache.

the class S3FileSystemTest method matchGlobWithSlashes.

@Test
public void matchGlobWithSlashes() throws IOException {
    S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Config("mys3"));
    S3ResourceId path = S3ResourceId.fromUri("mys3://testbucket/foo/bar\\baz*");
    ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(path.getBucket()).prefix(path.getKeyNonWildcardPrefix()).continuationToken(null).build();
    // Expected to be returned; prefix and wildcard/regex match
    S3Object firstMatch = S3Object.builder().key("foo/bar\\baz0").size(100L).lastModified(Instant.ofEpochMilli(1540000000001L)).build();
    // Expected to not be returned; prefix matches, but substring after wildcard does not
    S3Object secondMatch = S3Object.builder().key("foo/bar/baz1").size(200L).lastModified(Instant.ofEpochMilli(1540000000002L)).build();
    // Expected first request returns continuation token
    ListObjectsV2Response response = ListObjectsV2Response.builder().contents(firstMatch, secondMatch).build();
    when(s3FileSystem.getS3Client().listObjectsV2(argThat(new ListObjectsV2RequestArgumentMatches(request)))).thenReturn(response);
    // Expect object metadata queries for content encoding
    HeadObjectResponse headObjectResponse = HeadObjectResponse.builder().contentEncoding("").build();
    when(s3FileSystem.getS3Client().headObject(any(HeadObjectRequest.class))).thenReturn(headObjectResponse);
    assertThat(s3FileSystem.matchGlobPaths(ImmutableList.of(path)).get(0), MatchResultMatcher.create(ImmutableList.of(MatchResult.Metadata.builder().setIsReadSeekEfficient(true).setResourceId(S3ResourceId.fromComponents("mys3", path.getBucket(), firstMatch.key())).setSizeBytes(firstMatch.size()).setLastModifiedMillis(firstMatch.lastModified().toEpochMilli()).build())));
}

Example 7 with ListObjectsV2Response

use of software.amazon.awssdk.services.s3.model.ListObjectsV2Response in project beam by apache.

the class S3FileSystemTest method matchGlob.

@Test
public void matchGlob() throws IOException {
    S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Config("mys3"));
    S3ResourceId path = S3ResourceId.fromUri("mys3://testbucket/foo/bar*baz");
    ListObjectsV2Request firstRequest = ListObjectsV2Request.builder().bucket(path.getBucket()).prefix(path.getKeyNonWildcardPrefix()).continuationToken(null).build();
    // Expected to be returned; prefix and wildcard/regex match
    S3Object firstMatch = S3Object.builder().key("foo/bar0baz").size(100L).lastModified(Instant.ofEpochMilli(1540000000001L)).build();
    // Expected to not be returned; prefix matches, but substring after wildcard does not
    S3Object secondMatch = S3Object.builder().key("foo/bar1qux").size(200L).lastModified(Instant.ofEpochMilli(1540000000002L)).build();
    // Expected first request returns continuation token
    ListObjectsV2Response firstResponse = ListObjectsV2Response.builder().nextContinuationToken("token").contents(firstMatch, secondMatch).build();
    when(s3FileSystem.getS3Client().listObjectsV2(argThat(new ListObjectsV2RequestArgumentMatches(firstRequest)))).thenReturn(firstResponse);
    // Expect second request with continuation token
    ListObjectsV2Request secondRequest = ListObjectsV2Request.builder().bucket(path.getBucket()).prefix(path.getKeyNonWildcardPrefix()).continuationToken("token").build();
    // Expected to be returned; prefix and wildcard/regex match
    S3Object thirdMatch = S3Object.builder().key("foo/bar2baz").size(300L).lastModified(Instant.ofEpochMilli(1540000000003L)).build();
    // Expected second request returns third prefix match and no continuation token
    ListObjectsV2Response secondResponse = ListObjectsV2Response.builder().nextContinuationToken(null).contents(thirdMatch).build();
    when(s3FileSystem.getS3Client().listObjectsV2(argThat(new ListObjectsV2RequestArgumentMatches(secondRequest)))).thenReturn(secondResponse);
    // Expect object metadata queries for content encoding
    HeadObjectResponse headObjectResponse = HeadObjectResponse.builder().contentEncoding("").build();
    when(s3FileSystem.getS3Client().headObject(any(HeadObjectRequest.class))).thenReturn(headObjectResponse);
    assertThat(s3FileSystem.matchGlobPaths(ImmutableList.of(path)).get(0), MatchResultMatcher.create(ImmutableList.of(MatchResult.Metadata.builder().setIsReadSeekEfficient(true).setResourceId(S3ResourceId.fromComponents("mys3", path.getBucket(), firstMatch.key())).setSizeBytes(firstMatch.size()).setLastModifiedMillis(firstMatch.lastModified().toEpochMilli()).build(), MatchResult.Metadata.builder().setIsReadSeekEfficient(true).setResourceId(S3ResourceId.fromComponents("mys3", path.getBucket(), thirdMatch.key())).setSizeBytes(thirdMatch.size()).setLastModifiedMillis(thirdMatch.lastModified().toEpochMilli()).build())));
}

Example 8 with ListObjectsV2Response

use of software.amazon.awssdk.services.s3.model.ListObjectsV2Response in project syndesis-qe by syndesisio.

the class S3Utils method deleteS3Bucket.

public void deleteS3Bucket(String bucketName) {
    ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder().bucket(bucketName).build();
    ListObjectsV2Response listObjectsV2Response;
    do {
        listObjectsV2Response = s3client.listObjectsV2(listObjectsV2Request);
        for (S3Object s3Object : listObjectsV2Response.contents()) {
            s3client.deleteObject(DeleteObjectRequest.builder().bucket(bucketName).key(s3Object.key()).build());
        }
        listObjectsV2Request = ListObjectsV2Request.builder().bucket(bucketName).continuationToken(listObjectsV2Response.nextContinuationToken()).build();
    } while (listObjectsV2Response.isTruncated());
    s3client.deleteBucket(b -> b.bucket(bucketName));
}

Example 9 with ListObjectsV2Response

use of software.amazon.awssdk.services.s3.model.ListObjectsV2Response in project flyway by flyway.

the class AwsS3Scanner method scanForResources.

/**
 * Scans S3 for the resources. In AWS SDK v2, only the region that the client is configured with can be used.
 * The format of the path is expected to be {@code s3:{bucketName}/{optional prefix}}.
 *
 * @param location The location in S3 to start searching. Subdirectories are also searched.
 * @return The resources that were found.
 */
@Override
public Collection<LoadableResource> scanForResources(final Location location) {
    String bucketName = getBucketName(location);
    String prefix = getPrefix(bucketName, location.getPath());
    S3Client s3Client = S3ClientFactory.getClient();
    try {
        ListObjectsV2Request.Builder builder = ListObjectsV2Request.builder().bucket(bucketName).prefix(prefix);
        ListObjectsV2Request request = builder.build();
        ListObjectsV2Response listObjectResult = s3Client.listObjectsV2(request);
        return getLoadableResources(bucketName, listObjectResult);
    } catch (SdkClientException e) {
        if (throwOnMissingLocations) {
            throw new FlywayException("Could not access s3 location:" + bucketName + prefix + " due to error: " + e.getMessage());
        }
        LOG.error("Skipping s3 location:" + bucketName + prefix + " due to error: " + e.getMessage());
        return Collections.emptyList();
    }
}

Example 10 with ListObjectsV2Response

use of software.amazon.awssdk.services.s3.model.ListObjectsV2Response in project flyway by flyway.

the class AwsS3Scanner method getLoadableResources.

private Collection<LoadableResource> getLoadableResources(String bucketName, final ListObjectsV2Response listObjectResult) {
    List<S3Object> objectSummaries = listObjectResult.contents();
    Set<LoadableResource> resources = new TreeSet<>();
    for (S3Object objectSummary : objectSummaries) {
        LOG.debug("Found Amazon S3 resource: " + bucketName.concat("/").concat(objectSummary.key()));
        resources.add(new AwsS3Resource(bucketName, objectSummary, encoding));
    }
    return resources;
}