Examples with S3Object software.amazon.awssdk.services.s3.model.S3Object used on opensource projects

Example 96 with S3Object

use of software.amazon.awssdk.services.s3.model.S3Object in project beam by apache.

the class S3FileSystemTest method matchVariousInvokeThreadPool.

@Test
public void matchVariousInvokeThreadPool() throws IOException {
    S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Config("mys3"));
    SdkServiceException notFoundException = S3Exception.builder().message("mock exception").statusCode(404).build();
    S3ResourceId pathNotExist = S3ResourceId.fromUri("mys3://testbucket/testdirectory/nonexistentfile");
    HeadObjectRequest headObjectRequestNotExist = HeadObjectRequest.builder().bucket(pathNotExist.getBucket()).key(pathNotExist.getKey()).build();
    when(s3FileSystem.getS3Client().headObject(argThat(new GetHeadObjectRequestMatcher(headObjectRequestNotExist)))).thenThrow(notFoundException);
    SdkServiceException forbiddenException = SdkServiceException.builder().message("mock exception").statusCode(403).build();
    S3ResourceId pathForbidden = S3ResourceId.fromUri("mys3://testbucket/testdirectory/forbiddenfile");
    HeadObjectRequest headObjectRequestForbidden = HeadObjectRequest.builder().bucket(pathForbidden.getBucket()).key(pathForbidden.getKey()).build();
    when(s3FileSystem.getS3Client().headObject(argThat(new GetHeadObjectRequestMatcher(headObjectRequestForbidden)))).thenThrow(forbiddenException);
    S3ResourceId pathExist = S3ResourceId.fromUri("mys3://testbucket/testdirectory/filethatexists");
    HeadObjectRequest headObjectRequestExist = HeadObjectRequest.builder().bucket(pathExist.getBucket()).key(pathExist.getKey()).build();
    HeadObjectResponse s3ObjectMetadata = HeadObjectResponse.builder().contentLength(100L).contentEncoding("not-gzip").lastModified(Instant.ofEpochMilli(1540000000000L)).build();
    when(s3FileSystem.getS3Client().headObject(argThat(new GetHeadObjectRequestMatcher(headObjectRequestExist)))).thenReturn(s3ObjectMetadata);
    S3ResourceId pathGlob = S3ResourceId.fromUri("mys3://testbucket/path/part*");
    S3Object foundListObject = S3Object.builder().key("path/part-0").size(200L).lastModified(Instant.ofEpochMilli(1541000000000L)).build();
    ListObjectsV2Response listObjectsResponse = ListObjectsV2Response.builder().continuationToken(null).contents(foundListObject).build();
    when(s3FileSystem.getS3Client().listObjectsV2((ListObjectsV2Request) notNull())).thenReturn(listObjectsResponse);
    HeadObjectResponse headObjectResponse = HeadObjectResponse.builder().contentEncoding("").build();
    when(s3FileSystem.getS3Client().headObject(argThat(new GetHeadObjectRequestMatcher(HeadObjectRequest.builder().bucket(pathGlob.getBucket()).key("path/part-0").build())))).thenReturn(headObjectResponse);
    assertThat(s3FileSystem.match(ImmutableList.of(pathNotExist.toString(), pathForbidden.toString(), pathExist.toString(), pathGlob.toString())), contains(MatchResultMatcher.create(MatchResult.Status.NOT_FOUND, new FileNotFoundException()), MatchResultMatcher.create(MatchResult.Status.ERROR, new IOException(forbiddenException)), MatchResultMatcher.create(100, 1540000000000L, pathExist, true), MatchResultMatcher.create(200, 1541000000000L, S3ResourceId.fromComponents("mys3", pathGlob.getBucket(), foundListObject.key()), true)));
}

Example 97 with S3Object

use of software.amazon.awssdk.services.s3.model.S3Object in project beam by apache.

the class S3FileSystemTest method matchGlobWithSlashes.

@Test
public void matchGlobWithSlashes() throws IOException {
    S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Config("mys3"));
    S3ResourceId path = S3ResourceId.fromUri("mys3://testbucket/foo/bar\\baz*");
    ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(path.getBucket()).prefix(path.getKeyNonWildcardPrefix()).continuationToken(null).build();
    // Expected to be returned; prefix and wildcard/regex match
    S3Object firstMatch = S3Object.builder().key("foo/bar\\baz0").size(100L).lastModified(Instant.ofEpochMilli(1540000000001L)).build();
    // Expected to not be returned; prefix matches, but substring after wildcard does not
    S3Object secondMatch = S3Object.builder().key("foo/bar/baz1").size(200L).lastModified(Instant.ofEpochMilli(1540000000002L)).build();
    // Expected first request returns continuation token
    ListObjectsV2Response response = ListObjectsV2Response.builder().contents(firstMatch, secondMatch).build();
    when(s3FileSystem.getS3Client().listObjectsV2(argThat(new ListObjectsV2RequestArgumentMatches(request)))).thenReturn(response);
    // Expect object metadata queries for content encoding
    HeadObjectResponse headObjectResponse = HeadObjectResponse.builder().contentEncoding("").build();
    when(s3FileSystem.getS3Client().headObject(any(HeadObjectRequest.class))).thenReturn(headObjectResponse);
    assertThat(s3FileSystem.matchGlobPaths(ImmutableList.of(path)).get(0), MatchResultMatcher.create(ImmutableList.of(MatchResult.Metadata.builder().setIsReadSeekEfficient(true).setResourceId(S3ResourceId.fromComponents("mys3", path.getBucket(), firstMatch.key())).setSizeBytes(firstMatch.size()).setLastModifiedMillis(firstMatch.lastModified().toEpochMilli()).build())));
}

Example 98 with S3Object

use of software.amazon.awssdk.services.s3.model.S3Object in project beam by apache.

the class S3FileSystemTest method matchGlob.

@Test
public void matchGlob() throws IOException {
    S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Config("mys3"));
    S3ResourceId path = S3ResourceId.fromUri("mys3://testbucket/foo/bar*baz");
    ListObjectsV2Request firstRequest = ListObjectsV2Request.builder().bucket(path.getBucket()).prefix(path.getKeyNonWildcardPrefix()).continuationToken(null).build();
    // Expected to be returned; prefix and wildcard/regex match
    S3Object firstMatch = S3Object.builder().key("foo/bar0baz").size(100L).lastModified(Instant.ofEpochMilli(1540000000001L)).build();
    // Expected to not be returned; prefix matches, but substring after wildcard does not
    S3Object secondMatch = S3Object.builder().key("foo/bar1qux").size(200L).lastModified(Instant.ofEpochMilli(1540000000002L)).build();
    // Expected first request returns continuation token
    ListObjectsV2Response firstResponse = ListObjectsV2Response.builder().nextContinuationToken("token").contents(firstMatch, secondMatch).build();
    when(s3FileSystem.getS3Client().listObjectsV2(argThat(new ListObjectsV2RequestArgumentMatches(firstRequest)))).thenReturn(firstResponse);
    // Expect second request with continuation token
    ListObjectsV2Request secondRequest = ListObjectsV2Request.builder().bucket(path.getBucket()).prefix(path.getKeyNonWildcardPrefix()).continuationToken("token").build();
    // Expected to be returned; prefix and wildcard/regex match
    S3Object thirdMatch = S3Object.builder().key("foo/bar2baz").size(300L).lastModified(Instant.ofEpochMilli(1540000000003L)).build();
    // Expected second request returns third prefix match and no continuation token
    ListObjectsV2Response secondResponse = ListObjectsV2Response.builder().nextContinuationToken(null).contents(thirdMatch).build();
    when(s3FileSystem.getS3Client().listObjectsV2(argThat(new ListObjectsV2RequestArgumentMatches(secondRequest)))).thenReturn(secondResponse);
    // Expect object metadata queries for content encoding
    HeadObjectResponse headObjectResponse = HeadObjectResponse.builder().contentEncoding("").build();
    when(s3FileSystem.getS3Client().headObject(any(HeadObjectRequest.class))).thenReturn(headObjectResponse);
    assertThat(s3FileSystem.matchGlobPaths(ImmutableList.of(path)).get(0), MatchResultMatcher.create(ImmutableList.of(MatchResult.Metadata.builder().setIsReadSeekEfficient(true).setResourceId(S3ResourceId.fromComponents("mys3", path.getBucket(), firstMatch.key())).setSizeBytes(firstMatch.size()).setLastModifiedMillis(firstMatch.lastModified().toEpochMilli()).build(), MatchResult.Metadata.builder().setIsReadSeekEfficient(true).setResourceId(S3ResourceId.fromComponents("mys3", path.getBucket(), thirdMatch.key())).setSizeBytes(thirdMatch.size()).setLastModifiedMillis(thirdMatch.lastModified().toEpochMilli()).build())));
}

Example 99 with S3Object

use of software.amazon.awssdk.services.s3.model.S3Object in project hazelcast by hazelcast.

the class S3SinkTest method deleteObjects.

@AfterClass
public static void deleteObjects() {
    S3Client client = client();
    List<ObjectIdentifier> identifiers = listObjects(client).contents().stream().map(S3Object::key).map(key -> ObjectIdentifier.builder().key(key).build()).collect(Collectors.toList());
    if (!identifiers.isEmpty()) {
        client.deleteObjects(b -> b.bucket(BUCKET).delete(d -> d.objects(identifiers)));
    }
    int sleepMillis = (int) (SECONDS.toMillis(WAIT_AFTER_CLEANUP_IN_SECS) / 10);
    long deadline = System.currentTimeMillis() + SECONDS.toMillis(WAIT_AFTER_CLEANUP_IN_SECS);
    int keyCount;
    while ((keyCount = listObjects(client).keyCount()) != 0 && System.currentTimeMillis() < deadline) {
        logger.info("After sending the object cleanup request to S3, the bucket, " + BUCKET + ", still has " + keyCount + " keys.");
        sleepMillis(sleepMillis);
    }
    if (keyCount == 0) {
        logger.info("We finished waiting because we observe that the keys were deleted.");
    } else {
        logger.warning("There may still be keys in the bucket that are not deleted.\n" + "At our last check, keyCount was " + keyCount + "\n We finished waiting because of the timeout.");
    }
}

Example 100 with S3Object

use of software.amazon.awssdk.services.s3.model.S3Object in project syndesis-qe by syndesisio.

the class S3Utils method deleteS3Bucket.

public void deleteS3Bucket(String bucketName) {
    ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder().bucket(bucketName).build();
    ListObjectsV2Response listObjectsV2Response;
    do {
        listObjectsV2Response = s3client.listObjectsV2(listObjectsV2Request);
        for (S3Object s3Object : listObjectsV2Response.contents()) {
            s3client.deleteObject(DeleteObjectRequest.builder().bucket(bucketName).key(s3Object.key()).build());
        }
        listObjectsV2Request = ListObjectsV2Request.builder().bucket(bucketName).continuationToken(listObjectsV2Response.nextContinuationToken()).build();
    } while (listObjectsV2Response.isTruncated());
    s3client.deleteBucket(b -> b.bucket(bucketName));
}