Examples with HeadObjectResponse software.amazon.awssdk.services.s3.model.HeadObjectResponse used on opensource projects

Example 6 with HeadObjectResponse

use of software.amazon.awssdk.services.s3.model.HeadObjectResponse in project beam by apache.

the class S3FileSystemTest method matchNonGlobNotReadSeekEfficientWithS3Options.

@Test
public void matchNonGlobNotReadSeekEfficientWithS3Options() {
    S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options());
    S3ResourceId path = S3ResourceId.fromUri("s3://testbucket/testdirectory/filethatexists");
    long lastModifiedMillis = 1540000000000L;
    HeadObjectResponse headObjectResponse = HeadObjectResponse.builder().contentLength(100L).lastModified(Instant.ofEpochMilli(lastModifiedMillis)).contentEncoding("gzip").build();
    when(s3FileSystem.getS3Client().headObject(argThat(new GetHeadObjectRequestMatcher(HeadObjectRequest.builder().bucket(path.getBucket()).key(path.getKey()).build())))).thenReturn(headObjectResponse);
    MatchResult result = s3FileSystem.matchNonGlobPath(path);
    assertThat(result, MatchResultMatcher.create(ImmutableList.of(MatchResult.Metadata.builder().setSizeBytes(100).setLastModifiedMillis(lastModifiedMillis).setResourceId(path).setIsReadSeekEfficient(false).build())));
}

Example 7 with HeadObjectResponse

use of software.amazon.awssdk.services.s3.model.HeadObjectResponse in project beam by apache.

the class S3FileSystemTest method matchNonGlobWithS3Options.

@Test
public void matchNonGlobWithS3Options() {
    S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options());
    S3ResourceId path = S3ResourceId.fromUri("s3://testbucket/testdirectory/filethatexists");
    long lastModifiedMillis = 1540000000000L;
    HeadObjectResponse headObjectResponse = HeadObjectResponse.builder().contentLength(100L).contentEncoding("read-seek-efficient").lastModified(Instant.ofEpochMilli(lastModifiedMillis)).build();
    when(s3FileSystem.getS3Client().headObject(argThat(new GetHeadObjectRequestMatcher(HeadObjectRequest.builder().bucket(path.getBucket()).key(path.getKey()).build())))).thenReturn(headObjectResponse);
    MatchResult result = s3FileSystem.matchNonGlobPath(path);
    assertThat(result, MatchResultMatcher.create(ImmutableList.of(MatchResult.Metadata.builder().setSizeBytes(100).setLastModifiedMillis(lastModifiedMillis).setResourceId(path).setIsReadSeekEfficient(true).build())));
}

Example 8 with HeadObjectResponse

use of software.amazon.awssdk.services.s3.model.HeadObjectResponse in project beam by apache.

the class S3FileSystemTest method matchVariousInvokeThreadPool.

@Test
public void matchVariousInvokeThreadPool() throws IOException {
    S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Config("mys3"));
    SdkServiceException notFoundException = S3Exception.builder().message("mock exception").statusCode(404).build();
    S3ResourceId pathNotExist = S3ResourceId.fromUri("mys3://testbucket/testdirectory/nonexistentfile");
    HeadObjectRequest headObjectRequestNotExist = HeadObjectRequest.builder().bucket(pathNotExist.getBucket()).key(pathNotExist.getKey()).build();
    when(s3FileSystem.getS3Client().headObject(argThat(new GetHeadObjectRequestMatcher(headObjectRequestNotExist)))).thenThrow(notFoundException);
    SdkServiceException forbiddenException = SdkServiceException.builder().message("mock exception").statusCode(403).build();
    S3ResourceId pathForbidden = S3ResourceId.fromUri("mys3://testbucket/testdirectory/forbiddenfile");
    HeadObjectRequest headObjectRequestForbidden = HeadObjectRequest.builder().bucket(pathForbidden.getBucket()).key(pathForbidden.getKey()).build();
    when(s3FileSystem.getS3Client().headObject(argThat(new GetHeadObjectRequestMatcher(headObjectRequestForbidden)))).thenThrow(forbiddenException);
    S3ResourceId pathExist = S3ResourceId.fromUri("mys3://testbucket/testdirectory/filethatexists");
    HeadObjectRequest headObjectRequestExist = HeadObjectRequest.builder().bucket(pathExist.getBucket()).key(pathExist.getKey()).build();
    HeadObjectResponse s3ObjectMetadata = HeadObjectResponse.builder().contentLength(100L).contentEncoding("not-gzip").lastModified(Instant.ofEpochMilli(1540000000000L)).build();
    when(s3FileSystem.getS3Client().headObject(argThat(new GetHeadObjectRequestMatcher(headObjectRequestExist)))).thenReturn(s3ObjectMetadata);
    S3ResourceId pathGlob = S3ResourceId.fromUri("mys3://testbucket/path/part*");
    S3Object foundListObject = S3Object.builder().key("path/part-0").size(200L).lastModified(Instant.ofEpochMilli(1541000000000L)).build();
    ListObjectsV2Response listObjectsResponse = ListObjectsV2Response.builder().continuationToken(null).contents(foundListObject).build();
    when(s3FileSystem.getS3Client().listObjectsV2((ListObjectsV2Request) notNull())).thenReturn(listObjectsResponse);
    HeadObjectResponse headObjectResponse = HeadObjectResponse.builder().contentEncoding("").build();
    when(s3FileSystem.getS3Client().headObject(argThat(new GetHeadObjectRequestMatcher(HeadObjectRequest.builder().bucket(pathGlob.getBucket()).key("path/part-0").build())))).thenReturn(headObjectResponse);
    assertThat(s3FileSystem.match(ImmutableList.of(pathNotExist.toString(), pathForbidden.toString(), pathExist.toString(), pathGlob.toString())), contains(MatchResultMatcher.create(MatchResult.Status.NOT_FOUND, new FileNotFoundException()), MatchResultMatcher.create(MatchResult.Status.ERROR, new IOException(forbiddenException)), MatchResultMatcher.create(100, 1540000000000L, pathExist, true), MatchResultMatcher.create(200, 1541000000000L, S3ResourceId.fromComponents("mys3", pathGlob.getBucket(), foundListObject.key()), true)));
}

Example 9 with HeadObjectResponse

use of software.amazon.awssdk.services.s3.model.HeadObjectResponse in project beam by apache.

the class S3FileSystemTest method testCopy.

private void testCopy(S3FileSystem s3FileSystem, SSECustomerKey sseCustomerKey) throws IOException {
    S3ResourceId sourcePath = S3ResourceId.fromUri(s3FileSystem.getScheme() + "://bucket/from");
    S3ResourceId destinationPath = S3ResourceId.fromUri(s3FileSystem.getScheme() + "://bucket/to");
    HeadObjectResponse.Builder builder = HeadObjectResponse.builder().contentLength(0L);
    String sseCustomerKeyMd5 = toMd5(sseCustomerKey);
    if (sseCustomerKeyMd5 != null) {
        builder.sseCustomerKeyMD5(sseCustomerKeyMd5);
    }
    HeadObjectResponse headObjectResponse = builder.build();
    assertGetObjectHead(s3FileSystem, createObjectHeadRequest(sourcePath, sseCustomerKey), sseCustomerKeyMd5, headObjectResponse);
    s3FileSystem.copy(sourcePath, destinationPath);
    verify(s3FileSystem.getS3Client(), times(1)).copyObject(any(CopyObjectRequest.class));
    // we simulate a big object >= 5GB so it takes the multiPart path
    HeadObjectResponse bigHeadObjectResponse = headObjectResponse.toBuilder().contentLength(5_368_709_120L).build();
    assertGetObjectHead(s3FileSystem, createObjectHeadRequest(sourcePath, sseCustomerKey), sseCustomerKeyMd5, bigHeadObjectResponse);
    try {
        s3FileSystem.copy(sourcePath, destinationPath);
    } catch (NullPointerException e) {
    // ignore failing unmocked path, this is covered by testMultipartCopy test
    }
    verify(s3FileSystem.getS3Client(), never()).copyObject((CopyObjectRequest) null);
}

Example 10 with HeadObjectResponse

use of software.amazon.awssdk.services.s3.model.HeadObjectResponse in project beam by apache.

the class S3FileSystemTest method matchNonGlob.

@Test
public void matchNonGlob() {
    S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Config("mys3"));
    S3ResourceId path = S3ResourceId.fromUri("mys3://testbucket/testdirectory/filethatexists");
    long lastModifiedMillis = 1540000000000L;
    HeadObjectResponse headObjectResponse = HeadObjectResponse.builder().contentLength(100L).contentEncoding("read-seek-efficient").lastModified(Instant.ofEpochMilli(lastModifiedMillis)).build();
    when(s3FileSystem.getS3Client().headObject(argThat(new GetHeadObjectRequestMatcher(HeadObjectRequest.builder().bucket(path.getBucket()).key(path.getKey()).build())))).thenReturn(headObjectResponse);
    MatchResult result = s3FileSystem.matchNonGlobPath(path);
    assertThat(result, MatchResultMatcher.create(ImmutableList.of(MatchResult.Metadata.builder().setSizeBytes(100).setLastModifiedMillis(lastModifiedMillis).setResourceId(path).setIsReadSeekEfficient(true).build())));
}