Examples with S3FileSystem org.apache.hop.vfs.s3.s3.vfs.S3FileSystem used on opensource projects

Example 21 with S3FileSystem

use of org.apache.hop.vfs.s3.s3.vfs.S3FileSystem in project beam by apache.

the class S3FileSystemTest method matchNonGlobNotReadSeekEfficientWithS3Options.

@Test
public void matchNonGlobNotReadSeekEfficientWithS3Options() {
    S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options());
    S3ResourceId path = S3ResourceId.fromUri("s3://testbucket/testdirectory/filethatexists");
    long lastModifiedMillis = 1540000000000L;
    HeadObjectResponse headObjectResponse = HeadObjectResponse.builder().contentLength(100L).lastModified(Instant.ofEpochMilli(lastModifiedMillis)).contentEncoding("gzip").build();
    when(s3FileSystem.getS3Client().headObject(argThat(new GetHeadObjectRequestMatcher(HeadObjectRequest.builder().bucket(path.getBucket()).key(path.getKey()).build())))).thenReturn(headObjectResponse);
    MatchResult result = s3FileSystem.matchNonGlobPath(path);
    assertThat(result, MatchResultMatcher.create(ImmutableList.of(MatchResult.Metadata.builder().setSizeBytes(100).setLastModifiedMillis(lastModifiedMillis).setResourceId(path).setIsReadSeekEfficient(false).build())));
}

Example 22 with S3FileSystem

use of org.apache.hop.vfs.s3.s3.vfs.S3FileSystem in project beam by apache.

the class S3FileSystemTest method matchGlobWithSlashes.

@Test
public void matchGlobWithSlashes() throws IOException {
    S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Config("mys3"));
    S3ResourceId path = S3ResourceId.fromUri("mys3://testbucket/foo/bar\\baz*");
    ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(path.getBucket()).prefix(path.getKeyNonWildcardPrefix()).continuationToken(null).build();
    // Expected to be returned; prefix and wildcard/regex match
    S3Object firstMatch = S3Object.builder().key("foo/bar\\baz0").size(100L).lastModified(Instant.ofEpochMilli(1540000000001L)).build();
    // Expected to not be returned; prefix matches, but substring after wildcard does not
    S3Object secondMatch = S3Object.builder().key("foo/bar/baz1").size(200L).lastModified(Instant.ofEpochMilli(1540000000002L)).build();
    // Expected first request returns continuation token
    ListObjectsV2Response response = ListObjectsV2Response.builder().contents(firstMatch, secondMatch).build();
    when(s3FileSystem.getS3Client().listObjectsV2(argThat(new ListObjectsV2RequestArgumentMatches(request)))).thenReturn(response);
    // Expect object metadata queries for content encoding
    HeadObjectResponse headObjectResponse = HeadObjectResponse.builder().contentEncoding("").build();
    when(s3FileSystem.getS3Client().headObject(any(HeadObjectRequest.class))).thenReturn(headObjectResponse);
    assertThat(s3FileSystem.matchGlobPaths(ImmutableList.of(path)).get(0), MatchResultMatcher.create(ImmutableList.of(MatchResult.Metadata.builder().setIsReadSeekEfficient(true).setResourceId(S3ResourceId.fromComponents("mys3", path.getBucket(), firstMatch.key())).setSizeBytes(firstMatch.size()).setLastModifiedMillis(firstMatch.lastModified().toEpochMilli()).build())));
}

Example 23 with S3FileSystem

use of org.apache.hop.vfs.s3.s3.vfs.S3FileSystem in project beam by apache.

the class S3FileSystemTest method matchNonGlobNullContentEncoding.

@Test
public void matchNonGlobNullContentEncoding() {
    S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Config("s3"));
    S3ResourceId path = S3ResourceId.fromUri("s3://testbucket/testdirectory/filethatexists");
    long lastModifiedMillis = 1540000000000L;
    HeadObjectResponse headObjectResponse = HeadObjectResponse.builder().contentLength(100L).lastModified(Instant.ofEpochMilli(lastModifiedMillis)).contentEncoding(null).build();
    when(s3FileSystem.getS3Client().headObject(argThat(new GetHeadObjectRequestMatcher(HeadObjectRequest.builder().bucket(path.getBucket()).key(path.getKey()).build())))).thenReturn(headObjectResponse);
    MatchResult result = s3FileSystem.matchNonGlobPath(path);
    assertThat(result, MatchResultMatcher.create(ImmutableList.of(MatchResult.Metadata.builder().setSizeBytes(100).setLastModifiedMillis(lastModifiedMillis).setResourceId(path).setIsReadSeekEfficient(true).build())));
}

Example 24 with S3FileSystem

use of org.apache.hop.vfs.s3.s3.vfs.S3FileSystem in project beam by apache.

the class S3FileSystemTest method matchGlob.

@Test
public void matchGlob() throws IOException {
    S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Config("mys3"));
    S3ResourceId path = S3ResourceId.fromUri("mys3://testbucket/foo/bar*baz");
    ListObjectsV2Request firstRequest = ListObjectsV2Request.builder().bucket(path.getBucket()).prefix(path.getKeyNonWildcardPrefix()).continuationToken(null).build();
    // Expected to be returned; prefix and wildcard/regex match
    S3Object firstMatch = S3Object.builder().key("foo/bar0baz").size(100L).lastModified(Instant.ofEpochMilli(1540000000001L)).build();
    // Expected to not be returned; prefix matches, but substring after wildcard does not
    S3Object secondMatch = S3Object.builder().key("foo/bar1qux").size(200L).lastModified(Instant.ofEpochMilli(1540000000002L)).build();
    // Expected first request returns continuation token
    ListObjectsV2Response firstResponse = ListObjectsV2Response.builder().nextContinuationToken("token").contents(firstMatch, secondMatch).build();
    when(s3FileSystem.getS3Client().listObjectsV2(argThat(new ListObjectsV2RequestArgumentMatches(firstRequest)))).thenReturn(firstResponse);
    // Expect second request with continuation token
    ListObjectsV2Request secondRequest = ListObjectsV2Request.builder().bucket(path.getBucket()).prefix(path.getKeyNonWildcardPrefix()).continuationToken("token").build();
    // Expected to be returned; prefix and wildcard/regex match
    S3Object thirdMatch = S3Object.builder().key("foo/bar2baz").size(300L).lastModified(Instant.ofEpochMilli(1540000000003L)).build();
    // Expected second request returns third prefix match and no continuation token
    ListObjectsV2Response secondResponse = ListObjectsV2Response.builder().nextContinuationToken(null).contents(thirdMatch).build();
    when(s3FileSystem.getS3Client().listObjectsV2(argThat(new ListObjectsV2RequestArgumentMatches(secondRequest)))).thenReturn(secondResponse);
    // Expect object metadata queries for content encoding
    HeadObjectResponse headObjectResponse = HeadObjectResponse.builder().contentEncoding("").build();
    when(s3FileSystem.getS3Client().headObject(any(HeadObjectRequest.class))).thenReturn(headObjectResponse);
    assertThat(s3FileSystem.matchGlobPaths(ImmutableList.of(path)).get(0), MatchResultMatcher.create(ImmutableList.of(MatchResult.Metadata.builder().setIsReadSeekEfficient(true).setResourceId(S3ResourceId.fromComponents("mys3", path.getBucket(), firstMatch.key())).setSizeBytes(firstMatch.size()).setLastModifiedMillis(firstMatch.lastModified().toEpochMilli()).build(), MatchResult.Metadata.builder().setIsReadSeekEfficient(true).setResourceId(S3ResourceId.fromComponents("mys3", path.getBucket(), thirdMatch.key())).setSizeBytes(thirdMatch.size()).setLastModifiedMillis(thirdMatch.lastModified().toEpochMilli()).build())));
}

Example 25 with S3FileSystem

use of org.apache.hop.vfs.s3.s3.vfs.S3FileSystem in project beam by apache.

the class S3FileSystemTest method testGetSchemeWithS3Options.

@Test
public void testGetSchemeWithS3Options() {
    S3FileSystem s3FileSystem = new S3FileSystem(s3Options());
    assertEquals("s3", s3FileSystem.getScheme());
}