Examples with S3FileSystem org.apache.hop.vfs.s3.s3.vfs.S3FileSystem used on opensource projects

Example 6 with S3FileSystem

use of org.apache.hop.vfs.s3.s3.vfs.S3FileSystem in project beam by apache.

the class S3FileSystemTest method testGetScheme.

@Test
public void testGetScheme() {
    S3FileSystem s3FileSystem = new S3FileSystem(s3Config("s3"));
    assertEquals("s3", s3FileSystem.getScheme());
    s3FileSystem = new S3FileSystem(s3Config("other"));
    assertEquals("other", s3FileSystem.getScheme());
}

Example 7 with S3FileSystem

use of org.apache.hop.vfs.s3.s3.vfs.S3FileSystem in project beam by apache.

the class S3FileSystemTest method matchNonGlobNullContentEncodingWithOptions.

@Test
public void matchNonGlobNullContentEncodingWithOptions() {
    S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options());
    S3ResourceId path = S3ResourceId.fromUri("s3://testbucket/testdirectory/filethatexists");
    long lastModifiedMillis = 1540000000000L;
    HeadObjectResponse headObjectResponse = HeadObjectResponse.builder().contentLength(100L).lastModified(Instant.ofEpochMilli(lastModifiedMillis)).contentEncoding(null).build();
    when(s3FileSystem.getS3Client().headObject(argThat(new GetHeadObjectRequestMatcher(HeadObjectRequest.builder().bucket(path.getBucket()).key(path.getKey()).build())))).thenReturn(headObjectResponse);
    MatchResult result = s3FileSystem.matchNonGlobPath(path);
    assertThat(result, MatchResultMatcher.create(ImmutableList.of(MatchResult.Metadata.builder().setSizeBytes(100).setLastModifiedMillis(lastModifiedMillis).setResourceId(path).setIsReadSeekEfficient(true).build())));
}

Example 8 with S3FileSystem

use of org.apache.hop.vfs.s3.s3.vfs.S3FileSystem in project beam by apache.

the class S3FileSystemTest method matchNonGlobForbidden.

@Test
public void matchNonGlobForbidden() {
    S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Config("s3"));
    SdkServiceException exception = S3Exception.builder().message("mock exception").statusCode(403).build();
    S3ResourceId path = S3ResourceId.fromUri("s3://testbucket/testdirectory/keyname");
    when(s3FileSystem.getS3Client().headObject(argThat(new GetHeadObjectRequestMatcher(HeadObjectRequest.builder().bucket(path.getBucket()).key(path.getKey()).build())))).thenThrow(exception);
    assertThat(s3FileSystem.matchNonGlobPath(path), MatchResultMatcher.create(MatchResult.Status.ERROR, new IOException(exception)));
}

Example 9 with S3FileSystem

use of org.apache.hop.vfs.s3.s3.vfs.S3FileSystem in project beam by apache.

the class S3FileSystemTest method deleteThousandsOfObjectsInMultipleBucketsWithS3Options.

@Test
public void deleteThousandsOfObjectsInMultipleBucketsWithS3Options() throws IOException {
    S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options());
    List<String> buckets = ImmutableList.of("bucket1", "bucket2");
    List<String> keys = new ArrayList<>();
    for (int i = 0; i < 2500; i++) {
        keys.add(String.format("key-%d", i));
    }
    List<S3ResourceId> paths = new ArrayList<>();
    for (String bucket : buckets) {
        for (String key : keys) {
            paths.add(S3ResourceId.fromComponents("s3", bucket, key));
        }
    }
    s3FileSystem.delete(paths);
    // Should require 6 calls to delete 2500 objects in each of 2 buckets.
    verify(s3FileSystem.getS3Client(), times(6)).deleteObjects(any(DeleteObjectsRequest.class));
}

Example 10 with S3FileSystem

use of org.apache.hop.vfs.s3.s3.vfs.S3FileSystem in project beam by apache.

the class S3FileSystemTest method matchNonGlobNotReadSeekEfficient.

@Test
public void matchNonGlobNotReadSeekEfficient() {
    S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Config("s3"));
    S3ResourceId path = S3ResourceId.fromUri("s3://testbucket/testdirectory/filethatexists");
    long lastModifiedMillis = 1540000000000L;
    HeadObjectResponse headObjectResponse = HeadObjectResponse.builder().contentLength(100L).lastModified(Instant.ofEpochMilli(lastModifiedMillis)).contentEncoding("gzip").build();
    when(s3FileSystem.getS3Client().headObject(argThat(new GetHeadObjectRequestMatcher(HeadObjectRequest.builder().bucket(path.getBucket()).key(path.getKey()).build())))).thenReturn(headObjectResponse);
    MatchResult result = s3FileSystem.matchNonGlobPath(path);
    assertThat(result, MatchResultMatcher.create(ImmutableList.of(MatchResult.Metadata.builder().setSizeBytes(100).setLastModifiedMillis(lastModifiedMillis).setResourceId(path).setIsReadSeekEfficient(false).build())));
}