Examples with S3FileSystem org.apache.hop.vfs.s3.s3.vfs.S3FileSystem used on opensource projects

Example 11 with S3FileSystem

use of org.apache.hop.vfs.s3.s3.vfs.S3FileSystem in project beam by apache.

the class S3FileSystemTest method matchNonGlobNotFound.

@Test
public void matchNonGlobNotFound() {
    S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Config("s3"));
    S3ResourceId path = S3ResourceId.fromUri("s3://testbucket/testdirectory/nonexistentfile");
    SdkServiceException exception = S3Exception.builder().message("mock exception").statusCode(404).build();
    when(s3FileSystem.getS3Client().headObject(argThat(new GetHeadObjectRequestMatcher(HeadObjectRequest.builder().bucket(path.getBucket()).key(path.getKey()).build())))).thenThrow(exception);
    MatchResult result = s3FileSystem.matchNonGlobPath(path);
    assertThat(result, MatchResultMatcher.create(MatchResult.Status.NOT_FOUND, new FileNotFoundException()));
}

Example 12 with S3FileSystem

use of org.apache.hop.vfs.s3.s3.vfs.S3FileSystem in project beam by apache.

the class S3FileSystemTest method matchNonGlobForbiddenWithS3Options.

@Test
public void matchNonGlobForbiddenWithS3Options() {
    S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options());
    SdkServiceException exception = S3Exception.builder().message("mock exception").statusCode(403).build();
    S3ResourceId path = S3ResourceId.fromUri("s3://testbucket/testdirectory/keyname");
    when(s3FileSystem.getS3Client().headObject(argThat(new GetHeadObjectRequestMatcher(HeadObjectRequest.builder().bucket(path.getBucket()).key(path.getKey()).build())))).thenThrow(exception);
    assertThat(s3FileSystem.matchNonGlobPath(path), MatchResultMatcher.create(MatchResult.Status.ERROR, new IOException(exception)));
}

Example 13 with S3FileSystem

use of org.apache.hop.vfs.s3.s3.vfs.S3FileSystem in project beam by apache.

the class S3FileSystemTest method testGetPathStyleAccessEnabledWithS3Options.

@Test
public void testGetPathStyleAccessEnabledWithS3Options() throws URISyntaxException {
    S3FileSystem s3FileSystem = new S3FileSystem(s3OptionsWithPathStyleAccessEnabled());
    URL s3Url = s3FileSystem.getS3Client().utilities().getUrl(GetUrlRequest.builder().bucket("bucket").key("file").build());
    assertEquals("https://s3.us-west-1.amazonaws.com/bucket/file", s3Url.toURI().toString());
}

Example 14 with S3FileSystem

use of org.apache.hop.vfs.s3.s3.vfs.S3FileSystem in project beam by apache.

the class S3FileSystemTest method matchNonGlob.

@Test
public void matchNonGlob() {
    S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Config("mys3"));
    S3ResourceId path = S3ResourceId.fromUri("mys3://testbucket/testdirectory/filethatexists");
    long lastModifiedMillis = 1540000000000L;
    HeadObjectResponse headObjectResponse = HeadObjectResponse.builder().contentLength(100L).contentEncoding("read-seek-efficient").lastModified(Instant.ofEpochMilli(lastModifiedMillis)).build();
    when(s3FileSystem.getS3Client().headObject(argThat(new GetHeadObjectRequestMatcher(HeadObjectRequest.builder().bucket(path.getBucket()).key(path.getKey()).build())))).thenReturn(headObjectResponse);
    MatchResult result = s3FileSystem.matchNonGlobPath(path);
    assertThat(result, MatchResultMatcher.create(ImmutableList.of(MatchResult.Metadata.builder().setSizeBytes(100).setLastModifiedMillis(lastModifiedMillis).setResourceId(path).setIsReadSeekEfficient(true).build())));
}

Example 15 with S3FileSystem

use of org.apache.hop.vfs.s3.s3.vfs.S3FileSystem in project beam by apache.

the class S3FileSystemTest method matchNonGlobNotFoundWithS3Options.

@Test
public void matchNonGlobNotFoundWithS3Options() {
    S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options());
    S3ResourceId path = S3ResourceId.fromUri("s3://testbucket/testdirectory/nonexistentfile");
    SdkServiceException exception = S3Exception.builder().message("mock exception").statusCode(404).build();
    when(s3FileSystem.getS3Client().headObject(argThat(new GetHeadObjectRequestMatcher(HeadObjectRequest.builder().bucket(path.getBucket()).key(path.getKey()).build())))).thenThrow(exception);
    MatchResult result = s3FileSystem.matchNonGlobPath(path);
    assertThat(result, MatchResultMatcher.create(MatchResult.Status.NOT_FOUND, new FileNotFoundException()));
}