Examples with S3Options org.apache.beam.sdk.io.aws2.options.S3Options used on opensource projects

Example 6 with S3Options

use of org.apache.beam.sdk.io.aws2.options.S3Options in project beam by apache.

the class S3TestUtils method configBuilder.

private static S3FileSystemConfiguration.Builder configBuilder(String scheme) {
    S3Options options = PipelineOptionsFactory.as(S3Options.class);
    options.setAwsRegion(Region.US_WEST_1);
    options.setS3UploadBufferSizeBytes(5_242_880);
    return S3FileSystemConfiguration.builderFrom(options).setScheme(scheme);
}

Example 7 with S3Options

use of org.apache.beam.sdk.io.aws2.options.S3Options in project beam by apache.

the class S3ResourceIdTest method testResourceIdTester.

@Test
public void testResourceIdTester() {
    S3Options options = PipelineOptionsFactory.create().as(S3Options.class);
    options.setAwsRegion(Region.US_WEST_1);
    FileSystems.setDefaultPipelineOptions(options);
    ResourceIdTester.runResourceIdBattery(S3ResourceId.fromUri("s3://bucket/foo/"));
}

Example 8 with S3Options

use of org.apache.beam.sdk.io.aws2.options.S3Options in project beam by apache.

the class S3FileSystemTest method matchNonGlobNotReadSeekEfficientWithS3Options.

@Test
public void matchNonGlobNotReadSeekEfficientWithS3Options() {
    S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options());
    S3ResourceId path = S3ResourceId.fromUri("s3://testbucket/testdirectory/filethatexists");
    long lastModifiedMillis = 1540000000000L;
    HeadObjectResponse headObjectResponse = HeadObjectResponse.builder().contentLength(100L).lastModified(Instant.ofEpochMilli(lastModifiedMillis)).contentEncoding("gzip").build();
    when(s3FileSystem.getS3Client().headObject(argThat(new GetHeadObjectRequestMatcher(HeadObjectRequest.builder().bucket(path.getBucket()).key(path.getKey()).build())))).thenReturn(headObjectResponse);
    MatchResult result = s3FileSystem.matchNonGlobPath(path);
    assertThat(result, MatchResultMatcher.create(ImmutableList.of(MatchResult.Metadata.builder().setSizeBytes(100).setLastModifiedMillis(lastModifiedMillis).setResourceId(path).setIsReadSeekEfficient(false).build())));
}

Example 9 with S3Options

use of org.apache.beam.sdk.io.aws2.options.S3Options in project beam by apache.

the class S3FileSystemTest method matchNonGlobNotFoundWithS3Options.

@Test
public void matchNonGlobNotFoundWithS3Options() {
    S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options());
    S3ResourceId path = S3ResourceId.fromUri("s3://testbucket/testdirectory/nonexistentfile");
    SdkServiceException exception = S3Exception.builder().message("mock exception").statusCode(404).build();
    when(s3FileSystem.getS3Client().headObject(argThat(new GetHeadObjectRequestMatcher(HeadObjectRequest.builder().bucket(path.getBucket()).key(path.getKey()).build())))).thenThrow(exception);
    MatchResult result = s3FileSystem.matchNonGlobPath(path);
    assertThat(result, MatchResultMatcher.create(MatchResult.Status.NOT_FOUND, new FileNotFoundException()));
}

Example 10 with S3Options

use of org.apache.beam.sdk.io.aws2.options.S3Options in project beam by apache.

the class S3FileSystemTest method matchNonGlobWithS3Options.

@Test
public void matchNonGlobWithS3Options() {
    S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options());
    S3ResourceId path = S3ResourceId.fromUri("s3://testbucket/testdirectory/filethatexists");
    long lastModifiedMillis = 1540000000000L;
    HeadObjectResponse headObjectResponse = HeadObjectResponse.builder().contentLength(100L).contentEncoding("read-seek-efficient").lastModified(Instant.ofEpochMilli(lastModifiedMillis)).build();
    when(s3FileSystem.getS3Client().headObject(argThat(new GetHeadObjectRequestMatcher(HeadObjectRequest.builder().bucket(path.getBucket()).key(path.getKey()).build())))).thenReturn(headObjectResponse);
    MatchResult result = s3FileSystem.matchNonGlobPath(path);
    assertThat(result, MatchResultMatcher.create(ImmutableList.of(MatchResult.Metadata.builder().setSizeBytes(100).setLastModifiedMillis(lastModifiedMillis).setResourceId(path).setIsReadSeekEfficient(true).build())));
}