Examples with Read org.apache.beam.sdk.io.aws2.kinesis.KinesisIO.Read used on opensource projects

Example 6 with Read

use of org.apache.beam.sdk.io.aws2.kinesis.KinesisIO.Read in project beam by apache.

the class S3FileSystemTest method matchNonGlobWithS3Options.

@Test
public void matchNonGlobWithS3Options() {
    S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options());
    S3ResourceId path = S3ResourceId.fromUri("s3://testbucket/testdirectory/filethatexists");
    long lastModifiedMillis = 1540000000000L;
    HeadObjectResponse headObjectResponse = HeadObjectResponse.builder().contentLength(100L).contentEncoding("read-seek-efficient").lastModified(Instant.ofEpochMilli(lastModifiedMillis)).build();
    when(s3FileSystem.getS3Client().headObject(argThat(new GetHeadObjectRequestMatcher(HeadObjectRequest.builder().bucket(path.getBucket()).key(path.getKey()).build())))).thenReturn(headObjectResponse);
    MatchResult result = s3FileSystem.matchNonGlobPath(path);
    assertThat(result, MatchResultMatcher.create(ImmutableList.of(MatchResult.Metadata.builder().setSizeBytes(100).setLastModifiedMillis(lastModifiedMillis).setResourceId(path).setIsReadSeekEfficient(true).build())));
}

Example 7 with Read

use of org.apache.beam.sdk.io.aws2.kinesis.KinesisIO.Read in project beam by apache.

the class S3FileSystemTest method matchNonGlob.

@Test
public void matchNonGlob() {
    S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Config("mys3"));
    S3ResourceId path = S3ResourceId.fromUri("mys3://testbucket/testdirectory/filethatexists");
    long lastModifiedMillis = 1540000000000L;
    HeadObjectResponse headObjectResponse = HeadObjectResponse.builder().contentLength(100L).contentEncoding("read-seek-efficient").lastModified(Instant.ofEpochMilli(lastModifiedMillis)).build();
    when(s3FileSystem.getS3Client().headObject(argThat(new GetHeadObjectRequestMatcher(HeadObjectRequest.builder().bucket(path.getBucket()).key(path.getKey()).build())))).thenReturn(headObjectResponse);
    MatchResult result = s3FileSystem.matchNonGlobPath(path);
    assertThat(result, MatchResultMatcher.create(ImmutableList.of(MatchResult.Metadata.builder().setSizeBytes(100).setLastModifiedMillis(lastModifiedMillis).setResourceId(path).setIsReadSeekEfficient(true).build())));
}

Example 8 with Read

use of org.apache.beam.sdk.io.aws2.kinesis.KinesisIO.Read in project beam by apache.

the class SqsIOReadTest method testBuildWithCredentialsProviderAndRegion.

@Test
public void testBuildWithCredentialsProviderAndRegion() {
    Region region = Region.US_EAST_1;
    AwsCredentialsProvider credentialsProvider = DefaultCredentialsProvider.create();
    Read read = SqsIO.read().withSqsClientProvider(credentialsProvider, region.id());
    assertThat(read.clientConfiguration()).isEqualTo(ClientConfiguration.create(credentialsProvider, region, null));
}

Example 9 with Read

use of org.apache.beam.sdk.io.aws2.kinesis.KinesisIO.Read in project beam by apache.

the class DynamoDBIOIT method runRead.

/**
 * Read test dataset from DynamoDB.
 */
private void runRead() {
    int rows = env.options().getNumberOfRows();
    PCollection<Map<String, AttributeValue>> records = pipelineRead.apply("Read from DynamoDB", DynamoDBIO.read().withScanRequestFn(in -> buildScanRequest()).items()).apply("Flatten result", Flatten.iterables());
    PAssert.thatSingleton(records.apply("Count All", Count.globally())).isEqualTo((long) rows);
    PCollection<String> consolidatedHashcode = records.apply(MapElements.into(strings()).via(record -> record.get(COL_NAME).s())).apply("Hash records", Combine.globally(new HashingFn()).withoutDefaults());
    PAssert.that(consolidatedHashcode).containsInAnyOrder(getExpectedHashForRowCount(rows));
    pipelineRead.run().waitUntilFinish();
}

Example 10 with Read

use of org.apache.beam.sdk.io.aws2.kinesis.KinesisIO.Read in project beam by apache.

the class KinesisIOReadTest method testBuildWithBasicCredentials.

@Test
public void testBuildWithBasicCredentials() {
    Region region = Region.US_EAST_1;
    AwsBasicCredentials credentials = AwsBasicCredentials.create(KEY, SECRET);
    StaticCredentialsProvider credentialsProvider = StaticCredentialsProvider.create(credentials);
    Read read = KinesisIO.read().withAWSClientsProvider(KEY, SECRET, region);
    assertThat(read.getClientConfiguration()).isEqualTo(ClientConfiguration.create(credentialsProvider, region, null));
}