Examples with Write org.apache.beam.sdk.io.aws2.sqs.SqsIO.Write used on opensource projects

Example 1 with Write

use of org.apache.beam.sdk.io.aws2.sqs.SqsIO.Write in project beam by apache.

the class SqsIOWriteTest method testBuildWithCredentialsProviderAndRegionAndEndpoint.

@Test
public void testBuildWithCredentialsProviderAndRegionAndEndpoint() {
    Region region = Region.US_EAST_1;
    AwsCredentialsProvider credentialsProvider = DefaultCredentialsProvider.create();
    URI endpoint = URI.create("localhost:9999");
    Write write = SqsIO.write().withSqsClientProvider(credentialsProvider, region.id(), endpoint);
    assertThat(write.getClientConfiguration()).isEqualTo(ClientConfiguration.create(credentialsProvider, region, endpoint));
}

Example 2 with Write

use of org.apache.beam.sdk.io.aws2.sqs.SqsIO.Write in project beam by apache.

the class DynamoDBIOIT method runWrite.

/**
 * Write test dataset to DynamoDB.
 */
private void runWrite() {
    int rows = env.options().getNumberOfRows();
    pipelineWrite.apply("Generate Sequence", GenerateSequence.from(0).to(rows)).apply("Prepare TestRows", ParDo.of(new DeterministicallyConstructTestRowFn())).apply("Write to DynamoDB", DynamoDBIO.<TestRow>write().withWriteRequestMapperFn(row -> buildWriteRequest(row)));
    pipelineWrite.run().waitUntilFinish();
}

Example 3 with Write

use of org.apache.beam.sdk.io.aws2.sqs.SqsIO.Write in project beam by apache.

the class SnsIOIT method testWriteThenRead.

@Test
public void testWriteThenRead() {
    ITOptions opts = env.options();
    int rows = opts.getNumberOfRows();
    // Write test dataset to SNS
    pipelineWrite.apply("Generate Sequence", GenerateSequence.from(0).to(rows)).apply("Prepare TestRows", ParDo.of(new DeterministicallyConstructTestRowFn())).apply("Write to SNS", SnsIO.<TestRow>write().withTopicArn(resources.snsTopic).withPublishRequestBuilder(r -> PublishRequest.builder().message(r.name())));
    // Read test dataset from SQS.
    PCollection<String> output = pipelineRead.apply("Read from SQS", SqsIO.read().withQueueUrl(resources.sqsQueue).withMaxNumRecords(rows)).apply("Extract message", MapElements.into(strings()).via(SnsIOIT::extractMessage));
    PAssert.thatSingleton(output.apply("Count All", Count.globally())).isEqualTo((long) rows);
    PAssert.that(output.apply(Combine.globally(new HashingFn()).withoutDefaults())).containsInAnyOrder(getExpectedHashForRowCount(rows));
    pipelineWrite.run();
    pipelineRead.run();
}

Example 4 with Write

use of org.apache.beam.sdk.io.aws2.sqs.SqsIO.Write in project beam by apache.

the class S3FileSystemTest method testWriteAndRead.

@Test
public void testWriteAndRead() throws IOException {
    S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Config("mys3"), client);
    client.createBucket(CreateBucketRequest.builder().bucket("testbucket").build());
    byte[] writtenArray = new byte[] { 0 };
    ByteBuffer bb = ByteBuffer.allocate(writtenArray.length);
    bb.put(writtenArray);
    // First create an object and write data to it
    S3ResourceId path = S3ResourceId.fromUri("mys3://testbucket/foo/bar.txt");
    WritableByteChannel writableByteChannel = s3FileSystem.create(path, CreateOptions.StandardCreateOptions.builder().setMimeType("application/text").build());
    writableByteChannel.write(bb);
    writableByteChannel.close();
    // Now read the same object
    ByteBuffer bb2 = ByteBuffer.allocate(writtenArray.length);
    ReadableByteChannel open = s3FileSystem.open(path);
    open.read(bb2);
    // And compare the content with the one that was written
    byte[] readArray = bb2.array();
    assertArrayEquals(readArray, writtenArray);
    open.close();
}

Example 5 with Write

use of org.apache.beam.sdk.io.aws2.sqs.SqsIO.Write in project beam by apache.

the class SqsIOWriteTest method testBuildWithCredentialsProviderAndRegion.

@Test
public void testBuildWithCredentialsProviderAndRegion() {
    Region region = Region.US_EAST_1;
    AwsCredentialsProvider credentialsProvider = DefaultCredentialsProvider.create();
    Write write = SqsIO.write().withSqsClientProvider(credentialsProvider, region.id());
    assertThat(write.getClientConfiguration()).isEqualTo(ClientConfiguration.create(credentialsProvider, region, null));
}