Examples with DeterministicallyConstructTestRowFn org.apache.beam.sdk.io.common.TestRow.DeterministicallyConstructTestRowFn used on opensource projects

Example 1 with DeterministicallyConstructTestRowFn

use of org.apache.beam.sdk.io.common.TestRow.DeterministicallyConstructTestRowFn in project beam by apache.

the class DynamoDBIOIT method runWrite.

/**
 * Write test dataset to DynamoDB.
 */
private void runWrite() {
    int rows = env.options().getNumberOfRows();
    pipelineWrite.apply("Generate Sequence", GenerateSequence.from(0).to(rows)).apply("Prepare TestRows", ParDo.of(new DeterministicallyConstructTestRowFn())).apply("Write to DynamoDB", DynamoDBIO.<TestRow>write().withAwsClientsProvider(clientProvider()).withWriteRequestMapperFn(row -> buildWriteRequest(row)));
    pipelineWrite.run().waitUntilFinish();
}

Example 2 with DeterministicallyConstructTestRowFn

use of org.apache.beam.sdk.io.common.TestRow.DeterministicallyConstructTestRowFn in project beam by apache.

the class S3FileSystemIT method testWriteThenRead.

@Test
public void testWriteThenRead() {
    int rows = env.options().getNumberOfRows();
    // Write test dataset to S3.
    pipelineWrite.apply("Generate Sequence", GenerateSequence.from(0).to(rows)).apply("Prepare TestRows", ParDo.of(new DeterministicallyConstructTestRowFn())).apply("Prepare file rows", ParDo.of(new SelectNameFn())).apply("Write to S3 file", TextIO.write().to("s3://" + s3Bucket.name + "/test"));
    pipelineWrite.run().waitUntilFinish();
    // Read test dataset from S3.
    PCollection<String> output = pipelineRead.apply(TextIO.read().from("s3://" + s3Bucket.name + "/test*"));
    PAssert.thatSingleton(output.apply("Count All", Count.globally())).isEqualTo((long) rows);
    PAssert.that(output.apply(Combine.globally(new HashingFn()).withoutDefaults())).containsInAnyOrder(getExpectedHashForRowCount(rows));
    pipelineRead.run().waitUntilFinish();
}

Example 3 with DeterministicallyConstructTestRowFn

use of org.apache.beam.sdk.io.common.TestRow.DeterministicallyConstructTestRowFn in project beam by apache.

the class DynamoDBIOIT method runWrite.

/**
 * Write test dataset to DynamoDB.
 */
private void runWrite() {
    int rows = env.options().getNumberOfRows();
    pipelineWrite.apply("Generate Sequence", GenerateSequence.from(0).to(rows)).apply("Prepare TestRows", ParDo.of(new DeterministicallyConstructTestRowFn())).apply("Write to DynamoDB", DynamoDBIO.<TestRow>write().withWriteRequestMapperFn(row -> buildWriteRequest(row)));
    pipelineWrite.run().waitUntilFinish();
}

Example 4 with DeterministicallyConstructTestRowFn

use of org.apache.beam.sdk.io.common.TestRow.DeterministicallyConstructTestRowFn in project beam by apache.

the class S3FileSystemIT method testWriteThenRead.

@Test
public void testWriteThenRead() {
    int rows = env.options().getNumberOfRows();
    // Write test dataset to S3.
    pipelineWrite.apply("Generate Sequence", GenerateSequence.from(0).to(rows)).apply("Prepare TestRows", ParDo.of(new DeterministicallyConstructTestRowFn())).apply("Prepare file rows", ParDo.of(new SelectNameFn())).apply("Write to S3 file", TextIO.write().to("s3://" + s3Bucket.name + "/test"));
    pipelineWrite.run().waitUntilFinish();
    // Read test dataset from S3.
    PCollection<String> output = pipelineRead.apply(TextIO.read().from("s3://" + s3Bucket.name + "/test*"));
    PAssert.thatSingleton(output.apply(Count.globally())).isEqualTo((long) rows);
    PAssert.that(output.apply(Combine.globally(new HashingFn()).withoutDefaults())).containsInAnyOrder(getExpectedHashForRowCount(rows));
    pipelineRead.run().waitUntilFinish();
}

Example 5 with DeterministicallyConstructTestRowFn

use of org.apache.beam.sdk.io.common.TestRow.DeterministicallyConstructTestRowFn in project beam by apache.

the class SnsIOIT method testWriteThenRead.

@Test
public void testWriteThenRead() {
    ITOptions opts = env.options();
    int rows = opts.getNumberOfRows();
    // Write test dataset to SNS
    pipelineWrite.apply("Generate Sequence", GenerateSequence.from(0).to(rows)).apply("Prepare TestRows", ParDo.of(new DeterministicallyConstructTestRowFn())).apply("Write to SNS", SnsIO.<TestRow>write().withTopicArn(resources.snsTopic).withPublishRequestBuilder(r -> PublishRequest.builder().message(r.name())));
    // Read test dataset from SQS.
    PCollection<String> output = pipelineRead.apply("Read from SQS", SqsIO.read().withQueueUrl(resources.sqsQueue).withMaxNumRecords(rows)).apply("Extract message", MapElements.into(strings()).via(SnsIOIT::extractMessage));
    PAssert.thatSingleton(output.apply("Count All", Count.globally())).isEqualTo((long) rows);
    PAssert.that(output.apply(Combine.globally(new HashingFn()).withoutDefaults())).containsInAnyOrder(getExpectedHashForRowCount(rows));
    pipelineWrite.run();
    pipelineRead.run();
}