Examples with TestRow org.apache.beam.sdk.io.common.TestRow used on opensource projects

Example 6 with TestRow

use of org.apache.beam.sdk.io.common.TestRow in project beam by apache.

the class StreamingSnowflakeIOIT method writeStreamToSnowflake.

private void writeStreamToSnowflake() {
    TestStream<TestRow> stringsStream = TestStream.create(SerializableCoder.of(TestRow.class)).advanceWatermarkTo(Instant.now()).addElements(testRows.get(0), testRows.subList(1, testRows.size()).toArray(new TestRow[0])).advanceWatermarkToInfinity();
    pipeline.apply(stringsStream).apply("Write SnowflakeIO", SnowflakeIO.<TestRow>write().withDataSourceConfiguration(dc).withUserDataMapper(getTestRowDataMapper()).withSnowPipe(options.getSnowPipe()).withStorageIntegrationName(storageIntegrationName).withStagingBucketName(stagingBucketName).withFlushTimeLimit(Duration.millis(18000)).withFlushRowLimit(50000).withDebugMode(StreamingLogLevel.ERROR));
    PipelineResult pipelineResult = pipeline.run(options);
    pipelineResult.waitUntilFinish();
}

Example 7 with TestRow

use of org.apache.beam.sdk.io.common.TestRow in project beam by apache.

the class BatchSnowflakeIOIT method runRead.

private PipelineResult runRead() {
    PCollection<TestRow> namesAndIds = pipelineRead.apply(SnowflakeIO.<TestRow>read().withDataSourceConfiguration(dataSourceConfiguration).fromTable(tableName).withStagingBucketName(stagingBucketName).withStorageIntegrationName(storageIntegrationName).withCsvMapper(getTestRowCsvMapper()).withCoder(SerializableCoder.of(TestRow.class)));
    PAssert.thatSingleton(namesAndIds.apply("Count All", Count.globally())).isEqualTo((long) numberOfRecords);
    PCollection<String> consolidatedHashcode = namesAndIds.apply(ParDo.of(new TestRow.SelectNameFn())).apply("Hash row contents", Combine.globally(new HashingFn()).withoutDefaults());
    PAssert.that(consolidatedHashcode).containsInAnyOrder(TestRow.getExpectedHashForRowCount(numberOfRecords));
    return pipelineRead.run();
}