Examples with PipelineOptions org.apache.beam.sdk.options.PipelineOptions used on opensource projects

Example 16 with PipelineOptions

use of org.apache.beam.sdk.options.PipelineOptions in project beam by apache.

the class UnboundedReadFromBoundedSourceTest method testReadBeforeStart.

@Test
public void testReadBeforeStart() throws Exception {
    thrown.expect(NoSuchElementException.class);
    BoundedSource<Long> countingSource = CountingSource.upTo(100);
    BoundedToUnboundedSourceAdapter<Long> unboundedSource = new BoundedToUnboundedSourceAdapter<>(countingSource);
    PipelineOptions options = PipelineOptionsFactory.create();
    unboundedSource.createReader(options, null).getCurrent();
}

Example 17 with PipelineOptions

use of org.apache.beam.sdk.options.PipelineOptions in project beam by apache.

the class ReadTest method testPrimitiveDisplayData.

private void testPrimitiveDisplayData(boolean isStreaming) {
    PipelineOptions options = DisplayDataEvaluator.getDefaultOptions();
    options.as(StreamingOptions.class).setStreaming(isStreaming);
    DisplayDataEvaluator evaluator = DisplayDataEvaluator.create(options);
    SerializableBoundedSource boundedSource = new SerializableBoundedSource() {

        @Override
        public void populateDisplayData(DisplayData.Builder builder) {
            builder.add(DisplayData.item("foo", "bar"));
        }
    };
    SerializableUnboundedSource unboundedSource = new SerializableUnboundedSource() {

        @Override
        public void populateDisplayData(DisplayData.Builder builder) {
            builder.add(DisplayData.item("foo", "bar"));
        }
    };
    Read.Bounded<String> bounded = Read.from(boundedSource);
    BoundedReadFromUnboundedSource<String> unbounded = Read.from(unboundedSource).withMaxNumRecords(1234);
    Set<DisplayData> boundedDisplayData = evaluator.displayDataForPrimitiveSourceTransforms(bounded);
    assertThat(boundedDisplayData, hasItem(hasDisplayItem("source", boundedSource.getClass())));
    assertThat(boundedDisplayData, hasItem(includesDisplayDataFor("source", boundedSource)));
    Set<DisplayData> unboundedDisplayData = evaluator.displayDataForPrimitiveSourceTransforms(unbounded);
    assertThat(unboundedDisplayData, hasItem(hasDisplayItem("source")));
    assertThat(unboundedDisplayData, hasItem(includesDisplayDataFor("source", unboundedSource)));
}

Example 18 with PipelineOptions

use of org.apache.beam.sdk.options.PipelineOptions in project beam by apache.

the class TextIOTest method testInitialSplitGzipModeTxt.

@Test
public void testInitialSplitGzipModeTxt() throws Exception {
    PipelineOptions options = TestPipeline.testingPipelineOptions();
    long desiredBundleSize = 1000;
    // Sanity check: file is at least 2 bundles long.
    assertThat(largeTxt.length(), greaterThan(2 * desiredBundleSize));
    FileBasedSource<String> source = TextIO.read().from(largeTxt.getPath()).withCompressionType(GZIP).getSource();
    List<? extends FileBasedSource<String>> splits = source.split(desiredBundleSize, options);
    // Exactly 1 split, even though splittable text file, since using GZIP mode.
    assertThat(splits, hasSize(equalTo(1)));
    SourceTestUtils.assertSourcesEqualReferenceSource(source, splits, options);
}

Example 19 with PipelineOptions

use of org.apache.beam.sdk.options.PipelineOptions in project beam by apache.

the class TextIOTest method testInitialSplitAutoModeGz.

@Test
public void testInitialSplitAutoModeGz() throws Exception {
    long desiredBundleSize = 1000;
    PipelineOptions options = TestPipeline.testingPipelineOptions();
    // Sanity check: file is at least 2 bundles long.
    assertThat(largeGz.length(), greaterThan(2 * desiredBundleSize));
    FileBasedSource<String> source = TextIO.read().from(largeGz.getPath()).getSource();
    List<? extends FileBasedSource<String>> splits = source.split(desiredBundleSize, options);
    // Exactly 1 split, even in AUTO mode, since it is a gzip file.
    assertThat(splits, hasSize(equalTo(1)));
    SourceTestUtils.assertSourcesEqualReferenceSource(source, splits, options);
}

Example 20 with PipelineOptions

use of org.apache.beam.sdk.options.PipelineOptions in project beam by apache.

the class ForwardingPTransformTest method validateDelegates.

@Test
public void validateDelegates() {
    @SuppressWarnings("unchecked") PipelineOptions options = Mockito.mock(PipelineOptions.class);
    Mockito.doThrow(RuntimeException.class).when(delegate).validate(options);
    thrown.expect(RuntimeException.class);
    forwarding.validate(options);
}