Examples with BoundedToUnboundedSourceAdapter org.apache.beam.runners.core.construction.UnboundedReadFromBoundedSource.BoundedToUnboundedSourceAdapter used on opensource projects

Example 1 with BoundedToUnboundedSourceAdapter

use of org.apache.beam.runners.core.construction.UnboundedReadFromBoundedSource.BoundedToUnboundedSourceAdapter in project beam by apache.

the class UnboundedReadFromBoundedSourceTest method testBoundedToUnboundedSourceAdapterCheckpointRestart.

private <T> void testBoundedToUnboundedSourceAdapterCheckpointRestart(BoundedSource<T> boundedSource, List<T> expectedElements) throws Exception {
    BoundedToUnboundedSourceAdapter<T> unboundedSource = new BoundedToUnboundedSourceAdapter<>(boundedSource);
    PipelineOptions options = PipelineOptionsFactory.create();
    BoundedToUnboundedSourceAdapter<T>.Reader reader = unboundedSource.createReader(options, null);
    List<T> actual = Lists.newArrayList();
    for (boolean hasNext = reader.start(); hasNext; ) {
        actual.add(reader.getCurrent());
        // checkpoint every 9 elements
        if (actual.size() % 9 == 0) {
            Checkpoint<T> checkpoint = reader.getCheckpointMark();
            Coder<Checkpoint<T>> checkpointCoder = unboundedSource.getCheckpointMarkCoder();
            Checkpoint<T> decodedCheckpoint = CoderUtils.decodeFromByteArray(checkpointCoder, CoderUtils.encodeToByteArray(checkpointCoder, checkpoint));
            reader.close();
            checkpoint.finalizeCheckpoint();
            BoundedToUnboundedSourceAdapter<T>.Reader restarted = unboundedSource.createReader(options, decodedCheckpoint);
            reader = restarted;
            hasNext = reader.start();
        } else {
            hasNext = reader.advance();
        }
    }
    Checkpoint<T> checkpointDone = reader.getCheckpointMark();
    assertTrue(checkpointDone.getResidualElements() == null || checkpointDone.getResidualElements().isEmpty());
    assertEquals(expectedElements.size(), actual.size());
    assertEquals(Sets.newHashSet(expectedElements), Sets.newHashSet(actual));
}

Example 2 with BoundedToUnboundedSourceAdapter

use of org.apache.beam.runners.core.construction.UnboundedReadFromBoundedSource.BoundedToUnboundedSourceAdapter in project beam by apache.

the class UnboundedReadFromBoundedSourceTest method testReadFromCheckpointBeforeStart.

@Test
public void testReadFromCheckpointBeforeStart() throws Exception {
    thrown.expect(NoSuchElementException.class);
    BoundedSource<Long> countingSource = CountingSource.upTo(100);
    BoundedToUnboundedSourceAdapter<Long> unboundedSource = new BoundedToUnboundedSourceAdapter<>(countingSource);
    PipelineOptions options = PipelineOptionsFactory.create();
    List<TimestampedValue<Long>> elements = ImmutableList.of(TimestampedValue.of(1L, new Instant(1L)));
    Checkpoint<Long> checkpoint = new Checkpoint<>(elements, countingSource);
    unboundedSource.createReader(options, checkpoint).getCurrent();
}

Example 3 with BoundedToUnboundedSourceAdapter

use of org.apache.beam.runners.core.construction.UnboundedReadFromBoundedSource.BoundedToUnboundedSourceAdapter in project beam by apache.

the class UnboundedReadFromBoundedSourceTest method testReadBeforeStart.

@Test
public void testReadBeforeStart() throws Exception {
    thrown.expect(NoSuchElementException.class);
    BoundedSource<Long> countingSource = CountingSource.upTo(100);
    BoundedToUnboundedSourceAdapter<Long> unboundedSource = new BoundedToUnboundedSourceAdapter<>(countingSource);
    PipelineOptions options = PipelineOptionsFactory.create();
    unboundedSource.createReader(options, null).getCurrent();
}

Example 4 with BoundedToUnboundedSourceAdapter

use of org.apache.beam.runners.core.construction.UnboundedReadFromBoundedSource.BoundedToUnboundedSourceAdapter in project beam by apache.

the class SplittableParDoTest method testConvertToPrimitiveReadsHappen.

@Test
public void testConvertToPrimitiveReadsHappen() {
    PipelineOptions deprecatedReadOptions = PipelineOptionsFactory.create();
    deprecatedReadOptions.setRunner(CrashingRunner.class);
    ExperimentalOptions.addExperiment(deprecatedReadOptions.as(ExperimentalOptions.class), "use_deprecated_read");
    Pipeline pipeline = Pipeline.create(deprecatedReadOptions);
    pipeline.apply(Read.from(new FakeBoundedSource()));
    pipeline.apply(Read.from(new BoundedToUnboundedSourceAdapter<>(new FakeBoundedSource())));
    SplittableParDo.convertReadBasedSplittableDoFnsToPrimitiveReadsIfNecessary(pipeline);
    AtomicBoolean sawPrimitiveBoundedRead = new AtomicBoolean();
    AtomicBoolean sawPrimitiveUnboundedRead = new AtomicBoolean();
    pipeline.traverseTopologically(new Defaults() {

        @Override
        public CompositeBehavior enterCompositeTransform(Node node) {
            assertThat(node.getTransform(), not(instanceOf(Read.Bounded.class)));
            assertThat(node.getTransform(), not(instanceOf(Read.Unbounded.class)));
            return super.enterCompositeTransform(node);
        }

        @Override
        public void visitPrimitiveTransform(Node node) {
            if (node.getTransform() instanceof SplittableParDo.PrimitiveBoundedRead) {
                sawPrimitiveBoundedRead.set(true);
            } else if (node.getTransform() instanceof SplittableParDo.PrimitiveUnboundedRead) {
                sawPrimitiveUnboundedRead.set(true);
            }
        }
    });
    assertTrue(sawPrimitiveBoundedRead.get());
    assertTrue(sawPrimitiveUnboundedRead.get());
}

Example 5 with BoundedToUnboundedSourceAdapter

use of org.apache.beam.runners.core.construction.UnboundedReadFromBoundedSource.BoundedToUnboundedSourceAdapter in project beam by apache.

the class SplittableParDoTest method testConvertIsSkippedWhenUsingDeprecatedRead.

@Test
public void testConvertIsSkippedWhenUsingDeprecatedRead() {
    Pipeline sdfRead = Pipeline.create();
    sdfRead.apply(Read.from(new FakeBoundedSource()));
    sdfRead.apply(Read.from(new BoundedToUnboundedSourceAdapter<>(new FakeBoundedSource())));
    SplittableParDo.convertReadBasedSplittableDoFnsToPrimitiveReadsIfNecessary(sdfRead);
    pipeline.traverseTopologically(new Defaults() {

        @Override
        public void visitPrimitiveTransform(Node node) {
            assertThat(node.getTransform(), not(instanceOf(SplittableParDo.PrimitiveBoundedRead.class)));
            assertThat(node.getTransform(), not(instanceOf(SplittableParDo.PrimitiveUnboundedRead.class)));
        }
    });
}