Examples with CheckpointMark org.apache.beam.sdk.io.UnboundedSource.CheckpointMark used on opensource projects

Example 1 with CheckpointMark

use of org.apache.beam.sdk.io.UnboundedSource.CheckpointMark in project beam by apache.

the class UnboundedEventSourceTest method resumeFromCheckpoint.

/**
 * Check aggressively checkpointing and resuming a reader gives us exactly the same event stream
 * as reading directly.
 */
@Ignore("TODO(BEAM-5070) Test is flaky. Fix before reenabling.")
@Test
public void resumeFromCheckpoint() throws IOException {
    Random random = new Random(297);
    int n = 47293;
    GeneratorConfig config = makeConfig(n);
    Generator modelGenerator = new Generator(config);
    EventIdChecker checker = new EventIdChecker();
    PipelineOptions options = TestPipeline.testingPipelineOptions();
    UnboundedEventSource source = new UnboundedEventSource(config, 1, 0, false);
    UnboundedReader<Event> reader = source.createReader(options, null);
    while (n > 0) {
        int m = Math.min(459 + random.nextInt(455), n);
        System.out.printf("reading %d...%n", m);
        checker.add(m, reader, modelGenerator);
        n -= m;
        System.out.printf("splitting with %d remaining...%n", n);
        CheckpointMark checkpointMark = reader.getCheckpointMark();
        reader = source.createReader(options, (GeneratorCheckpoint) checkpointMark);
    }
    assertFalse(reader.advance());
}