Examples with NeverDeduplicator org.apache.beam.runners.direct.UnboundedReadDeduplicator.NeverDeduplicator used on opensource projects

Example 1 with NeverDeduplicator

use of org.apache.beam.runners.direct.UnboundedReadDeduplicator.NeverDeduplicator in project beam by apache.

the class UnboundedReadEvaluatorFactoryTest method processElement.

private void processElement(final TestUnboundedSource<String> source) throws Exception {
    final EvaluationContext context = EvaluationContext.create(MockClock.fromInstant(Instant.now()), CloningBundleFactory.create(), DirectGraph.create(emptyMap(), emptyMap(), LinkedListMultimap.create(), emptySet(), emptyMap()), emptySet(), Executors.newCachedThreadPool());
    final UnboundedReadEvaluatorFactory factory = new UnboundedReadEvaluatorFactory(context, p.getOptions());
    final SplittableParDo.PrimitiveUnboundedRead<String> unbounded = new SplittableParDo.PrimitiveUnboundedRead(Read.from(source));
    final Pipeline pipeline = Pipeline.create(p.getOptions());
    final PCollection<String> pCollection = pipeline.apply(unbounded);
    final AppliedPTransform<PBegin, PCollection<String>, SplittableParDo.PrimitiveUnboundedRead<String>> application = AppliedPTransform.of("test", new HashMap<>(), singletonMap(new TupleTag(), pCollection), unbounded, ResourceHints.create(), pipeline);
    final TransformEvaluator<UnboundedSourceShard<String, TestCheckpointMark>> evaluator = factory.forApplication(application, null);
    final UnboundedSource.UnboundedReader<String> reader = source.createReader(p.getOptions(), null);
    final UnboundedSourceShard<String, TestCheckpointMark> shard = UnboundedSourceShard.of(source, new NeverDeduplicator(), reader, null);
    final WindowedValue<UnboundedSourceShard<String, TestCheckpointMark>> value = WindowedValue.of(shard, BoundedWindow.TIMESTAMP_MAX_VALUE, GlobalWindow.INSTANCE, PaneInfo.NO_FIRING);
    TestUnboundedSource.readerClosedCount = 0;
    evaluator.processElement(value);
}