Examples with ReadyCheckingSideInputReader org.apache.beam.runners.core.ReadyCheckingSideInputReader used on opensource projects

Example 1 with ReadyCheckingSideInputReader

use of org.apache.beam.runners.core.ReadyCheckingSideInputReader in project beam by apache.

the class ParDoEvaluator method create.

public static <InputT, OutputT> ParDoEvaluator<InputT> create(EvaluationContext evaluationContext, DirectStepContext stepContext, AppliedPTransform<?, ?, ?> application, WindowingStrategy<?, ? extends BoundedWindow> windowingStrategy, DoFn<InputT, OutputT> fn, StructuralKey<?> key, List<PCollectionView<?>> sideInputs, TupleTag<OutputT> mainOutputTag, List<TupleTag<?>> additionalOutputTags, Map<TupleTag<?>, PCollection<?>> outputs, DoFnRunnerFactory<InputT, OutputT> runnerFactory) {
    BundleOutputManager outputManager = createOutputManager(evaluationContext, key, outputs);
    ReadyCheckingSideInputReader sideInputReader = evaluationContext.createSideInputReader(sideInputs);
    PushbackSideInputDoFnRunner<InputT, OutputT> runner = runnerFactory.createRunner(evaluationContext.getPipelineOptions(), fn, sideInputs, sideInputReader, outputManager, mainOutputTag, additionalOutputTags, stepContext, windowingStrategy);
    return create(runner, stepContext, application, outputManager);
}

Example 2 with ReadyCheckingSideInputReader

use of org.apache.beam.runners.core.ReadyCheckingSideInputReader in project beam by apache.

the class SideInputContainerTest method isReadyForSomeNotReadyViewsFalseUntilElements.

/**
 * Demonstrates that calling isReady returns false until elements are written to the {@link
 * PCollectionView}, {@link BoundedWindow} pair, at which point it returns true.
 */
@Test
public void isReadyForSomeNotReadyViewsFalseUntilElements() {
    ImmutableList.Builder<WindowedValue<?>> mapValuesBuilder = ImmutableList.builder();
    for (Object materializedValue : materializeValuesFor(mapView.getPipeline().getOptions(), View.asMap(), KV.of("one", 1))) {
        mapValuesBuilder.add(WindowedValue.of(materializedValue, SECOND_WINDOW.maxTimestamp().minus(Duration.millis(100L)), SECOND_WINDOW, PaneInfo.ON_TIME_AND_ONLY_FIRING));
    }
    container.write(mapView, mapValuesBuilder.build());
    ReadyCheckingSideInputReader reader = container.createReaderForViews(ImmutableList.of(mapView, singletonView));
    assertThat(reader.isReady(mapView, FIRST_WINDOW), is(false));
    assertThat(reader.isReady(mapView, SECOND_WINDOW), is(true));
    assertThat(reader.isReady(singletonView, SECOND_WINDOW), is(false));
    ImmutableList.Builder<WindowedValue<?>> newMapValuesBuilder = ImmutableList.builder();
    for (Object materializedValue : materializeValuesFor(mapView.getPipeline().getOptions(), View.asMap(), KV.of("too", 2))) {
        newMapValuesBuilder.add(WindowedValue.of(materializedValue, FIRST_WINDOW.maxTimestamp().minus(Duration.millis(100L)), FIRST_WINDOW, PaneInfo.ON_TIME_AND_ONLY_FIRING));
    }
    container.write(mapView, newMapValuesBuilder.build());
    // Cached value is false
    assertThat(reader.isReady(mapView, FIRST_WINDOW), is(false));
    ImmutableList.Builder<WindowedValue<?>> singletonValuesBuilder = ImmutableList.builder();
    for (Object materializedValue : materializeValuesFor(singletonView.getPipeline().getOptions(), View.asSingleton(), 1.25)) {
        singletonValuesBuilder.add(WindowedValue.of(materializedValue, SECOND_WINDOW.maxTimestamp().minus(Duration.millis(100L)), SECOND_WINDOW, PaneInfo.ON_TIME_AND_ONLY_FIRING));
    }
    container.write(singletonView, singletonValuesBuilder.build());
    assertThat(reader.isReady(mapView, SECOND_WINDOW), is(true));
    assertThat(reader.isReady(singletonView, SECOND_WINDOW), is(false));
    assertThat(reader.isReady(mapView, GlobalWindow.INSTANCE), is(false));
    assertThat(reader.isReady(singletonView, GlobalWindow.INSTANCE), is(false));
    reader = container.createReaderForViews(ImmutableList.of(mapView, singletonView));
    assertThat(reader.isReady(mapView, SECOND_WINDOW), is(true));
    assertThat(reader.isReady(singletonView, SECOND_WINDOW), is(true));
    assertThat(reader.isReady(mapView, FIRST_WINDOW), is(true));
}

Example 3 with ReadyCheckingSideInputReader

use of org.apache.beam.runners.core.ReadyCheckingSideInputReader in project beam by apache.

the class SideInputContainerTest method isReadyInEmptyReaderThrows.

/**
 * Demonstrates that calling isReady on an empty container throws an {@link
 * IllegalArgumentException}.
 */
@Test
public void isReadyInEmptyReaderThrows() {
    ReadyCheckingSideInputReader reader = container.createReaderForViews(ImmutableList.of());
    thrown.expect(IllegalArgumentException.class);
    thrown.expectMessage("does not contain");
    thrown.expectMessage(ImmutableList.of().toString());
    reader.isReady(mapView, GlobalWindow.INSTANCE);
}

Example 4 with ReadyCheckingSideInputReader

use of org.apache.beam.runners.core.ReadyCheckingSideInputReader in project beam by apache.

the class SideInputContainerTest method isReadyForEmptyWindowTrue.

@Test
public void isReadyForEmptyWindowTrue() throws Exception {
    CountDownLatch onComplete = new CountDownLatch(1);
    immediatelyInvokeCallback(mapView, GlobalWindow.INSTANCE);
    CountDownLatch latch = invokeLatchedCallback(singletonView, GlobalWindow.INSTANCE, onComplete);
    ReadyCheckingSideInputReader reader = container.createReaderForViews(ImmutableList.of(mapView, singletonView));
    assertThat(reader.isReady(mapView, GlobalWindow.INSTANCE), is(true));
    assertThat(reader.isReady(singletonView, GlobalWindow.INSTANCE), is(false));
    latch.countDown();
    if (!onComplete.await(1500L, TimeUnit.MILLISECONDS)) {
        fail("Callback to set empty values did not complete!");
    }
    // The cached value was false, so it continues to be true
    assertThat(reader.isReady(singletonView, GlobalWindow.INSTANCE), is(false));
    // A new reader for the same container gets a fresh look
    reader = container.createReaderForViews(ImmutableList.of(mapView, singletonView));
    assertThat(reader.isReady(singletonView, GlobalWindow.INSTANCE), is(true));
}

Example 5 with ReadyCheckingSideInputReader

use of org.apache.beam.runners.core.ReadyCheckingSideInputReader in project beam by apache.

the class ParDoEvaluator method create.

public static <InputT, OutputT> ParDoEvaluator<InputT> create(EvaluationContext evaluationContext, PipelineOptions options, DirectStepContext stepContext, AppliedPTransform<?, ?, ?> application, Coder<InputT> inputCoder, WindowingStrategy<?, ? extends BoundedWindow> windowingStrategy, DoFn<InputT, OutputT> fn, StructuralKey<?> key, List<PCollectionView<?>> sideInputs, TupleTag<OutputT> mainOutputTag, List<TupleTag<?>> additionalOutputTags, Map<TupleTag<?>, PCollection<?>> outputs, DoFnSchemaInformation doFnSchemaInformation, Map<String, PCollectionView<?>> sideInputMapping, DoFnRunnerFactory<InputT, OutputT> runnerFactory) {
    BundleOutputManager outputManager = createOutputManager(evaluationContext, key, outputs);
    ReadyCheckingSideInputReader sideInputReader = evaluationContext.createSideInputReader(sideInputs);
    Map<TupleTag<?>, Coder<?>> outputCoders = outputs.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue().getCoder()));
    PushbackSideInputDoFnRunner<InputT, OutputT> runner = runnerFactory.createRunner(options, fn, sideInputs, sideInputReader, outputManager, mainOutputTag, additionalOutputTags, stepContext, inputCoder, outputCoders, windowingStrategy, doFnSchemaInformation, sideInputMapping);
    return create(runner, stepContext, application, outputManager);
}