Examples with RemoteOutputReceiver org.apache.beam.runners.fnexecution.control.RemoteOutputReceiver used on opensource projects

Example 1 with RemoteOutputReceiver

use of org.apache.beam.runners.fnexecution.control.RemoteOutputReceiver in project beam by apache.

the class ProcessBundleBenchmark method testState.

private static void testState(StatefulTransform statefulTransform, StateRequestHandler stateRequestHandler) throws Exception {
    Map<String, ? super Coder<WindowedValue<?>>> remoteOutputCoders = statefulTransform.descriptor.getRemoteOutputCoders();
    Map<String, RemoteOutputReceiver<?>> outputReceivers = new HashMap<>();
    AtomicInteger outputValuesCount = new AtomicInteger();
    for (Entry<String, ? super Coder<WindowedValue<?>>> remoteOutputCoder : remoteOutputCoders.entrySet()) {
        outputReceivers.put(remoteOutputCoder.getKey(), RemoteOutputReceiver.of((Coder) remoteOutputCoder.getValue(), (FnDataReceiver<? super WindowedValue<?>>) (WindowedValue<?> value) -> outputValuesCount.incrementAndGet()));
    }
    String key = Strings.padStart(Long.toHexString(Thread.currentThread().getId()), 16, '0');
    try (RemoteBundle bundle = statefulTransform.processor.newBundle(outputReceivers, stateRequestHandler, BundleProgressHandler.ignored())) {
        Iterables.getOnlyElement(bundle.getInputReceivers().values()).accept(valueInGlobalWindow(KV.of(key, "zero")));
    }
    try (RemoteBundle bundle = statefulTransform.processor.newBundle(outputReceivers, stateRequestHandler, BundleProgressHandler.ignored())) {
        Iterables.getOnlyElement(bundle.getInputReceivers().values()).accept(valueInGlobalWindow(KV.of(key, "one")));
    }
    try (RemoteBundle bundle = statefulTransform.processor.newBundle(outputReceivers, stateRequestHandler, BundleProgressHandler.ignored())) {
        Iterables.getOnlyElement(bundle.getInputReceivers().values()).accept(valueInGlobalWindow(KV.of(key, "two")));
    }
    try (RemoteBundle bundle = statefulTransform.processor.newBundle(outputReceivers, stateRequestHandler, BundleProgressHandler.ignored())) {
        Iterables.getOnlyElement(bundle.getInputReceivers().values()).accept(valueInGlobalWindow(KV.of(key, "flush")));
    }
    assertEquals(3, outputValuesCount.getAndSet(0));
}

Example 2 with RemoteOutputReceiver

use of org.apache.beam.runners.fnexecution.control.RemoteOutputReceiver in project beam by apache.

the class ProcessBundleBenchmark method testTinyBundle.

@Benchmark
// Use several threads since we expect contention during bundle processing.
@Threads(16)
public void testTinyBundle(TrivialTransform trivialTransform) throws Exception {
    Map<String, ? super Coder<WindowedValue<?>>> remoteOutputCoders = trivialTransform.descriptor.getRemoteOutputCoders();
    Map<String, RemoteOutputReceiver<?>> outputReceivers = new HashMap<>();
    AtomicInteger outputValuesCount = new AtomicInteger();
    for (Entry<String, ? super Coder<WindowedValue<?>>> remoteOutputCoder : remoteOutputCoders.entrySet()) {
        outputReceivers.put(remoteOutputCoder.getKey(), RemoteOutputReceiver.of((Coder) remoteOutputCoder.getValue(), (FnDataReceiver<? super WindowedValue<?>>) (WindowedValue<?> value) -> outputValuesCount.incrementAndGet()));
    }
    try (RemoteBundle bundle = trivialTransform.processor.newBundle(outputReceivers, BundleProgressHandler.ignored())) {
        Iterables.getOnlyElement(bundle.getInputReceivers().values()).accept(valueInGlobalWindow(new byte[0]));
    }
    assertEquals(3, outputValuesCount.getAndSet(0));
}

Example 3 with RemoteOutputReceiver

use of org.apache.beam.runners.fnexecution.control.RemoteOutputReceiver in project beam by apache.

the class ProcessBundleBenchmark method testLargeBundle.

@Benchmark
// Use several threads since we expect contention during bundle processing.
@Threads(16)
public void testLargeBundle(TrivialTransform trivialTransform) throws Exception {
    Map<String, ? super Coder<WindowedValue<?>>> remoteOutputCoders = trivialTransform.descriptor.getRemoteOutputCoders();
    Map<String, RemoteOutputReceiver<?>> outputReceivers = new HashMap<>();
    AtomicInteger outputValuesCount = new AtomicInteger();
    for (Entry<String, ? super Coder<WindowedValue<?>>> remoteOutputCoder : remoteOutputCoders.entrySet()) {
        outputReceivers.put(remoteOutputCoder.getKey(), RemoteOutputReceiver.of((Coder) remoteOutputCoder.getValue(), (FnDataReceiver<? super WindowedValue<?>>) (WindowedValue<?> value) -> outputValuesCount.incrementAndGet()));
    }
    try (RemoteBundle bundle = trivialTransform.processor.newBundle(outputReceivers, BundleProgressHandler.ignored())) {
        for (int i = 0; i < 1_000; i++) {
            Iterables.getOnlyElement(bundle.getInputReceivers().values()).accept(valueInGlobalWindow(new byte[0]));
        }
    }
    assertEquals(3_000, outputValuesCount.getAndSet(0));
}