Examples with BiConsumer java.util.function.BiConsumer used on opensource projects

Example 76 with BiConsumer

use of java.util.function.BiConsumer in project beam by apache.

the class ExecutableStageDoFnOperatorTest method testWatermarkHandling.

@Test
public void testWatermarkHandling() throws Exception {
    TupleTag<Integer> mainOutput = new TupleTag<>("main-output");
    DoFnOperator.MultiOutputOutputManagerFactory<Integer> outputManagerFactory = new DoFnOperator.MultiOutputOutputManagerFactory(mainOutput, VoidCoder.of(), new SerializablePipelineOptions(FlinkPipelineOptions.defaults()));
    ExecutableStageDoFnOperator<KV<String, Integer>, Integer> operator = getOperator(mainOutput, Collections.emptyList(), outputManagerFactory, WindowingStrategy.of(FixedWindows.of(Duration.millis(10))), StringUtf8Coder.of(), WindowedValue.getFullCoder(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of()), IntervalWindow.getCoder()));
    KeyedOneInputStreamOperatorTestHarness<String, WindowedValue<KV<String, Integer>>, WindowedValue<Integer>> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, val -> val.getValue().getKey(), new CoderTypeInformation<>(StringUtf8Coder.of(), FlinkPipelineOptions.defaults()));
    RemoteBundle bundle = Mockito.mock(RemoteBundle.class);
    when(bundle.getInputReceivers()).thenReturn(ImmutableMap.<String, FnDataReceiver<WindowedValue>>builder().put("input", Mockito.mock(FnDataReceiver.class)).build());
    when(bundle.getTimerReceivers()).thenReturn(ImmutableMap.<KV<String, String>, FnDataReceiver<WindowedValue>>builder().put(KV.of("transform", "timer"), Mockito.mock(FnDataReceiver.class)).put(KV.of("transform", "timer2"), Mockito.mock(FnDataReceiver.class)).put(KV.of("transform", "timer3"), Mockito.mock(FnDataReceiver.class)).build());
    when(stageBundleFactory.getBundle(any(), any(), any(), any(), any(), any())).thenReturn(bundle);
    testHarness.open();
    assertThat(operator.getCurrentOutputWatermark(), is(BoundedWindow.TIMESTAMP_MIN_VALUE.getMillis()));
    // No bundle has been started, watermark can be freely advanced
    testHarness.processWatermark(0);
    assertThat(operator.getCurrentOutputWatermark(), is(0L));
    // Trigger a new bundle
    IntervalWindow intervalWindow = new IntervalWindow(new Instant(0), new Instant(9));
    WindowedValue<KV<String, Integer>> windowedValue = WindowedValue.of(KV.of("one", 1), Instant.now(), intervalWindow, PaneInfo.NO_FIRING);
    testHarness.processElement(new StreamRecord<>(windowedValue));
    // The output watermark should be held back during the bundle
    testHarness.processWatermark(1);
    assertThat(operator.getEffectiveInputWatermark(), is(1L));
    assertThat(operator.getCurrentOutputWatermark(), is(0L));
    // After the bundle has been finished, the watermark should be advanced
    operator.invokeFinishBundle();
    assertThat(operator.getCurrentOutputWatermark(), is(1L));
    // Bundle finished, watermark can be freely advanced
    testHarness.processWatermark(2);
    assertThat(operator.getEffectiveInputWatermark(), is(2L));
    assertThat(operator.getCurrentOutputWatermark(), is(2L));
    // Trigger a new bundle
    testHarness.processElement(new StreamRecord<>(windowedValue));
    // cleanup timer
    assertThat(testHarness.numEventTimeTimers(), is(1));
    // Set at timer
    Instant timerTarget = new Instant(5);
    Instant timerTarget2 = new Instant(6);
    operator.getLockToAcquireForStateAccessDuringBundles().lock();
    BiConsumer<String, Instant> timerConsumer = (timerId, timestamp) -> operator.setTimer(Timer.of(windowedValue.getValue().getKey(), "", windowedValue.getWindows(), timestamp, timestamp, PaneInfo.NO_FIRING), TimerInternals.TimerData.of("", TimerReceiverFactory.encodeToTimerDataTimerId("transform", timerId), StateNamespaces.window(IntervalWindow.getCoder(), intervalWindow), timestamp, timestamp, TimeDomain.EVENT_TIME));
    timerConsumer.accept("timer", timerTarget);
    timerConsumer.accept("timer2", timerTarget2);
    assertThat(testHarness.numEventTimeTimers(), is(3));
    // Advance input watermark past the timer
    // Check the output watermark is held back
    long targetWatermark = timerTarget.getMillis() + 100;
    testHarness.processWatermark(targetWatermark);
    // Do not yet advance the output watermark because we are still processing a bundle
    assertThat(testHarness.numEventTimeTimers(), is(3));
    assertThat(operator.getCurrentOutputWatermark(), is(2L));
    // Check that the timers are fired but the output watermark is advanced no further than
    // the minimum timer timestamp of the previous bundle because we are still processing a
    // bundle which might contain more timers.
    // Timers can create loops if they keep rescheduling themselves when firing
    // Thus, we advance the watermark asynchronously to allow for checkpointing to run
    operator.invokeFinishBundle();
    assertThat(testHarness.numEventTimeTimers(), is(3));
    testHarness.setProcessingTime(testHarness.getProcessingTime() + 1);
    assertThat(testHarness.numEventTimeTimers(), is(0));
    assertThat(operator.getCurrentOutputWatermark(), is(5L));
    // Output watermark is advanced synchronously when the bundle finishes,
    // no more timers are scheduled
    operator.invokeFinishBundle();
    assertThat(operator.getCurrentOutputWatermark(), is(targetWatermark));
    assertThat(testHarness.numEventTimeTimers(), is(0));
    // Watermark is advanced in a blocking fashion on close, not via a timers
    // Create a bundle with a pending timer to simulate that
    testHarness.processElement(new StreamRecord<>(windowedValue));
    timerConsumer.accept("timer3", new Instant(targetWatermark));
    assertThat(testHarness.numEventTimeTimers(), is(1));
    // This should be blocking until the watermark reaches Long.MAX_VALUE.
    testHarness.close();
    assertThat(testHarness.numEventTimeTimers(), is(0));
    assertThat(operator.getCurrentOutputWatermark(), is(Long.MAX_VALUE));
}

Example 77 with BiConsumer

use of java.util.function.BiConsumer in project hazelcast by hazelcast.

the class DistributedObjectCounterCollector method forEachMetric.

@Override
public void forEachMetric(Node node, BiConsumer<PhoneHomeMetrics, String> metricsConsumer) {
    InternalProxyService proxyService = node.nodeEngine.getProxyService();
    Map<String, Long> objectsPerService = proxyService.getAllDistributedObjects().stream().filter(obj -> INTERNAL_OBJECTS_PREFIXES.stream().noneMatch(prefix -> obj.getName().startsWith(prefix))).filter(obj -> SERVICE_NAME_TO_METRIC_NAME.containsKey(obj.getServiceName())).collect(groupingBy(DistributedObject::getServiceName, Collectors.counting()));
    SERVICE_NAME_TO_METRIC_NAME.forEach((serviceName, metricNames) -> {
        metricsConsumer.accept(metricNames[0], String.valueOf(objectsPerService.getOrDefault(serviceName, 0L)));
        metricsConsumer.accept(metricNames[1], String.valueOf(proxyService.getCreatedCount(serviceName)));
    });
}

Example 78 with BiConsumer

use of java.util.function.BiConsumer in project hazelcast by hazelcast.

the class MasterContext method invokeOnParticipant.

private void invokeOnParticipant(MemberInfo memberInfo, Supplier<Operation> operationSupplier, @Nullable Consumer<Collection<Map.Entry<MemberInfo, Object>>> completionCallback, @Nullable BiConsumer<Address, Object> individualCallback, boolean retryOnTimeoutException, ConcurrentMap<MemberInfo, Object> collectedResponses, AtomicInteger remainingCount) {
    Operation operation = operationSupplier.get();
    InternalCompletableFuture<Object> future = nodeEngine.getOperationService().createInvocationBuilder(JetServiceBackend.SERVICE_NAME, operation, memberInfo.getAddress()).invoke();
    future.whenCompleteAsync(withTryCatch(logger, (r, throwable) -> {
        Object response = r != null ? r : throwable != null ? peel(throwable) : NULL_OBJECT;
        if (retryOnTimeoutException && throwable instanceof OperationTimeoutException) {
            logger.warning("Retrying " + operation.getClass().getName() + " that failed with " + OperationTimeoutException.class.getSimpleName() + " in " + jobIdString());
            invokeOnParticipant(memberInfo, operationSupplier, completionCallback, individualCallback, retryOnTimeoutException, collectedResponses, remainingCount);
            return;
        }
        if (individualCallback != null) {
            individualCallback.accept(memberInfo.getAddress(), throwable != null ? peel(throwable) : r);
        }
        Object oldResponse = collectedResponses.put(memberInfo, response);
        assert oldResponse == null : "Duplicate response for " + memberInfo.getAddress() + ". Old=" + oldResponse + ", new=" + response;
        if (remainingCount.decrementAndGet() == 0 && completionCallback != null) {
            completionCallback.accept(collectedResponses.entrySet().stream().map(e -> e.getValue() == NULL_OBJECT ? entry(e.getKey(), null) : e).collect(Collectors.toList()));
        }
    }));
}

Example 79 with BiConsumer

use of java.util.function.BiConsumer in project hazelcast by hazelcast.

the class MapProxySupport method putAllInternal.

/**
 * This method will group all entries per partition and send one operation
 * per member. If there are e.g. five keys for a single member, even if
 * they are from different partitions, there will only be a single remote
 * invocation instead of five.
 * <p>
 * There is also an optional support for batching to send smaller packages.
 * Takes care about {@code null} checks for keys and values.
 *
 * @param future iff not-null, execute asynchronously by completing this future.
 *               Batching is not supported in async mode
 */
@SuppressWarnings({ "checkstyle:MethodLength", "checkstyle:CyclomaticComplexity", "checkstyle:NPathComplexity" })
protected void putAllInternal(Map<? extends K, ? extends V> map, @Nullable InternalCompletableFuture<Void> future, boolean triggerMapLoader) {
    try {
        int mapSize = map.size();
        if (mapSize == 0) {
            if (future != null) {
                future.complete(null);
            }
            return;
        }
        boolean useBatching = future == null && isPutAllUseBatching(mapSize);
        int partitionCount = partitionService.getPartitionCount();
        int initialSize = getPutAllInitialSize(useBatching, mapSize, partitionCount);
        Map<Address, List<Integer>> memberPartitionsMap = partitionService.getMemberPartitionsMap();
        // init counters for batching
        MutableLong[] counterPerMember = null;
        Address[] addresses = null;
        if (useBatching) {
            counterPerMember = new MutableLong[partitionCount];
            addresses = new Address[partitionCount];
            for (Entry<Address, List<Integer>> addressListEntry : memberPartitionsMap.entrySet()) {
                MutableLong counter = new MutableLong();
                Address address = addressListEntry.getKey();
                for (int partitionId : addressListEntry.getValue()) {
                    counterPerMember[partitionId] = counter;
                    addresses[partitionId] = address;
                }
            }
        }
        // fill entriesPerPartition
        MapEntries[] entriesPerPartition = new MapEntries[partitionCount];
        for (Entry entry : map.entrySet()) {
            checkNotNull(entry.getKey(), NULL_KEY_IS_NOT_ALLOWED);
            checkNotNull(entry.getValue(), NULL_VALUE_IS_NOT_ALLOWED);
            Data keyData = toDataWithStrategy(entry.getKey());
            int partitionId = partitionService.getPartitionId(keyData);
            MapEntries entries = entriesPerPartition[partitionId];
            if (entries == null) {
                entries = new MapEntries(initialSize);
                entriesPerPartition[partitionId] = entries;
            }
            entries.add(keyData, toData(entry.getValue()));
            if (useBatching) {
                long currentSize = ++counterPerMember[partitionId].value;
                if (currentSize % putAllBatchSize == 0) {
                    List<Integer> partitions = memberPartitionsMap.get(addresses[partitionId]);
                    invokePutAllOperation(addresses[partitionId], partitions, entriesPerPartition, triggerMapLoader).get();
                }
            }
        }
        // invoke operations for entriesPerPartition
        AtomicInteger counter = new AtomicInteger(memberPartitionsMap.size());
        InternalCompletableFuture<Void> resultFuture = future != null ? future : new InternalCompletableFuture<>();
        BiConsumer<Void, Throwable> callback = (response, t) -> {
            if (t != null) {
                resultFuture.completeExceptionally(t);
            }
            if (counter.decrementAndGet() == 0) {
                try {
                    // don't ignore errors here, see https://github.com/hazelcast/hazelcast-jet/issues/3046
                    finalizePutAll(map);
                } catch (Throwable e) {
                    resultFuture.completeExceptionally(e);
                    return;
                }
                if (!resultFuture.isDone()) {
                    resultFuture.complete(null);
                }
            }
        };
        for (Entry<Address, List<Integer>> entry : memberPartitionsMap.entrySet()) {
            invokePutAllOperation(entry.getKey(), entry.getValue(), entriesPerPartition, triggerMapLoader).whenCompleteAsync(callback);
        }
        // if executing in sync mode, block for the responses
        if (future == null) {
            resultFuture.get();
        }
    } catch (Throwable e) {
        throw rethrow(e);
    }
}

Example 80 with BiConsumer

use of java.util.function.BiConsumer in project hazelcast by hazelcast.

the class MultiMapProxySupport method putAllInternal.

// NB: this method is generally copied from MapProxySupport#putAllInternal
@SuppressWarnings({ "checkstyle:npathcomplexity", "checkstyle:methodlength" })
protected void putAllInternal(Map<Data, Data> map, @Nullable InternalCompletableFuture<Void> future) {
    // get partition to entries mapping
    try {
        int mapSize = map.size();
        if (mapSize == 0) {
            if (future != null) {
                future.complete(null);
            }
            return;
        }
        int partitionCount = partitionService.getPartitionCount();
        int initialSize = getPutAllInitialSize(mapSize, partitionCount);
        // get node to partition mapping
        Map<Address, List<Integer>> memberPartitionsMap = partitionService.getMemberPartitionsMap();
        // fill entriesPerPartition
        MapEntries[] entriesPerPartition = new MapEntries[partitionCount];
        for (Map.Entry<Data, Data> entry : map.entrySet()) {
            checkNotNull(entry.getKey(), NULL_KEY_IS_NOT_ALLOWED);
            checkNotNull(entry.getValue(), NULL_VALUE_IS_NOT_ALLOWED);
            Data keyData = entry.getKey();
            int partitionId = partitionService.getPartitionId(keyData);
            MapEntries entries = entriesPerPartition[partitionId];
            if (entries == null) {
                entries = new MapEntries(initialSize);
                entriesPerPartition[partitionId] = entries;
            }
            entries.add(keyData, entry.getValue());
        }
        // invoke operations for entriesPerPartition
        AtomicInteger counter = new AtomicInteger(memberPartitionsMap.size());
        InternalCompletableFuture<Void> resultFuture = future != null ? future : new InternalCompletableFuture<>();
        BiConsumer<Void, Throwable> callback = (response, t) -> {
            if (t != null) {
                resultFuture.completeExceptionally(t);
            }
            if (counter.decrementAndGet() == 0) {
                if (!resultFuture.isDone()) {
                    resultFuture.complete(null);
                }
            }
        };
        for (Map.Entry<Address, List<Integer>> entry : memberPartitionsMap.entrySet()) {
            invokePutAllOperation(entry.getKey(), entry.getValue(), entriesPerPartition).whenCompleteAsync(callback);
        }
        // if executing in sync mode, block for the responses
        if (future == null) {
            resultFuture.get();
        }
    } catch (Throwable e) {
        throw rethrow(e);
    }
}