Examples with Watermark com.hazelcast.jet.core.Watermark used on opensource projects

Example 16 with Watermark

use of com.hazelcast.jet.core.Watermark in project hazelcast by hazelcast.

the class AsyncTransformUsingServiceOrderedP method tryFlushQueue.

/**
 * Drains items from the queue until either:
 * <ul><li>
 *     encountering a non-completed item
 * </li><li>
 *     the outbox gets full
 * </li></ul>
 *
 * @return true if there are no more in-flight items and everything was emitted
 *         to the outbox
 */
boolean tryFlushQueue() {
    // queue. It also doesn't shuffle the stream items.
    for (; ; ) {
        if (!emitFromTraverser(currentTraverser)) {
            return false;
        }
        Object o = queue.peek();
        if (o == null) {
            return true;
        }
        if (o instanceof Watermark) {
            watermarkTraverser.accept((Watermark) o);
            currentTraverser = watermarkTraverser;
            queuedWmCount--;
        } else {
            @SuppressWarnings("unchecked") Tuple2<T, CompletableFuture<IR>> cast = (Tuple2<T, CompletableFuture<IR>>) o;
            T item = cast.f0();
            CompletableFuture<IR> future = cast.f1();
            assert future != null;
            if (!future.isDone()) {
                return false;
            }
            try {
                currentTraverser = mapResultFn.apply(item, future.get());
                if (currentTraverser == null) {
                    currentTraverser = Traversers.empty();
                }
            } catch (Throwable e) {
                throw new JetException("Async operation completed exceptionally: " + e, e);
            }
        }
        queue.remove();
    }
}

Example 17 with Watermark

use of com.hazelcast.jet.core.Watermark in project hazelcast by hazelcast.

the class Util method logLateEvent.

/**
 * Logs a late event that was dropped.
 */
public static void logLateEvent(ILogger logger, long currentWm, @Nonnull Object item) {
    if (!logger.isInfoEnabled()) {
        return;
    }
    if (item instanceof JetEvent) {
        JetEvent event = (JetEvent) item;
        logger.info(format("Event dropped, late by %d ms. currentWatermark=%s, eventTime=%s, event=%s", currentWm - event.timestamp(), toLocalTime(currentWm), toLocalTime(event.timestamp()), event.payload()));
    } else {
        logger.info(format("Late event dropped. currentWatermark=%s, event=%s", new Watermark(currentWm), item));
    }
}

Example 18 with Watermark

use of com.hazelcast.jet.core.Watermark in project hazelcast by hazelcast.

the class TestSupport method runTest.

private void runTest(TestMode testMode) throws Exception {
    beforeEachRun.run();
    assert testMode.isSnapshotsEnabled() || testMode.snapshotRestoreInterval() == 0 : "Illegal combination: don't do snapshots, but do restore";
    boolean doSnapshots = testMode.doSnapshots;
    int doRestoreEvery = testMode.restoreInterval;
    IdleStrategy idler = new BackoffIdleStrategy(0, 0, MICROSECONDS.toNanos(1), MILLISECONDS.toNanos(1));
    int idleCount = 0;
    System.out.println("### Running the test, mode=" + testMode.toString());
    TestInbox inbox = new TestInbox();
    int inboxOrdinal = -1;
    Processor[] processor = { newProcessorFromSupplier() };
    boolean isCooperative = processor[0].isCooperative();
    // we'll use 1-capacity outbox to test outbox rejection
    TestOutbox[] outbox = { createOutbox() };
    List<List<Object>> actualOutputs = new ArrayList<>(outputOrdinalCount);
    for (int i = 0; i < outputOrdinalCount; i++) {
        actualOutputs.add(new ArrayList<>());
    }
    // create instance of your processor and call the init() method
    initProcessor(processor[0], outbox[0]);
    int[] restoreCount = { 0 };
    // do snapshot+restore before processing any item. This will test saveToSnapshot() in this edge case
    snapshotAndRestore(processor, outbox, actualOutputs, doSnapshots, doRestoreEvery, restoreCount);
    // call the process() method
    List<ObjectWithOrdinal> input = mixInputs(inputs, priorities);
    int inputPosition = 0;
    while (inputPosition < input.size() || !inbox.isEmpty()) {
        if (inbox.isEmpty() && inputPosition < input.size()) {
            inboxOrdinal = input.get(inputPosition).ordinal;
            for (int added = 0; inputPosition < input.size() && added < testMode.inboxLimit() && inboxOrdinal == input.get(inputPosition).ordinal && (added == 0 || !(input.get(inputPosition).item instanceof Watermark)); added++) {
                ObjectWithOrdinal objectWithOrdinal = input.get(inputPosition++);
                inbox.queue().add(objectWithOrdinal.item);
                inboxOrdinal = objectWithOrdinal.ordinal;
            }
            if (logInputOutput) {
                System.out.println(LocalTime.now() + " Input-" + inboxOrdinal + ": " + inbox);
            }
        }
        int lastInboxSize = inbox.size();
        String methodName;
        methodName = processInbox(inbox, inboxOrdinal, isCooperative, processor);
        boolean madeProgress = inbox.size() < lastInboxSize || (outbox[0].bucketCount() > 0 && !outbox[0].queue(0).isEmpty());
        assertTrue(methodName + "() call without progress", !assertProgress || madeProgress);
        idleCount = idle(idler, idleCount, madeProgress);
        if (outbox[0].bucketCount() > 0 && outbox[0].queue(0).size() == 1 && !inbox.isEmpty()) {
            // if the outbox is full, call the process() method again. Cooperative
            // processor must be able to cope with this situation and not try to put
            // more items to the outbox.
            outbox[0].reset();
            processInbox(inbox, inboxOrdinal, isCooperative, processor);
        }
        outbox[0].drainQueuesAndReset(actualOutputs, logInputOutput);
        if (inbox.isEmpty()) {
            snapshotAndRestore(processor, outbox, actualOutputs, doSnapshots, doRestoreEvery, restoreCount);
        }
    }
    if (logInputOutput && !inputs.isEmpty()) {
        System.out.println(LocalTime.now() + " Input processed, calling complete()");
    }
    // call the complete() method
    if (callComplete) {
        long completeStart = System.nanoTime();
        long outputMatchedAt = Long.MAX_VALUE;
        boolean[] done = { false };
        do {
            doCall("complete", isCooperative, () -> done[0] = processor[0].complete());
            boolean madeProgress = done[0] || (outbox[0].bucketCount() > 0 && !outbox[0].queue(0).isEmpty());
            assertTrue("complete() call without progress", !assertProgress || madeProgress);
            outbox[0].drainQueuesAndReset(actualOutputs, logInputOutput);
            if (outbox[0].hasUnfinishedItem()) {
                assertFalse("outbox has unfinished items, but complete() claims to be done", done[0]);
                outbox[0].block();
            } else {
                outbox[0].unblock();
                snapshotAndRestore(processor, outbox, actualOutputs, madeProgress && doSnapshots && !done[0], doRestoreEvery, restoreCount);
            }
            idleCount = idle(idler, idleCount, madeProgress);
            long now = System.nanoTime();
            if (runUntilOutputMatchesTimeoutMillis >= 0) {
                try {
                    assertOutputFn.accept(testMode, actualOutputs);
                    outputMatchedAt = Math.min(outputMatchedAt, now);
                } catch (AssertionError e) {
                    if (outputMatchedAt < Long.MAX_VALUE) {
                        throw new AssertionError("the output already matched, but doesn't match now", e);
                    }
                // ignore the failure otherwise and continue calling complete()
                }
                long elapsedSinceStart = NANOSECONDS.toMillis(now - completeStart);
                long elapsedSinceMatch = NANOSECONDS.toMillis(subtractClamped(now, outputMatchedAt));
                if (elapsedSinceStart > runUntilOutputMatchesTimeoutMillis || elapsedSinceMatch > runUntilOutputMatchesExtraTimeMillis) {
                    break;
                }
            }
        } while (!done[0]);
        assertTrue("complete returned true in a run-until-output-matches mode", !done[0] || runUntilOutputMatchesTimeoutMillis <= 0);
    }
    processor[0].close();
    assertOutputFn.accept(testMode, actualOutputs);
}

Example 19 with Watermark

use of com.hazelcast.jet.core.Watermark in project hazelcast by hazelcast.

the class ProcessorTasklet method stateMachineStep.

@SuppressWarnings("checkstyle:returncount")
private void stateMachineStep() {
    switch(state) {
        case PROCESS_WATERMARK:
            if (pendingWatermark == null) {
                long wm = watermarkCoalescer.checkWmHistory();
                if (wm == NO_NEW_WM) {
                    state = NULLARY_PROCESS;
                    // recursion
                    stateMachineStep();
                    break;
                }
                pendingWatermark = new Watermark(wm);
            }
            if (pendingWatermark.equals(IDLE_MESSAGE) ? outbox.offer(IDLE_MESSAGE) : doWithClassLoader(context.classLoader(), () -> processor.tryProcessWatermark(pendingWatermark))) {
                state = NULLARY_PROCESS;
                pendingWatermark = null;
            }
            break;
        case NULLARY_PROCESS:
            // if currInstream is null, maybe fillInbox wasn't called yet. Avoid calling tryProcess in that case.
            if (currInstream == null || isSnapshotInbox() || doWithClassLoader(context.classLoader(), () -> processor.tryProcess())) {
                state = PROCESS_INBOX;
                outbox.reset();
                // recursion
                stateMachineStep();
            }
            break;
        case PROCESS_INBOX:
            processInbox();
            return;
        case COMPLETE_EDGE:
            if (isSnapshotInbox() ? doWithClassLoader(context.classLoader(), () -> processor.finishSnapshotRestore()) : doWithClassLoader(context.classLoader(), () -> processor.completeEdge(currInstream.ordinal()))) {
                assert !outbox.hasUnfinishedItem() || !isSnapshotInbox() : "outbox has an unfinished item after successful finishSnapshotRestore()";
                progTracker.madeProgress();
                state = processingState();
            }
            return;
        case SAVE_SNAPSHOT:
            if (doWithClassLoader(context.classLoader(), () -> processor.saveToSnapshot())) {
                progTracker.madeProgress();
                state = ssContext.isExportOnly() ? EMIT_BARRIER : SNAPSHOT_COMMIT_PREPARE;
                // recursion
                stateMachineStep();
            }
            return;
        case SNAPSHOT_COMMIT_PREPARE:
            if (doWithClassLoader(context.classLoader(), () -> processor.snapshotCommitPrepare())) {
                progTracker.madeProgress();
                state = EMIT_BARRIER;
                // recursion
                stateMachineStep();
            }
            return;
        case EMIT_BARRIER:
            assert currentBarrier != null : "currentBarrier == null";
            if (outbox.offerToEdgesAndSnapshot(currentBarrier)) {
                progTracker.madeProgress();
                if (currentBarrier.isTerminal()) {
                    state = WAITING_FOR_SNAPSHOT_COMPLETED;
                } else {
                    currentBarrier = null;
                    receivedBarriers.clear();
                    pendingSnapshotId1++;
                    state = processingState();
                }
            }
            return;
        case SNAPSHOT_COMMIT_FINISH__PROCESS:
        case SNAPSHOT_COMMIT_FINISH__COMPLETE:
        case SNAPSHOT_COMMIT_FINISH__FINAL:
            if (ssContext.isExportOnly() || doWithClassLoader(context.classLoader(), () -> processor.snapshotCommitFinish(ssContext.isLastPhase1Successful()))) {
                pendingSnapshotId2++;
                ssContext.phase2DoneForTasklet();
                progTracker.madeProgress();
                switch(state) {
                    case SNAPSHOT_COMMIT_FINISH__PROCESS:
                        state = PROCESS_INBOX;
                        break;
                    case SNAPSHOT_COMMIT_FINISH__COMPLETE:
                        state = COMPLETE;
                        break;
                    case SNAPSHOT_COMMIT_FINISH__FINAL:
                        state = PRE_EMIT_DONE_ITEM;
                        break;
                    default:
                        throw new RuntimeException("unexpected state: " + state);
                }
            }
            return;
        case WAITING_FOR_SNAPSHOT_COMPLETED:
            long currSnapshotId2 = ssContext.activeSnapshotIdPhase2();
            if (currSnapshotId2 >= pendingSnapshotId2) {
                state = SNAPSHOT_COMMIT_FINISH__FINAL;
                // recursion
                stateMachineStep();
            }
            return;
        case COMPLETE:
            complete();
            return;
        case PRE_EMIT_DONE_ITEM:
            ssContext.processorTaskletDone(pendingSnapshotId2 - 1);
            state = EMIT_DONE_ITEM;
            stateMachineStep();
            return;
        case EMIT_DONE_ITEM:
            if (outbox.offerToEdgesAndSnapshot(DONE_ITEM)) {
                progTracker.madeProgress();
                state = CLOSE;
                stateMachineStep();
            }
            return;
        case CLOSE:
            if (isCooperative() && !processor.closeIsCooperative()) {
                if (closeFuture == null) {
                    ClassLoader contextCl = Thread.currentThread().getContextClassLoader();
                    closeFuture = executionService.submit(() -> doWithClassLoader(contextCl, this::closeProcessor));
                    progTracker.madeProgress();
                }
                if (!closeFuture.isDone()) {
                    return;
                }
                progTracker.madeProgress();
            } else {
                closeProcessor();
            }
            state = END;
            progTracker.done();
            return;
        default:
            // note ProcessorState.END goes here
            throw new JetException("Unexpected state: " + state);
    }
}

Example 20 with Watermark

use of com.hazelcast.jet.core.Watermark in project hazelcast by hazelcast.

the class ProcessorTasklet method fillInbox.

private void fillInbox() {
    assert inbox.isEmpty() : "inbox is not empty";
    assert pendingWatermark == null : "null wm expected, but was " + pendingWatermark;
    // We need to collect metrics before draining the queues into Inbox,
    // otherwise they would appear empty even for slow processors
    queuesCapacity.set(instreamCursor == null ? 0 : sum(instreamCursor.getList(), InboundEdgeStream::capacities));
    queuesSize.set(instreamCursor == null ? 0 : sum(instreamCursor.getList(), InboundEdgeStream::sizes));
    if (instreamCursor == null) {
        return;
    }
    final InboundEdgeStream first = instreamCursor.value();
    ProgressState result;
    do {
        currInstream = instreamCursor.value();
        result = NO_PROGRESS;
        // skip ordinals where a snapshot barrier has already been received
        if (waitForAllBarriers && receivedBarriers.get(currInstream.ordinal())) {
            instreamCursor.advance();
            continue;
        }
        result = currInstream.drainTo(addToInboxFunction);
        progTracker.madeProgress(result.isMadeProgress());
        // check if the last drained item is special
        Object lastItem = inbox.queue().peekLast();
        if (lastItem instanceof Watermark) {
            long newWmValue = ((Watermark) inbox.queue().removeLast()).timestamp();
            long wm = watermarkCoalescer.observeWm(currInstream.ordinal(), newWmValue);
            if (wm != NO_NEW_WM) {
                pendingWatermark = new Watermark(wm);
            }
        } else if (lastItem instanceof SnapshotBarrier) {
            SnapshotBarrier barrier = (SnapshotBarrier) inbox.queue().removeLast();
            observeBarrier(currInstream.ordinal(), barrier);
        } else if (lastItem != null && !(lastItem instanceof BroadcastItem)) {
            watermarkCoalescer.observeEvent(currInstream.ordinal());
        }
        if (result.isDone()) {
            receivedBarriers.clear(currInstream.ordinal());
            long wm = watermarkCoalescer.queueDone(currInstream.ordinal());
            // In this case we might overwrite the WM here, but that's fine since the second WM should be newer.
            if (wm != NO_NEW_WM) {
                assert pendingWatermark == null || pendingWatermark.timestamp() < wm : "trying to assign lower WM. Old=" + pendingWatermark.timestamp() + ", new=" + wm;
                pendingWatermark = new Watermark(wm);
            }
            instreamCursor.remove();
            numActiveOrdinals--;
        }
        // pop current priority group
        if (!instreamCursor.advance()) {
            instreamCursor = popInstreamGroup();
            break;
        }
    } while (!result.isMadeProgress() && instreamCursor.value() != first);
    // we are the only updating thread, no need for CAS operations
    lazyAdd(receivedCounts, currInstream.ordinal(), inbox.size());
    if (!inbox.isEmpty()) {
        lazyIncrement(receivedBatches, currInstream.ordinal());
    }
}