Examples with ProgressState com.hazelcast.jet.impl.util.ProgressState used on opensource projects

Example 11 with ProgressState

use of com.hazelcast.jet.impl.util.ProgressState in project hazelcast by hazelcast.

the class ProcessorTaskletTest method callUntil.

private static void callUntil(Tasklet tasklet, ProgressState expectedState) {
    int iterCount = 0;
    for (ProgressState r; (r = tasklet.call()) != expectedState; ) {
        assertEquals("Failed to make progress", MADE_PROGRESS, r);
        assertTrue(String.format("tasklet.call() invoked %d times without reaching %s. Last state was %s", CALL_COUNT_LIMIT, expectedState, r), ++iterCount < CALL_COUNT_LIMIT);
    }
}

Example 12 with ProgressState

use of com.hazelcast.jet.impl.util.ProgressState in project hazelcast by hazelcast.

the class ProcessorTasklet method fillInbox.

private void fillInbox() {
    assert inbox.isEmpty() : "inbox is not empty";
    assert pendingWatermark == null : "null wm expected, but was " + pendingWatermark;
    // We need to collect metrics before draining the queues into Inbox,
    // otherwise they would appear empty even for slow processors
    queuesCapacity.set(instreamCursor == null ? 0 : sum(instreamCursor.getList(), InboundEdgeStream::capacities));
    queuesSize.set(instreamCursor == null ? 0 : sum(instreamCursor.getList(), InboundEdgeStream::sizes));
    if (instreamCursor == null) {
        return;
    }
    final InboundEdgeStream first = instreamCursor.value();
    ProgressState result;
    do {
        currInstream = instreamCursor.value();
        result = NO_PROGRESS;
        // skip ordinals where a snapshot barrier has already been received
        if (waitForAllBarriers && receivedBarriers.get(currInstream.ordinal())) {
            instreamCursor.advance();
            continue;
        }
        result = currInstream.drainTo(addToInboxFunction);
        progTracker.madeProgress(result.isMadeProgress());
        // check if the last drained item is special
        Object lastItem = inbox.queue().peekLast();
        if (lastItem instanceof Watermark) {
            long newWmValue = ((Watermark) inbox.queue().removeLast()).timestamp();
            long wm = watermarkCoalescer.observeWm(currInstream.ordinal(), newWmValue);
            if (wm != NO_NEW_WM) {
                pendingWatermark = new Watermark(wm);
            }
        } else if (lastItem instanceof SnapshotBarrier) {
            SnapshotBarrier barrier = (SnapshotBarrier) inbox.queue().removeLast();
            observeBarrier(currInstream.ordinal(), barrier);
        } else if (lastItem != null && !(lastItem instanceof BroadcastItem)) {
            watermarkCoalescer.observeEvent(currInstream.ordinal());
        }
        if (result.isDone()) {
            receivedBarriers.clear(currInstream.ordinal());
            long wm = watermarkCoalescer.queueDone(currInstream.ordinal());
            // In this case we might overwrite the WM here, but that's fine since the second WM should be newer.
            if (wm != NO_NEW_WM) {
                assert pendingWatermark == null || pendingWatermark.timestamp() < wm : "trying to assign lower WM. Old=" + pendingWatermark.timestamp() + ", new=" + wm;
                pendingWatermark = new Watermark(wm);
            }
            instreamCursor.remove();
            numActiveOrdinals--;
        }
        // pop current priority group
        if (!instreamCursor.advance()) {
            instreamCursor = popInstreamGroup();
            break;
        }
    } while (!result.isMadeProgress() && instreamCursor.value() != first);
    // we are the only updating thread, no need for CAS operations
    lazyAdd(receivedCounts, currInstream.ordinal(), inbox.size());
    if (!inbox.isEmpty()) {
        lazyIncrement(receivedBatches, currInstream.ordinal());
    }
}

Example 13 with ProgressState

use of com.hazelcast.jet.impl.util.ProgressState in project hazelcast by hazelcast.

the class SenderTasklet method tryFillInbox.

private void tryFillInbox() {
    if (!inbox.isEmpty()) {
        progTracker.notDone();
        return;
    }
    if (instreamExhausted) {
        return;
    }
    progTracker.notDone();
    final ProgressState result = inboundEdgeStream.drainTo(addToInboxFunction);
    progTracker.madeProgress(result.isMadeProgress());
    instreamExhausted = result.isDone();
    if (instreamExhausted) {
        inbox.add(new ObjectWithPartitionId(DONE_ITEM, -1));
    }
}

Example 14 with ProgressState

use of com.hazelcast.jet.impl.util.ProgressState in project hazelcast by hazelcast.

the class ProcessorTaskletTest_Snapshots method callUntil.

private static void callUntil(ProcessorTasklet tasklet, ProgressState expectedState) {
    int iterCount = 0;
    for (ProgressState r; (r = tasklet.call()) != expectedState; ) {
        assertEquals("Failed to make progress after " + iterCount + " iterations", MADE_PROGRESS, r);
        assertTrue(String.format("tasklet.call() invoked %d times without reaching %s. Last state was %s", CALL_COUNT_LIMIT, expectedState, r), ++iterCount < CALL_COUNT_LIMIT);
    }
}

Example 15 with ProgressState

use of com.hazelcast.jet.impl.util.ProgressState in project hazelcast by hazelcast.

the class ProcessorTaskletTest_Blocking method callUntil.

private static void callUntil(Tasklet tasklet, ProgressState expectedState) {
    System.out.println("================= call tasklet");
    int iterCount = 0;
    for (ProgressState r; (r = tasklet.call()) != expectedState; ) {
        assertEquals("Failed to make progress", MADE_PROGRESS, r);
        assertTrue(String.format("tasklet.call() invoked %d times without reaching %s. Last state was %s", CALL_COUNT_LIMIT, expectedState, r), ++iterCount < CALL_COUNT_LIMIT);
        System.out.println("================= call tasklet");
    }
}