Examples with ProgressState com.hazelcast.jet.impl.util.ProgressState used on opensource projects

Example 16 with ProgressState

use of com.hazelcast.jet.impl.util.ProgressState in project hazelcast by hazelcast.

the class ProcessorTaskletTest_Watermarks method callUntil.

private static void callUntil(ProcessorTasklet tasklet) {
    int iterCount = 0;
    for (ProgressState r; (r = tasklet.call()) != NO_PROGRESS; ) {
        assertEquals("Failed to make progress", MADE_PROGRESS, r);
        assertTrue(String.format("tasklet.call() invoked %d times without reaching %s. Last state was %s", CALL_COUNT_LIMIT, NO_PROGRESS, r), ++iterCount < CALL_COUNT_LIMIT);
    }
}

Example 17 with ProgressState

use of com.hazelcast.jet.impl.util.ProgressState in project hazelcast-jet by hazelcast.

the class SenderTasklet method tryFillInbox.

private void tryFillInbox() {
    if (!inbox.isEmpty()) {
        progTracker.notDone();
        return;
    }
    if (instreamExhausted) {
        return;
    }
    progTracker.notDone();
    final ProgressState result = inboundEdgeStream.drainTo(inbox::add);
    progTracker.madeProgress(result.isMadeProgress());
    instreamExhausted = result.isDone();
    if (instreamExhausted) {
        inbox.add(new ObjectWithPartitionId(DONE_ITEM, -1));
    }
}

Example 18 with ProgressState

use of com.hazelcast.jet.impl.util.ProgressState in project hazelcast-jet by hazelcast.

the class StoreSnapshotTasklet method stateMachineStep.

private void stateMachineStep() {
    switch(state) {
        case DRAIN:
            progTracker.notDone();
            ProgressState result = inboundEdgeStream.drainTo(o -> {
                if (o instanceof SnapshotBarrier) {
                    SnapshotBarrier barrier = (SnapshotBarrier) o;
                    assert pendingSnapshotId == barrier.snapshotId() : "Unexpected barrier, expected was " + pendingSnapshotId + ", but barrier was " + barrier.snapshotId() + ", this=" + this;
                    hasReachedBarrier = true;
                } else {
                    mapWriter.put((Entry<Data, Data>) o);
                }
            });
            if (result.isDone()) {
                inputIsDone = true;
            }
            if (result.isMadeProgress()) {
                progTracker.madeProgress();
                state = FLUSH;
                stateMachineStep();
            }
            return;
        case FLUSH:
            progTracker.notDone();
            CompletableFuture<Void> future = new CompletableFuture<>();
            future.whenComplete(withTryCatch(logger, (r, t) -> {
                // this callback may be called from a non-tasklet thread
                if (t != null) {
                    logger.severe("Error writing to snapshot map '" + currMapName() + "'", t);
                    snapshotContext.reportError(t);
                }
                // numActiveFlushes must be decremented last otherwise we may miss the error
                numActiveFlushes.decrementAndGet();
            }));
            if (mapWriter.tryFlushAsync(future)) {
                progTracker.madeProgress();
                numActiveFlushes.incrementAndGet();
                state = inputIsDone ? DONE : hasReachedBarrier ? REACHED_BARRIER : DRAIN;
            }
            return;
        case REACHED_BARRIER:
            progTracker.notDone();
            if (numActiveFlushes.get() == 0) {
                snapshotContext.snapshotDoneForTasklet();
                pendingSnapshotId++;
                mapWriter.setMapName(currMapName());
                state = inputIsDone ? DONE : DRAIN;
                hasReachedBarrier = false;
            }
            return;
        case DONE:
            if (numActiveFlushes.get() != 0) {
                progTracker.notDone();
            }
            snapshotContext.taskletDone(pendingSnapshotId - 1, isHigherPrioritySource);
            return;
        default:
            throw new JetException("Unexpected state: " + state);
    }
}

Example 19 with ProgressState

use of com.hazelcast.jet.impl.util.ProgressState in project hazelcast-jet by hazelcast.

the class ConcurrentInboundEdgeStream method drainTo.

// package-visible for testing
ProgressState drainTo(long now, Consumer<Object> dest) {
    tracker.reset();
    for (int queueIndex = 0; queueIndex < conveyor.queueCount(); queueIndex++) {
        final QueuedPipe<Object> q = conveyor.queue(queueIndex);
        if (q == null) {
            continue;
        }
        // skip queues where a snapshot barrier has already been received
        if (waitForSnapshot && receivedBarriers.get(queueIndex)) {
            continue;
        }
        ProgressState result = drainQueue(q, dest);
        tracker.mergeWith(result);
        if (itemDetector.item == DONE_ITEM) {
            conveyor.removeQueue(queueIndex);
            receivedBarriers.clear(queueIndex);
            numActiveQueues--;
            if (maybeEmitWm(watermarkCoalescer.queueDone(queueIndex), dest)) {
                return numActiveQueues == 0 ? DONE : MADE_PROGRESS;
            }
        } else if (itemDetector.item instanceof Watermark) {
            long wmTimestamp = ((Watermark) itemDetector.item).timestamp();
            boolean forwarded = maybeEmitWm(watermarkCoalescer.observeWm(now, queueIndex, wmTimestamp), dest);
            if (logger.isFinestEnabled()) {
                logger.finest("Received " + itemDetector.item + " from queue " + queueIndex + '/' + conveyor.queueCount() + (forwarded ? ", forwarded" : ", not forwarded"));
            }
            if (forwarded) {
                return MADE_PROGRESS;
            }
        } else if (itemDetector.item instanceof SnapshotBarrier) {
            observeBarrier(queueIndex, ((SnapshotBarrier) itemDetector.item).snapshotId());
        } else if (result.isMadeProgress()) {
            watermarkCoalescer.observeEvent(queueIndex);
        }
        if (numActiveQueues == 0) {
            return tracker.toProgressState();
        }
        if (itemDetector.item != null) {
            // if we have received the current snapshot from all active queues, forward it
            if (receivedBarriers.cardinality() == numActiveQueues) {
                dest.accept(new SnapshotBarrier(pendingSnapshotId));
                pendingSnapshotId++;
                receivedBarriers.clear();
                return MADE_PROGRESS;
            }
        }
    }
    // try to emit WM based on history
    if (maybeEmitWm(watermarkCoalescer.checkWmHistory(now), dest)) {
        return MADE_PROGRESS;
    }
    if (numActiveQueues > 0) {
        tracker.notDone();
    }
    return tracker.toProgressState();
}