Examples with ProgressState com.hazelcast.jet.impl.util.ProgressState used on opensource projects

Example 6 with ProgressState

use of com.hazelcast.jet.impl.util.ProgressState in project hazelcast-jet by hazelcast.

the class ReceiverTasklet method call.

@Override
@Nonnull
public ProgressState call() {
    if (receptionDone) {
        return collector.offerBroadcast(DONE_ITEM);
    }
    tracker.reset();
    tracker.notDone();
    tryFillInbox();
    for (ObjWithPtionIdAndSize o; (o = inbox.peek()) != null; ) {
        final Object item = o.getItem();
        if (item == DONE_ITEM) {
            receptionDone = true;
            inbox.remove();
            assert inbox.peek() == null : "Found something in the queue beyond the DONE_ITEM: " + inbox.remove();
            break;
        }
        ProgressState outcome = item instanceof BroadcastItem ? collector.offerBroadcast((BroadcastItem) item) : collector.offer(item, o.getPartitionId());
        if (!outcome.isDone()) {
            tracker.madeProgress(outcome.isMadeProgress());
            break;
        }
        tracker.madeProgress();
        inbox.remove();
        ackItem(o.estimatedMemoryFootprint);
    }
    return tracker.toProgressState();
}

Example 7 with ProgressState

use of com.hazelcast.jet.impl.util.ProgressState in project hazelcast-jet by hazelcast.

the class OutboxImpl method offerInternal.

private boolean offerInternal(@Nonnull int[] ordinals, @Nonnull Object item) {
    assert unfinishedItem == null || item.equals(unfinishedItem) : "Different item offered after previous call returned false: expected=" + unfinishedItem + ", got=" + item;
    assert unfinishedItemOrdinals == null || Arrays.equals(unfinishedItemOrdinals, ordinals) : "Offered to different ordinals after previous call returned false: expected=" + Arrays.toString(unfinishedItemOrdinals) + ", got=" + Arrays.toString(ordinals);
    assert numRemainingInBatch != -1 : "Outbox.offer() called again after it returned false, without a " + "call to reset(). You probably didn't return from Processor method after Outbox.offer() " + "or AbstractProcessor.tryEmit() returned false";
    numRemainingInBatch--;
    boolean done = true;
    if (numRemainingInBatch == -1) {
        done = false;
    } else {
        for (int i = 0; i < ordinals.length; i++) {
            if (broadcastTracker.get(i)) {
                continue;
            }
            ProgressState result = doOffer(outstreams[ordinals[i]], item);
            if (result.isMadeProgress()) {
                progTracker.madeProgress();
            }
            if (result.isDone()) {
                broadcastTracker.set(i);
            } else {
                done = false;
            }
        }
    }
    if (done) {
        broadcastTracker.clear();
        unfinishedItem = null;
        unfinishedItemOrdinals = null;
    } else {
        numRemainingInBatch = -1;
        unfinishedItem = item;
        // noinspection ConstantConditions,AssertWithSideEffects
        assert (unfinishedItemOrdinals = Arrays.copyOf(ordinals, ordinals.length)) != null;
    }
    return done;
}

Example 8 with ProgressState

use of com.hazelcast.jet.impl.util.ProgressState in project hazelcast by hazelcast.

the class StoreSnapshotTasklet method stateMachineStep.

private void stateMachineStep() {
    switch(state) {
        case DRAIN:
            progTracker.notDone();
            if (pendingEntry != null) {
                if (!ssWriter.offer(pendingEntry)) {
                    return;
                }
                progTracker.madeProgress();
            }
            pendingEntry = null;
            ProgressState result = inboundEdgeStream.drainTo(addToInboxFunction);
            if (result.isDone()) {
                assert ssWriter.isEmpty() : "input is done, but we had some entries and not the barrier";
                snapshotContext.storeSnapshotTaskletDone(pendingSnapshotId - 1, isHigherPrioritySource);
                state = DONE;
                progTracker.reset();
            }
            progTracker.madeProgress(result.isMadeProgress());
            if (hasReachedBarrier) {
                state = FLUSH;
                stateMachineStep();
            }
            break;
        case FLUSH:
            progTracker.notDone();
            if (ssWriter.flushAndResetMap()) {
                progTracker.madeProgress();
                state = REACHED_BARRIER;
            }
            break;
        case REACHED_BARRIER:
            if (ssWriter.hasPendingAsyncOps()) {
                progTracker.notDone();
                return;
            }
            // check for writing error
            Throwable error = ssWriter.getError();
            if (error != null) {
                logger.severe("Error writing to snapshot map", error);
                snapshotContext.reportError(error);
            }
            progTracker.madeProgress();
            long bytes = ssWriter.getTotalPayloadBytes();
            long keys = ssWriter.getTotalKeys();
            long chunks = ssWriter.getTotalChunks();
            snapshotContext.phase1DoneForTasklet(bytes, keys, chunks);
            metrics.set(new LongLongAccumulator(bytes, keys));
            ssWriter.resetStats();
            pendingSnapshotId++;
            hasReachedBarrier = false;
            state = DRAIN;
            progTracker.notDone();
            break;
        default:
            // note State.DONE also goes here
            throw new JetException("Unexpected state: " + state);
    }
}

Example 9 with ProgressState

use of com.hazelcast.jet.impl.util.ProgressState in project hazelcast by hazelcast.

the class OutboxImpl method offerInternal.

private boolean offerInternal(@Nonnull int[] ordinals, @Nonnull Object item) {
    if (shouldBlock()) {
        return false;
    }
    assert unfinishedItem == null || item.equals(unfinishedItem) : "Different item offered after previous call returned false: expected=" + unfinishedItem + ", got=" + item;
    assert unfinishedItemOrdinals == null || Arrays.equals(unfinishedItemOrdinals, ordinals) : "Offered to different ordinals after previous call returned false: expected=" + Arrays.toString(unfinishedItemOrdinals) + ", got=" + Arrays.toString(ordinals);
    assert numRemainingInBatch != -1 : "Outbox.offer() called again after it returned false, without a " + "call to reset(). You probably didn't return from Processor method after Outbox.offer() " + "or AbstractProcessor.tryEmit() returned false";
    numRemainingInBatch--;
    boolean done = true;
    if (numRemainingInBatch == -1) {
        done = false;
    } else {
        if (ordinals.length == 0) {
            // edge case - emitting to outbox with 0 ordinals is a progress
            progTracker.madeProgress();
        }
        for (int i = 0; i < ordinals.length; i++) {
            if (broadcastTracker.get(i)) {
                continue;
            }
            ProgressState result = doOffer(outstreams[ordinals[i]], item);
            if (result.isMadeProgress()) {
                progTracker.madeProgress();
            }
            if (result.isDone()) {
                broadcastTracker.set(i);
                if (!(item instanceof BroadcastItem)) {
                    // we are the only updating thread, no need for CAS operations
                    lazyIncrement(counters, ordinals[i]);
                }
            } else {
                done = false;
            }
        }
    }
    if (done) {
        broadcastTracker.clear();
        unfinishedItem = null;
        unfinishedItemOrdinals = null;
        if (item instanceof Watermark) {
            long wmTimestamp = ((Watermark) item).timestamp();
            if (wmTimestamp != WatermarkCoalescer.IDLE_MESSAGE.timestamp()) {
                // emitted to each ordinal, but we don't do that currently.
                assert lastForwardedWm.get() <= wmTimestamp : "current=" + lastForwardedWm.get() + ", new=" + wmTimestamp;
                lastForwardedWm.set(wmTimestamp);
            }
        }
    } else {
        numRemainingInBatch = -1;
        unfinishedItem = item;
        // noinspection ConstantConditions,AssertWithSideEffects
        assert (unfinishedItemOrdinals = Arrays.copyOf(ordinals, ordinals.length)) != null;
    }
    return done;
}

Example 10 with ProgressState

use of com.hazelcast.jet.impl.util.ProgressState in project hazelcast by hazelcast.

the class ReceiverTasklet method call.

@Override
@Nonnull
public ProgressState call() {
    if (receptionDone) {
        return collector.offerBroadcast(DONE_ITEM);
    }
    if (connectionChanged) {
        throw new RestartableException("The member was reconnected: " + sourceAddressString);
    }
    tracker.reset();
    tracker.notDone();
    tryFillInbox();
    int ackItemLocal = 0;
    for (ObjWithPtionIdAndSize o; (o = inbox.peek()) != null; ) {
        final Object item = o.getItem();
        if (item == DONE_ITEM) {
            receptionDone = true;
            inbox.remove();
            assert inbox.peek() == null : "Found something in the queue beyond the DONE_ITEM: " + inbox.remove();
            break;
        }
        ProgressState outcome = item instanceof BroadcastItem ? collector.offerBroadcast((BroadcastItem) item) : collector.offer(item, o.getPartitionId());
        if (!outcome.isDone()) {
            tracker.madeProgress(outcome.isMadeProgress());
            break;
        }
        tracker.madeProgress();
        inbox.remove();
        ackItemLocal += o.estimatedMemoryFootprint;
    }
    ackItem(ackItemLocal);
    numWaitingInInbox = inbox.size();
    return tracker.toProgressState();
}