Examples with Watermark com.hazelcast.jet.core.Watermark used on opensource projects

Example 6 with Watermark

use of com.hazelcast.jet.core.Watermark in project hazelcast-jet by hazelcast.

the class WriteLoggerPTest method test.

@Test
public void test() {
    // Given
    Processor p = supplierFrom(writeLoggerP()).get();
    TestInbox inbox = new TestInbox();
    Outbox outbox = mock(Outbox.class);
    ILogger logger = mock(ILogger.class);
    p.init(outbox, new TestProcessorContext().setLogger(logger));
    // When
    inbox.add(1);
    p.process(0, inbox);
    Watermark wm = new Watermark(2);
    p.tryProcessWatermark(wm);
    // Then
    verifyZeroInteractions(outbox);
    verify(logger).info("1");
    verify(logger).fine(wm.toString());
    verifyZeroInteractions(logger);
}

Example 7 with Watermark

use of com.hazelcast.jet.core.Watermark in project hazelcast-jet by hazelcast.

the class WatermarkMaxRetention_IntegrationTest method test_onEdgeCoalescing.

@Test
public void test_onEdgeCoalescing() {
    DAG dag = new DAG();
    // a vertex with two processor instances, one will emit a wm and the other won't
    Vertex source = dag.newVertex("source", (int count) -> asList(new ListSource(singletonList(new Watermark(1))), new StuckForeverSourceP())).localParallelism(2);
    Vertex map = dag.newVertex("map", MapWmToStringP::new).localParallelism(1);
    Vertex sink = dag.newVertex("sink", writeListP(SINK_NAME));
    dag.edge(between(source, map)).edge(between(map, sink));
    doTest(dag);
}

Example 8 with Watermark

use of com.hazelcast.jet.core.Watermark in project hazelcast-jet by hazelcast.

the class WatermarkMaxRetention_IntegrationTest method test_onVertexCoalescing.

@Test
public void test_onVertexCoalescing() {
    DAG dag = new DAG();
    Vertex source0 = dag.newVertex("source0", () -> new ListSource(singletonList(new Watermark(1)))).localParallelism(1);
    Vertex source1 = dag.newVertex("source1", StuckForeverSourceP::new);
    // a vertex with two inputs, one will emit a wm and the other won't
    Vertex map = dag.newVertex("map", MapWmToStringP::new).localParallelism(1);
    Vertex sink = dag.newVertex("sink", writeListP(SINK_NAME));
    dag.edge(from(source0).to(map, 0)).edge(from(source1).to(map, 1)).edge(between(map, sink));
    doTest(dag);
}

Example 9 with Watermark

use of com.hazelcast.jet.core.Watermark in project hazelcast by hazelcast.

the class TestSupport method processInbox.

private String processInbox(TestInbox inbox, int inboxOrdinal, boolean isCooperative, Processor[] processor) {
    if (inbox.peek() instanceof Watermark) {
        Watermark wm = ((Watermark) inbox.peek());
        doCall("tryProcessWatermark", isCooperative, () -> {
            if (processor[0].tryProcessWatermark(wm)) {
                inbox.remove();
            }
        });
        return "tryProcessWatermark";
    } else {
        doCall("process", isCooperative, () -> processor[0].process(inboxOrdinal, inbox));
        return "process";
    }
}

Example 10 with Watermark

use of com.hazelcast.jet.core.Watermark in project hazelcast by hazelcast.

the class OutboxImpl method offerInternal.

private boolean offerInternal(@Nonnull int[] ordinals, @Nonnull Object item) {
    if (shouldBlock()) {
        return false;
    }
    assert unfinishedItem == null || item.equals(unfinishedItem) : "Different item offered after previous call returned false: expected=" + unfinishedItem + ", got=" + item;
    assert unfinishedItemOrdinals == null || Arrays.equals(unfinishedItemOrdinals, ordinals) : "Offered to different ordinals after previous call returned false: expected=" + Arrays.toString(unfinishedItemOrdinals) + ", got=" + Arrays.toString(ordinals);
    assert numRemainingInBatch != -1 : "Outbox.offer() called again after it returned false, without a " + "call to reset(). You probably didn't return from Processor method after Outbox.offer() " + "or AbstractProcessor.tryEmit() returned false";
    numRemainingInBatch--;
    boolean done = true;
    if (numRemainingInBatch == -1) {
        done = false;
    } else {
        if (ordinals.length == 0) {
            // edge case - emitting to outbox with 0 ordinals is a progress
            progTracker.madeProgress();
        }
        for (int i = 0; i < ordinals.length; i++) {
            if (broadcastTracker.get(i)) {
                continue;
            }
            ProgressState result = doOffer(outstreams[ordinals[i]], item);
            if (result.isMadeProgress()) {
                progTracker.madeProgress();
            }
            if (result.isDone()) {
                broadcastTracker.set(i);
                if (!(item instanceof BroadcastItem)) {
                    // we are the only updating thread, no need for CAS operations
                    lazyIncrement(counters, ordinals[i]);
                }
            } else {
                done = false;
            }
        }
    }
    if (done) {
        broadcastTracker.clear();
        unfinishedItem = null;
        unfinishedItemOrdinals = null;
        if (item instanceof Watermark) {
            long wmTimestamp = ((Watermark) item).timestamp();
            if (wmTimestamp != WatermarkCoalescer.IDLE_MESSAGE.timestamp()) {
                // emitted to each ordinal, but we don't do that currently.
                assert lastForwardedWm.get() <= wmTimestamp : "current=" + lastForwardedWm.get() + ", new=" + wmTimestamp;
                lastForwardedWm.set(wmTimestamp);
            }
        }
    } else {
        numRemainingInBatch = -1;
        unfinishedItem = item;
        // noinspection ConstantConditions,AssertWithSideEffects
        assert (unfinishedItemOrdinals = Arrays.copyOf(ordinals, ordinals.length)) != null;
    }
    return done;
}