Examples with Watermark com.hazelcast.jet.core.Watermark used on opensource projects

Example 1 with Watermark

use of com.hazelcast.jet.core.Watermark in project hazelcast-jet by hazelcast.

the class StreamKafkaPTest method when_eventsInAllPartitions_then_watermarkOutputImmediately.

@Test
public void when_eventsInAllPartitions_then_watermarkOutputImmediately() {
    StreamKafkaP processor = createProcessor(1, StreamKafkaP::recordToEntry, 10_000);
    TestOutbox outbox = new TestOutbox(new int[] { 10 }, 10);
    processor.init(outbox, new TestProcessorContext());
    for (int i = 0; i < INITIAL_PARTITION_COUNT; i++) {
        Entry<Integer, String> event = entry(i + 100, Integer.toString(i));
        System.out.println("produced event " + event);
        produce(topic1Name, i, event.getKey(), event.getValue());
        if (i == INITIAL_PARTITION_COUNT - 1) {
            assertEquals(new Watermark(100 - LAG), consumeEventually(processor, outbox));
        }
        assertEquals(event, consumeEventually(processor, outbox));
    }
}

Example 2 with Watermark

use of com.hazelcast.jet.core.Watermark in project hazelcast-jet by hazelcast.

the class SessionWindowPTest method when_batchProcessing_then_flushEverything.

@Test
public void when_batchProcessing_then_flushEverything() {
    List<Object> inbox = new ArrayList<>();
    // Given
    inbox.addAll(eventsWithKey("a"));
    // This watermark will cause the first session to be emitted, but not the second.
    // The second session will be emitted in complete()
    inbox.add(new Watermark(25));
    verifyProcessor(supplier).input(inbox).expectOutput(asList(new WindowResult(1, 22, "a", 3), new Watermark(25), new WindowResult(30, 50, "a", 3)));
}

Example 3 with Watermark

use of com.hazelcast.jet.core.Watermark in project hazelcast-jet by hazelcast.

the class SessionWindowPTest method runBench.

@SuppressWarnings("checkstyle:emptystatement")
private void runBench() {
    Random rnd = ThreadLocalRandom.current();
    long start = System.nanoTime();
    long eventCount = 40_000_000;
    long keyCount = 2000;
    long eventsPerKey = eventCount / keyCount;
    int spread = 4000;
    int timestampStep = 20;
    int wmLag = 2000;
    long wmInterval = 100;
    System.out.format("keyCount %,d eventsPerKey %,d wmInterval %,d%n", keyCount, eventsPerKey, wmInterval);
    TestOutbox outbox = new TestOutbox(1024);
    // called for side-effect of assigning to lastSuppliedProcessor
    supplier.get();
    lastSuppliedProcessor.init(outbox, new TestProcessorContext());
    for (long idx = 0; idx < eventsPerKey; idx++) {
        long timestampBase = idx * timestampStep;
        for (long key = (timestampBase / SESSION_TIMEOUT) % 2; key < keyCount; key += 2) {
            while (!lastSuppliedProcessor.tryProcess(0, entry(key, timestampBase + rnd.nextInt(spread)))) {
            }
            while (!lastSuppliedProcessor.tryProcess(0, entry(key, timestampBase + rnd.nextInt(spread)))) {
            }
        }
        if (idx % wmInterval == 0) {
            long wm = timestampBase - wmLag;
            int winCount = 0;
            while (!lastSuppliedProcessor.tryProcessWatermark(new Watermark(wm))) {
                while (outbox.queue(0).poll() != null) {
                    winCount++;
                }
            }
            while (outbox.queue(0).poll() != null) {
                winCount++;
            }
        }
    }
    long took = System.nanoTime() - start;
    System.out.format("%nThroughput %,3d events/second%n", SECONDS.toNanos(1) * eventCount / took);
}

Example 4 with Watermark

use of com.hazelcast.jet.core.Watermark in project hazelcast-jet by hazelcast.

the class ProcessorTasklet method fillInbox.

private void fillInbox(long now) {
    assert inbox.isEmpty() : "inbox is not empty";
    assert pendingWatermark == null : "null wm expected, but was " + pendingWatermark;
    if (instreamCursor == null) {
        return;
    }
    final InboundEdgeStream first = instreamCursor.value();
    ProgressState result;
    do {
        currInstream = instreamCursor.value();
        result = NO_PROGRESS;
        // skip ordinals where a snapshot barrier has already been received
        if (ssContext != null && ssContext.processingGuarantee() == ProcessingGuarantee.EXACTLY_ONCE && receivedBarriers.get(currInstream.ordinal())) {
            instreamCursor.advance();
            continue;
        }
        result = currInstream.drainTo(inbox.queue()::add);
        progTracker.madeProgress(result.isMadeProgress());
        // check if the last drained item is special
        Object lastItem = inbox.queue().peekLast();
        if (lastItem instanceof Watermark) {
            long newWmValue = ((Watermark) inbox.queue().removeLast()).timestamp();
            long wm = watermarkCoalescer.observeWm(now, currInstream.ordinal(), newWmValue);
            if (wm != NO_NEW_WM) {
                pendingWatermark = new Watermark(wm);
            }
        } else if (lastItem instanceof SnapshotBarrier) {
            SnapshotBarrier barrier = (SnapshotBarrier) inbox.queue().removeLast();
            observeSnapshot(currInstream.ordinal(), barrier.snapshotId());
        } else if (lastItem != null && !(lastItem instanceof BroadcastItem)) {
            watermarkCoalescer.observeEvent(currInstream.ordinal());
        }
        if (result.isDone()) {
            receivedBarriers.clear(currInstream.ordinal());
            long wm = watermarkCoalescer.queueDone(currInstream.ordinal());
            // In this case we might overwrite the WM here, but that's fine since the second WM should be newer.
            if (wm != NO_NEW_WM) {
                assert pendingWatermark == null || pendingWatermark.timestamp() < wm : "trying to assign lower WM. Old=" + pendingWatermark.timestamp() + ", new=" + wm;
                pendingWatermark = new Watermark(wm);
            }
            instreamCursor.remove();
            numActiveOrdinals--;
        }
        // pop current priority group
        if (!instreamCursor.advance()) {
            instreamCursor = popInstreamGroup();
            return;
        }
    } while (!result.isMadeProgress() && instreamCursor.value() != first);
}

Example 5 with Watermark

use of com.hazelcast.jet.core.Watermark in project hazelcast-jet by hazelcast.

the class TestSupport method runTest.

private void runTest(boolean doSnapshots, int doRestoreEvery) throws Exception {
    assert doSnapshots || doRestoreEvery == 0 : "Illegal combination: don't do snapshots, but do restore";
    IdleStrategy idler = new BackoffIdleStrategy(0, 0, MICROSECONDS.toNanos(1), MILLISECONDS.toNanos(1));
    int idleCount = 0;
    if (doSnapshots && doRestoreEvery == 1) {
        // we do all 3 possible combinations: no snapshot, only snapshots and snapshots+restore
        runTest(false, 0);
        runTest(true, Integer.MAX_VALUE);
        runTest(true, 2);
    }
    System.out.println("### Running the test, mode=" + modeDescription(doSnapshots, doRestoreEvery));
    TestInbox inbox = new TestInbox();
    int inboxOrdinal = -1;
    Processor[] processor = { newProcessorFromSupplier() };
    boolean isCooperative = processor[0].isCooperative();
    // we'll use 1-capacity outbox to test outbox rejection
    TestOutbox[] outbox = { createOutbox() };
    List<List<Object>> actualOutputs = new ArrayList<>(expectedOutputs.size());
    for (int i = 0; i < expectedOutputs.size(); i++) {
        actualOutputs.add(new ArrayList());
    }
    // create instance of your processor and call the init() method
    initProcessor(processor[0], outbox[0]);
    int[] restoreCount = { 0 };
    // do snapshot+restore before processing any item. This will test saveToSnapshot() in this edge case
    snapshotAndRestore(processor, outbox, actualOutputs, doSnapshots, doRestoreEvery, restoreCount);
    // call the process() method
    List<ObjectWithOrdinal> input = mixInputs(inputs, priorities);
    Iterator<ObjectWithOrdinal> inputIterator = input.iterator();
    Watermark[] wmToProcess = { null };
    while (inputIterator.hasNext() || !inbox.isEmpty() || wmToProcess[0] != null) {
        if (inbox.isEmpty() && wmToProcess[0] == null && inputIterator.hasNext()) {
            ObjectWithOrdinal objectWithOrdinal = inputIterator.next();
            inbox.queue().add(objectWithOrdinal.item);
            inboxOrdinal = objectWithOrdinal.ordinal;
            if (logInputOutput) {
                System.out.println(LocalTime.now() + " Input-" + objectWithOrdinal.ordinal + ": " + inbox.peek());
            }
        }
        String methodName;
        if (wmToProcess[0] != null) {
            methodName = "offer";
            if (outbox[0].offer(wmToProcess[0])) {
                wmToProcess[0] = null;
            }
        } else {
            methodName = processInbox(inbox, inboxOrdinal, isCooperative, processor, wmToProcess);
        }
        boolean madeProgress = inbox.isEmpty() || !outbox[0].queue(0).isEmpty();
        assertTrue(methodName + "() call without progress", !assertProgress || madeProgress);
        idleCount = idle(idler, idleCount, madeProgress);
        if (outbox[0].queue(0).size() == 1 && !inbox.isEmpty()) {
            // if the outbox is full, call the process() method again. Cooperative
            // processor must be able to cope with this situation and not try to put
            // more items to the outbox.
            outbox[0].reset();
            processInbox(inbox, inboxOrdinal, isCooperative, processor, wmToProcess);
        }
        outbox[0].drainQueuesAndReset(actualOutputs, logInputOutput);
        if (inbox.isEmpty() && wmToProcess[0] == null) {
            snapshotAndRestore(processor, outbox, actualOutputs, doSnapshots, doRestoreEvery, restoreCount);
        }
    }
    if (logInputOutput && !inputs.isEmpty()) {
        System.out.println(LocalTime.now() + " Input processed, calling complete()");
    }
    // call the complete() method
    if (callComplete) {
        long completeStart = System.nanoTime();
        boolean[] done = { false };
        double elapsed;
        do {
            checkTime("complete", isCooperative, () -> done[0] = processor[0].complete());
            boolean madeProgress = done[0] || !outbox[0].queue(0).isEmpty();
            assertTrue("complete() call without progress", !assertProgress || madeProgress);
            outbox[0].drainQueuesAndReset(actualOutputs, logInputOutput);
            snapshotAndRestore(processor, outbox, actualOutputs, madeProgress && doSnapshots && !done[0], doRestoreEvery, restoreCount);
            idleCount = idle(idler, idleCount, madeProgress);
            if (runUntilCompletedTimeout > 0) {
                elapsed = toMillis(System.nanoTime() - completeStart);
                if (elapsed > runUntilCompletedTimeout) {
                    break;
                }
            }
        } while (!done[0]);
        assertTrue("complete returned true", !done[0] || runUntilCompletedTimeout <= 0);
    }
    processor[0].close(null);
    // assert the outbox
    for (int i = 0; i < expectedOutputs.size(); i++) {
        List<?> expectedOutput = expectedOutputs.get(i);
        List<?> actualOutput = actualOutputs.get(i);
        if (!outputChecker.test(expectedOutput, actualOutput)) {
            assertEquals("processor output in mode \"" + modeDescription(doSnapshots, doRestoreEvery) + "\" doesn't match", listToString(expectedOutput), listToString(actualOutput));
        }
    }
}