Examples with Watermark com.hazelcast.jet.core.Watermark used on opensource projects

Example 21 with Watermark

use of com.hazelcast.jet.core.Watermark in project hazelcast-jet by hazelcast.

the class WriteBufferedPTest method writeBuffered_smokeTest.

@Test
public void writeBuffered_smokeTest() {
    DistributedSupplier<Processor> supplier = getLoggingBufferedWriter();
    Processor p = supplier.get();
    Outbox outbox = mock(Outbox.class);
    p.init(outbox, mock(Context.class));
    TestInbox inbox = new TestInbox();
    inbox.add(1);
    inbox.add(2);
    p.process(0, inbox);
    inbox.add(3);
    inbox.add(4);
    p.process(0, inbox);
    // watermark should not be written
    p.tryProcessWatermark(new Watermark(0));
    // empty flush
    p.process(0, inbox);
    p.complete();
    assertEquals(asList("new", "add:1", "add:2", "flush", "add:3", "add:4", "flush", "flush", "dispose"), events);
    assertEquals(0, inbox.size());
    verifyZeroInteractions(outbox);
}

Example 22 with Watermark

use of com.hazelcast.jet.core.Watermark in project hazelcast-jet by hazelcast.

the class StreamKafkaPTest method when_eventsInSinglePartition_then_watermarkAfterIdleTime.

@Test
public void when_eventsInSinglePartition_then_watermarkAfterIdleTime() {
    // When
    StreamKafkaP processor = createProcessor(2, StreamKafkaP::recordToEntry, 10_000);
    TestOutbox outbox = new TestOutbox(new int[] { 10 }, 10);
    processor.init(outbox, new TestProcessorContext());
    produce(topic1Name, 10, "foo");
    // Then
    assertEquals(entry(10, "foo"), consumeEventually(processor, outbox));
    long time1 = System.nanoTime();
    assertEquals(new Watermark(10 - LAG), consumeEventually(processor, outbox));
    long time2 = System.nanoTime();
    long elapsedMs = NANOSECONDS.toMillis(time2 - time1);
    assertTrue("elapsed time: " + elapsedMs + " ms, should be larger", elapsedMs > 3000 && elapsedMs <= 10_000);
}

Example 23 with Watermark

use of com.hazelcast.jet.core.Watermark in project hazelcast-jet by hazelcast.

the class StreamKafkaPTest method when_noAssignedPartitionAndAddedLater_then_resumesFromIdle.

@Test
public void when_noAssignedPartitionAndAddedLater_then_resumesFromIdle() throws Exception {
    // we ask to create 5th out of 5 processors, but we have only 4 partitions and 1 topic
    // --> our processor will have nothing assigned
    StreamKafkaP processor = createProcessor(1, StreamKafkaP::recordToEntry, 10_000);
    TestOutbox outbox = new TestOutbox(new int[] { 10 }, 10);
    processor.init(outbox, new TestProcessorContext().setTotalParallelism(INITIAL_PARTITION_COUNT + 1).setGlobalProcessorIndex(INITIAL_PARTITION_COUNT));
    assertTrue(processor.currentAssignment.isEmpty());
    assertEquals(IDLE_MESSAGE, consumeEventually(processor, outbox));
    setPartitionCount(topic1Name, INITIAL_PARTITION_COUNT + 1);
    Thread.sleep(1000);
    // this allows production to the added partition
    resetProducer();
    // produce events until the event happens to go to the added partition
    Entry<Integer, String> event;
    for (int i = 0; ; i++) {
        event = entry(i, Integer.toString(i));
        Future<RecordMetadata> future = produce(topic1Name, event.getKey(), event.getValue());
        RecordMetadata recordMetadata = future.get();
        if (recordMetadata.partition() == 4) {
            break;
        }
    }
    assertEquals(new Watermark(event.getKey() - LAG), consumeEventually(processor, outbox));
    assertEquals(event, consumeEventually(processor, outbox));
}

Example 24 with Watermark

use of com.hazelcast.jet.core.Watermark in project hazelcast by hazelcast.

the class SessionWindowPTest method runBench.

@SuppressWarnings("checkstyle:emptystatement")
private void runBench() throws Exception {
    Random rnd = ThreadLocalRandom.current();
    long start = System.nanoTime();
    long eventCount = 40_000_000;
    long keyCount = 2000;
    long eventsPerKey = eventCount / keyCount;
    int spread = 4000;
    int timestampStep = 20;
    int wmLag = 2000;
    long wmInterval = 100;
    System.out.format("keyCount %,d eventsPerKey %,d wmInterval %,d%n", keyCount, eventsPerKey, wmInterval);
    TestOutbox outbox = new TestOutbox(1024);
    // called for side-effect of assigning to lastSuppliedProcessor
    supplier.get();
    lastSuppliedProcessor.init(outbox, new TestProcessorContext());
    for (long idx = 0; idx < eventsPerKey; idx++) {
        long timestampBase = idx * timestampStep;
        for (long key = (timestampBase / SESSION_TIMEOUT) % 2; key < keyCount; key += 2) {
            while (!lastSuppliedProcessor.tryProcess(0, entry(key, timestampBase + rnd.nextInt(spread)))) {
            }
            while (!lastSuppliedProcessor.tryProcess(0, entry(key, timestampBase + rnd.nextInt(spread)))) {
            }
        }
        if (idx % wmInterval == 0) {
            long wm = timestampBase - wmLag;
            int winCount = 0;
            while (!lastSuppliedProcessor.tryProcessWatermark(new Watermark(wm))) {
                while (outbox.queue(0).poll() != null) {
                    winCount++;
                }
            }
            while (outbox.queue(0).poll() != null) {
                winCount++;
            }
        }
    }
    long took = System.nanoTime() - start;
    System.out.format("%nThroughput %,3d events/second%n", SECONDS.toNanos(1) * eventCount / took);
}

Example 25 with Watermark

use of com.hazelcast.jet.core.Watermark in project hazelcast by hazelcast.

the class WriteLoggerPTest method test.

@Test
public void test() throws Exception {
    // Given
    Processor p = supplierFrom(writeLoggerP()).get();
    TestInbox inbox = new TestInbox();
    Outbox outbox = mock(Outbox.class);
    ILogger logger = mock(ILogger.class);
    p.init(outbox, new TestProcessorContext().setLogger(logger));
    // When
    inbox.add(1);
    p.process(0, inbox);
    Watermark wm = new Watermark(2);
    p.tryProcessWatermark(wm);
    // Then
    verifyZeroInteractions(outbox);
    verify(logger).info("1");
    verify(logger).fine(wm.toString());
    verifyZeroInteractions(logger);
}