Examples with Watermark com.hazelcast.jet.core.Watermark used on opensource projects

Example 31 with Watermark

use of com.hazelcast.jet.core.Watermark in project hazelcast-jet by hazelcast.

the class StreamKafkaPTest method when_eventsInSinglePartition_then_watermarkAfterIdleTime.

@Test
public void when_eventsInSinglePartition_then_watermarkAfterIdleTime() {
    // When
    StreamKafkaP processor = createProcessor(2, StreamKafkaP::recordToEntry, 10_000);
    TestOutbox outbox = new TestOutbox(new int[] { 10 }, 10);
    processor.init(outbox, new TestProcessorContext());
    produce(topic1Name, 10, "foo");
    // Then
    assertEquals(entry(10, "foo"), consumeEventually(processor, outbox));
    long time1 = System.nanoTime();
    assertEquals(new Watermark(10 - LAG), consumeEventually(processor, outbox));
    long time2 = System.nanoTime();
    long elapsedMs = NANOSECONDS.toMillis(time2 - time1);
    assertTrue("elapsed time: " + elapsedMs + " ms, should be larger", elapsedMs > 3000 && elapsedMs <= 10_000);
}

Example 32 with Watermark

use of com.hazelcast.jet.core.Watermark in project hazelcast-jet by hazelcast.

the class StreamKafkaPTest method when_noAssignedPartitionAndAddedLater_then_resumesFromIdle.

@Test
public void when_noAssignedPartitionAndAddedLater_then_resumesFromIdle() throws Exception {
    // we ask to create 5th out of 5 processors, but we have only 4 partitions and 1 topic
    // --> our processor will have nothing assigned
    StreamKafkaP processor = createProcessor(1, StreamKafkaP::recordToEntry, 10_000);
    TestOutbox outbox = new TestOutbox(new int[] { 10 }, 10);
    processor.init(outbox, new TestProcessorContext().setTotalParallelism(INITIAL_PARTITION_COUNT + 1).setGlobalProcessorIndex(INITIAL_PARTITION_COUNT));
    assertTrue(processor.currentAssignment.isEmpty());
    assertEquals(IDLE_MESSAGE, consumeEventually(processor, outbox));
    setPartitionCount(topic1Name, INITIAL_PARTITION_COUNT + 1);
    Thread.sleep(1000);
    // this allows production to the added partition
    resetProducer();
    // produce events until the event happens to go to the added partition
    Entry<Integer, String> event;
    for (int i = 0; ; i++) {
        event = entry(i, Integer.toString(i));
        Future<RecordMetadata> future = produce(topic1Name, event.getKey(), event.getValue());
        RecordMetadata recordMetadata = future.get();
        if (recordMetadata.partition() == 4) {
            break;
        }
    }
    assertEquals(new Watermark(event.getKey() - LAG), consumeEventually(processor, outbox));
    assertEquals(event, consumeEventually(processor, outbox));
}