Examples with TestOutbox com.hazelcast.jet.core.test.TestOutbox used on opensource projects

Example 1 with TestOutbox

use of com.hazelcast.jet.core.test.TestOutbox in project hazelcast-jet by hazelcast.

the class StreamKafkaPTest method when_snapshotSaved_then_offsetsRestored.

@Test
public void when_snapshotSaved_then_offsetsRestored() throws Exception {
    StreamKafkaP processor = createProcessor(2, StreamKafkaP::recordToEntry, 10_000);
    TestOutbox outbox = new TestOutbox(new int[] { 10 }, 10);
    processor.init(outbox, new TestProcessorContext().setProcessingGuarantee(ProcessingGuarantee.EXACTLY_ONCE));
    produce(topic1Name, 0, "0");
    assertEquals(entry(0, "0"), consumeEventually(processor, outbox));
    // create snapshot
    TestInbox snapshot = saveSnapshot(processor, outbox);
    Set snapshotItems = unwrapBroadcastKey(snapshot.queue());
    // consume one more item
    produce(topic1Name, 1, "1");
    assertEquals(entry(1, "1"), consumeEventually(processor, outbox));
    // create new processor and restore snapshot
    processor = createProcessor(2, StreamKafkaP::recordToEntry, 10_000);
    outbox = new TestOutbox(new int[] { 10 }, 10);
    processor.init(outbox, new TestProcessorContext().setProcessingGuarantee(ProcessingGuarantee.EXACTLY_ONCE));
    // restore snapshot
    processor.restoreFromSnapshot(snapshot);
    assertTrue("snapshot not fully processed", snapshot.isEmpty());
    TestInbox snapshot2 = saveSnapshot(processor, outbox);
    assertEquals("new snapshot not equal after restore", snapshotItems, unwrapBroadcastKey(snapshot2.queue()));
    // the second item should be produced one more time
    assertEquals(entry(1, "1"), consumeEventually(processor, outbox));
    assertNoMoreItems(processor, outbox);
}

Example 2 with TestOutbox

use of com.hazelcast.jet.core.test.TestOutbox in project hazelcast-jet by hazelcast.

the class StreamKafkaPTest method when_partitionAdded_then_consumedFromBeginning.

@Test
public void when_partitionAdded_then_consumedFromBeginning() throws Exception {
    properties.setProperty("metadata.max.age.ms", "100");
    StreamKafkaP processor = createProcessor(2, StreamKafkaP::recordToEntry, 10_000);
    TestOutbox outbox = new TestOutbox(new int[] { 10 }, 10);
    processor.init(outbox, new TestProcessorContext());
    produce(topic1Name, 0, "0");
    assertEquals(entry(0, "0"), consumeEventually(processor, outbox));
    setPartitionCount(topic1Name, INITIAL_PARTITION_COUNT + 2);
    Thread.sleep(1000);
    // this allows production to the added partition
    resetProducer();
    boolean somethingInPartition1 = false;
    for (int i = 1; i < 11; i++) {
        Future<RecordMetadata> future = produce(topic1Name, i, Integer.toString(i));
        RecordMetadata recordMetadata = future.get();
        System.out.println("Entry " + i + " produced to partition " + recordMetadata.partition());
        somethingInPartition1 |= recordMetadata.partition() == 1;
    }
    assertTrue("nothing was produced to partition-1", somethingInPartition1);
    Set receivedEvents = new HashSet();
    for (int i = 1; i < 11; i++) {
        try {
            receivedEvents.add(consumeEventually(processor, outbox));
        } catch (AssertionError e) {
            throw new AssertionError("Unable to receive 10 items, events so far: " + receivedEvents);
        }
    }
    assertEquals(range(1, 11).mapToObj(i -> entry(i, Integer.toString(i))).collect(toSet()), receivedEvents);
}

Example 3 with TestOutbox

use of com.hazelcast.jet.core.test.TestOutbox in project hazelcast-jet by hazelcast.

the class StreamKafkaPTest method when_eventsInAllPartitions_then_watermarkOutputImmediately.

@Test
public void when_eventsInAllPartitions_then_watermarkOutputImmediately() {
    StreamKafkaP processor = createProcessor(1, StreamKafkaP::recordToEntry, 10_000);
    TestOutbox outbox = new TestOutbox(new int[] { 10 }, 10);
    processor.init(outbox, new TestProcessorContext());
    for (int i = 0; i < INITIAL_PARTITION_COUNT; i++) {
        Entry<Integer, String> event = entry(i + 100, Integer.toString(i));
        System.out.println("produced event " + event);
        produce(topic1Name, i, event.getKey(), event.getValue());
        if (i == INITIAL_PARTITION_COUNT - 1) {
            assertEquals(new Watermark(100 - LAG), consumeEventually(processor, outbox));
        }
        assertEquals(event, consumeEventually(processor, outbox));
    }
}

Example 4 with TestOutbox

use of com.hazelcast.jet.core.test.TestOutbox in project hazelcast-jet by hazelcast.

the class InsertWatermarksPTest method setUp.

@Before
public void setUp() {
    outbox = new TestOutbox(outboxCapacity);
    context = new TestProcessorContext();
}

Example 5 with TestOutbox

use of com.hazelcast.jet.core.test.TestOutbox in project hazelcast-jet by hazelcast.

the class SessionWindowPTest method runBench.

@SuppressWarnings("checkstyle:emptystatement")
private void runBench() {
    Random rnd = ThreadLocalRandom.current();
    long start = System.nanoTime();
    long eventCount = 40_000_000;
    long keyCount = 2000;
    long eventsPerKey = eventCount / keyCount;
    int spread = 4000;
    int timestampStep = 20;
    int wmLag = 2000;
    long wmInterval = 100;
    System.out.format("keyCount %,d eventsPerKey %,d wmInterval %,d%n", keyCount, eventsPerKey, wmInterval);
    TestOutbox outbox = new TestOutbox(1024);
    // called for side-effect of assigning to lastSuppliedProcessor
    supplier.get();
    lastSuppliedProcessor.init(outbox, new TestProcessorContext());
    for (long idx = 0; idx < eventsPerKey; idx++) {
        long timestampBase = idx * timestampStep;
        for (long key = (timestampBase / SESSION_TIMEOUT) % 2; key < keyCount; key += 2) {
            while (!lastSuppliedProcessor.tryProcess(0, entry(key, timestampBase + rnd.nextInt(spread)))) {
            }
            while (!lastSuppliedProcessor.tryProcess(0, entry(key, timestampBase + rnd.nextInt(spread)))) {
            }
        }
        if (idx % wmInterval == 0) {
            long wm = timestampBase - wmLag;
            int winCount = 0;
            while (!lastSuppliedProcessor.tryProcessWatermark(new Watermark(wm))) {
                while (outbox.queue(0).poll() != null) {
                    winCount++;
                }
            }
            while (outbox.queue(0).poll() != null) {
                winCount++;
            }
        }
    }
    long took = System.nanoTime() - start;
    System.out.format("%nThroughput %,3d events/second%n", SECONDS.toNanos(1) * eventCount / took);
}