Examples with TestInbox com.hazelcast.jet.core.test.TestInbox used on opensource projects

Example 6 with TestInbox

use of com.hazelcast.jet.core.test.TestInbox in project hazelcast by hazelcast.

the class AsyncTransformUsingServicePTest method test_watermarksConflated.

@Test
public void test_watermarksConflated() throws Exception {
    Processor processor = getSupplier(2, (ctx, item) -> new CompletableFuture<>()).get(1).iterator().next();
    processor.init(new TestOutbox(128), new TestProcessorContext());
    TestInbox inbox = new TestInbox();
    // i have to add an item first because otherwise the WMs are forwarded right away
    inbox.add("foo");
    processor.process(0, inbox);
    assertTrue("inbox not empty", inbox.isEmpty());
    assertTrue("wm rejected", processor.tryProcessWatermark(wm(0)));
    assertTrue("wm rejected", processor.tryProcessWatermark(wm(1)));
    assertTrue("wm rejected", processor.tryProcessWatermark(wm(2)));
}

Example 7 with TestInbox

use of com.hazelcast.jet.core.test.TestInbox in project hazelcast by hazelcast.

the class SinksTest method mapWithMerging_when_multipleValuesForSingleKeyInABatch.

@Test
public void mapWithMerging_when_multipleValuesForSingleKeyInABatch() throws Exception {
    ProcessorMetaSupplier metaSupplier = adaptSupplier(SinkProcessors.<Entry<String, Integer>, String, Integer>mergeMapP(sinkName, Entry::getKey, Entry::getValue, Integer::sum));
    TestProcessorSupplierContext psContext = new TestProcessorSupplierContext().setHazelcastInstance(member);
    Processor p = TestSupport.supplierFrom(metaSupplier, psContext).get();
    TestOutbox outbox = new TestOutbox();
    p.init(outbox, new TestProcessorContext().setHazelcastInstance(member));
    TestInbox inbox = new TestInbox();
    inbox.add(entry("k", 1));
    inbox.add(entry("k", 2));
    p.process(0, inbox);
    assertTrue("inbox.isEmpty()", inbox.isEmpty());
    assertTrueEventually(() -> {
        assertTrue("p.complete()", p.complete());
    }, 10);
    p.close();
    // assert the output map contents
    IMap<Object, Object> actual = member.getMap(sinkName);
    assertEquals(1, actual.size());
    assertEquals(3, actual.get("k"));
}

Example 8 with TestInbox

use of com.hazelcast.jet.core.test.TestInbox in project hazelcast by hazelcast.

the class WriteKafkaPTest method when_transactionRolledBackHeuristically_then_sinkIgnoresIt.

@Test
public void when_transactionRolledBackHeuristically_then_sinkIgnoresIt() throws Exception {
    /*
        Design of the test:
        We'll create a processor, process 1 item and do phase-1 of the snapshot and then throw
        it away. Then we'll create a new processor and will try to restore the snapshot. It should
        try to commit the transaction from the previous processor, but that transaction timed out,
        which should be logged and ignored.
         */
    int txnTimeout = 2000;
    properties.setProperty("transaction.timeout.ms", String.valueOf(txnTimeout));
    Processor processor = WriteKafkaP.supplier(properties, o -> new ProducerRecord<>(topic, o), true).get();
    TestOutbox outbox = new TestOutbox(new int[0], 1024);
    TestProcessorContext procContext = new TestProcessorContext().setProcessingGuarantee(ProcessingGuarantee.EXACTLY_ONCE);
    processor.init(outbox, procContext);
    TestInbox inbox = new TestInbox();
    inbox.add("foo");
    processor.process(0, inbox);
    assertEquals("inbox size", 0, inbox.size());
    assertTrue(processor.saveToSnapshot());
    processor.close();
    inbox.addAll(outbox.snapshotQueue());
    // transaction.abort.timed.out.transaction.cleanup.interval.ms is set to 200, allow it to kick in
    sleepMillis(txnTimeout + 1000);
    // create the 2nd processor
    processor = WriteKafkaP.supplier(properties, o -> new ProducerRecord<>(topic, o), true).get();
    processor.init(outbox, procContext);
    processor.restoreFromSnapshot(inbox);
    processor.finishSnapshotRestore();
}

Example 9 with TestInbox

use of com.hazelcast.jet.core.test.TestInbox in project hazelcast by hazelcast.

the class StreamKafkaPTest method saveSnapshot.

private TestInbox saveSnapshot(StreamKafkaP streamKafkaP, TestOutbox outbox) {
    TestInbox snapshot = new TestInbox();
    assertTrue(streamKafkaP.saveToSnapshot());
    outbox.drainSnapshotQueueAndReset(snapshot.queue(), false);
    return snapshot;
}

Example 10 with TestInbox

use of com.hazelcast.jet.core.test.TestInbox in project hazelcast by hazelcast.

the class StreamKafkaPTest method when_snapshotSaved_then_offsetsRestored.

@Test
public void when_snapshotSaved_then_offsetsRestored() throws Exception {
    StreamKafkaP processor = createProcessor(properties(), 2, r -> entry(r.key(), r.value()), 10_000);
    TestOutbox outbox = new TestOutbox(new int[] { 10 }, 10);
    processor.init(outbox, new TestProcessorContext().setProcessingGuarantee(EXACTLY_ONCE));
    kafkaTestSupport.produce(topic1Name, 0, "0");
    assertEquals(entry(0, "0"), consumeEventually(processor, outbox));
    // create snapshot
    TestInbox snapshot = saveSnapshot(processor, outbox);
    Set<Entry<Object, Object>> snapshotItems = unwrapBroadcastKey(snapshot.queue());
    // consume one more item
    kafkaTestSupport.produce(topic1Name, 1, "1");
    assertEquals(entry(1, "1"), consumeEventually(processor, outbox));
    // create new processor and restore snapshot
    processor = createProcessor(properties(), 2, r -> entry(r.key(), r.value()), 10_000);
    outbox = new TestOutbox(new int[] { 10 }, 10);
    processor.init(outbox, new TestProcessorContext().setProcessingGuarantee(EXACTLY_ONCE));
    // restore snapshot
    processor.restoreFromSnapshot(snapshot);
    assertTrue("snapshot not fully processed", snapshot.isEmpty());
    TestInbox snapshot2 = saveSnapshot(processor, outbox);
    assertEquals("new snapshot not equal after restore", snapshotItems, unwrapBroadcastKey(snapshot2.queue()));
    // the second item should be produced one more time
    assertEquals(entry(1, "1"), consumeEventually(processor, outbox));
    assertNoMoreItems(processor, outbox);
}