use of com.hazelcast.jet.core.test.TestInbox in project hazelcast by hazelcast.
the class AsyncTransformUsingServicePTest method test_watermarksConflated.
@Test
public void test_watermarksConflated() throws Exception {
Processor processor = getSupplier(2, (ctx, item) -> new CompletableFuture<>()).get(1).iterator().next();
processor.init(new TestOutbox(128), new TestProcessorContext());
TestInbox inbox = new TestInbox();
// i have to add an item first because otherwise the WMs are forwarded right away
inbox.add("foo");
processor.process(0, inbox);
assertTrue("inbox not empty", inbox.isEmpty());
assertTrue("wm rejected", processor.tryProcessWatermark(wm(0)));
assertTrue("wm rejected", processor.tryProcessWatermark(wm(1)));
assertTrue("wm rejected", processor.tryProcessWatermark(wm(2)));
}
use of com.hazelcast.jet.core.test.TestInbox in project hazelcast by hazelcast.
the class SinksTest method mapWithMerging_when_multipleValuesForSingleKeyInABatch.
@Test
public void mapWithMerging_when_multipleValuesForSingleKeyInABatch() throws Exception {
ProcessorMetaSupplier metaSupplier = adaptSupplier(SinkProcessors.<Entry<String, Integer>, String, Integer>mergeMapP(sinkName, Entry::getKey, Entry::getValue, Integer::sum));
TestProcessorSupplierContext psContext = new TestProcessorSupplierContext().setHazelcastInstance(member);
Processor p = TestSupport.supplierFrom(metaSupplier, psContext).get();
TestOutbox outbox = new TestOutbox();
p.init(outbox, new TestProcessorContext().setHazelcastInstance(member));
TestInbox inbox = new TestInbox();
inbox.add(entry("k", 1));
inbox.add(entry("k", 2));
p.process(0, inbox);
assertTrue("inbox.isEmpty()", inbox.isEmpty());
assertTrueEventually(() -> {
assertTrue("p.complete()", p.complete());
}, 10);
p.close();
// assert the output map contents
IMap<Object, Object> actual = member.getMap(sinkName);
assertEquals(1, actual.size());
assertEquals(3, actual.get("k"));
}
use of com.hazelcast.jet.core.test.TestInbox in project hazelcast by hazelcast.
the class WriteKafkaPTest method when_transactionRolledBackHeuristically_then_sinkIgnoresIt.
@Test
public void when_transactionRolledBackHeuristically_then_sinkIgnoresIt() throws Exception {
/*
Design of the test:
We'll create a processor, process 1 item and do phase-1 of the snapshot and then throw
it away. Then we'll create a new processor and will try to restore the snapshot. It should
try to commit the transaction from the previous processor, but that transaction timed out,
which should be logged and ignored.
*/
int txnTimeout = 2000;
properties.setProperty("transaction.timeout.ms", String.valueOf(txnTimeout));
Processor processor = WriteKafkaP.supplier(properties, o -> new ProducerRecord<>(topic, o), true).get();
TestOutbox outbox = new TestOutbox(new int[0], 1024);
TestProcessorContext procContext = new TestProcessorContext().setProcessingGuarantee(ProcessingGuarantee.EXACTLY_ONCE);
processor.init(outbox, procContext);
TestInbox inbox = new TestInbox();
inbox.add("foo");
processor.process(0, inbox);
assertEquals("inbox size", 0, inbox.size());
assertTrue(processor.saveToSnapshot());
processor.close();
inbox.addAll(outbox.snapshotQueue());
// transaction.abort.timed.out.transaction.cleanup.interval.ms is set to 200, allow it to kick in
sleepMillis(txnTimeout + 1000);
// create the 2nd processor
processor = WriteKafkaP.supplier(properties, o -> new ProducerRecord<>(topic, o), true).get();
processor.init(outbox, procContext);
processor.restoreFromSnapshot(inbox);
processor.finishSnapshotRestore();
}
use of com.hazelcast.jet.core.test.TestInbox in project hazelcast by hazelcast.
the class StreamKafkaPTest method saveSnapshot.
private TestInbox saveSnapshot(StreamKafkaP streamKafkaP, TestOutbox outbox) {
TestInbox snapshot = new TestInbox();
assertTrue(streamKafkaP.saveToSnapshot());
outbox.drainSnapshotQueueAndReset(snapshot.queue(), false);
return snapshot;
}
use of com.hazelcast.jet.core.test.TestInbox in project hazelcast by hazelcast.
the class StreamKafkaPTest method when_snapshotSaved_then_offsetsRestored.
@Test
public void when_snapshotSaved_then_offsetsRestored() throws Exception {
StreamKafkaP processor = createProcessor(properties(), 2, r -> entry(r.key(), r.value()), 10_000);
TestOutbox outbox = new TestOutbox(new int[] { 10 }, 10);
processor.init(outbox, new TestProcessorContext().setProcessingGuarantee(EXACTLY_ONCE));
kafkaTestSupport.produce(topic1Name, 0, "0");
assertEquals(entry(0, "0"), consumeEventually(processor, outbox));
// create snapshot
TestInbox snapshot = saveSnapshot(processor, outbox);
Set<Entry<Object, Object>> snapshotItems = unwrapBroadcastKey(snapshot.queue());
// consume one more item
kafkaTestSupport.produce(topic1Name, 1, "1");
assertEquals(entry(1, "1"), consumeEventually(processor, outbox));
// create new processor and restore snapshot
processor = createProcessor(properties(), 2, r -> entry(r.key(), r.value()), 10_000);
outbox = new TestOutbox(new int[] { 10 }, 10);
processor.init(outbox, new TestProcessorContext().setProcessingGuarantee(EXACTLY_ONCE));
// restore snapshot
processor.restoreFromSnapshot(snapshot);
assertTrue("snapshot not fully processed", snapshot.isEmpty());
TestInbox snapshot2 = saveSnapshot(processor, outbox);
assertEquals("new snapshot not equal after restore", snapshotItems, unwrapBroadcastKey(snapshot2.queue()));
// the second item should be produced one more time
assertEquals(entry(1, "1"), consumeEventually(processor, outbox));
assertNoMoreItems(processor, outbox);
}
Aggregations