Examples with TestOutbox com.hazelcast.jet.core.test.TestOutbox used on opensource projects

Example 31 with TestOutbox

use of com.hazelcast.jet.core.test.TestOutbox in project hazelcast by hazelcast.

the class SessionWindowPTest method runBench.

@SuppressWarnings("checkstyle:emptystatement")
private void runBench() throws Exception {
    Random rnd = ThreadLocalRandom.current();
    long start = System.nanoTime();
    long eventCount = 40_000_000;
    long keyCount = 2000;
    long eventsPerKey = eventCount / keyCount;
    int spread = 4000;
    int timestampStep = 20;
    int wmLag = 2000;
    long wmInterval = 100;
    System.out.format("keyCount %,d eventsPerKey %,d wmInterval %,d%n", keyCount, eventsPerKey, wmInterval);
    TestOutbox outbox = new TestOutbox(1024);
    // called for side-effect of assigning to lastSuppliedProcessor
    supplier.get();
    lastSuppliedProcessor.init(outbox, new TestProcessorContext());
    for (long idx = 0; idx < eventsPerKey; idx++) {
        long timestampBase = idx * timestampStep;
        for (long key = (timestampBase / SESSION_TIMEOUT) % 2; key < keyCount; key += 2) {
            while (!lastSuppliedProcessor.tryProcess(0, entry(key, timestampBase + rnd.nextInt(spread)))) {
            }
            while (!lastSuppliedProcessor.tryProcess(0, entry(key, timestampBase + rnd.nextInt(spread)))) {
            }
        }
        if (idx % wmInterval == 0) {
            long wm = timestampBase - wmLag;
            int winCount = 0;
            while (!lastSuppliedProcessor.tryProcessWatermark(new Watermark(wm))) {
                while (outbox.queue(0).poll() != null) {
                    winCount++;
                }
            }
            while (outbox.queue(0).poll() != null) {
                winCount++;
            }
        }
    }
    long took = System.nanoTime() - start;
    System.out.format("%nThroughput %,3d events/second%n", SECONDS.toNanos(1) * eventCount / took);
}

Example 32 with TestOutbox

use of com.hazelcast.jet.core.test.TestOutbox in project hazelcast by hazelcast.

the class SlidingWindowP_twoStageSnapshotTest method test.

@Test
public void test() throws Exception {
    SlidingWindowP stage1p1 = stage1Supplier.get();
    SlidingWindowP stage1p2 = stage1Supplier.get();
    SlidingWindowP stage2p = stage2Supplier.get();
    TestOutbox stage1p1Outbox = newOutbox();
    TestOutbox stage1p2Outbox = newOutbox();
    TestOutbox stage2Outbox = newOutbox();
    TestInbox inbox = new TestInbox();
    TestProcessorContext context = new TestProcessorContext().setProcessingGuarantee(ProcessingGuarantee.EXACTLY_ONCE);
    stage1p1.init(stage1p1Outbox, context);
    stage1p2.init(stage1p2Outbox, context);
    stage2p.init(stage2Outbox, context);
    // process some events in the 1st stage
    // entry key is time
    assertTrue(stage1p1.tryProcess(0, entry(1L, 1L)));
    assertTrue(stage1p2.tryProcess(0, entry(2L, 2L)));
    assertTrue(stage1p1Outbox.queue(0).isEmpty() && stage2Outbox.queue(0).isEmpty());
    // save state in stage1
    assertTrue(stage1p1.saveToSnapshot());
    assertTrue(stage1p2.saveToSnapshot());
    assertTrue("something put to snapshot outbox in stage1", stage1p1Outbox.snapshotQueue().isEmpty() && stage1p2Outbox.snapshotQueue().isEmpty());
    assertEmptyState(stage1p1);
    assertEmptyState(stage1p2);
    // process normal outbox in stage2
    processStage2(stage2p, stage1p1Outbox, stage1p2Outbox, inbox);
    if (simulateRestore) {
        // create new instances for stage1
        stage1p1 = stage1Supplier.get();
        stage1p2 = stage1Supplier.get();
        stage1p1Outbox = newOutbox();
        stage1p2Outbox = newOutbox();
        stage1p1.init(stage1p1Outbox, context);
        stage1p2.init(stage1p2Outbox, context);
    }
    // process some more events in 1st stage
    assertTrue(stage1p1.tryProcess(0, entry(3L, 3L)));
    assertTrue(stage1p1.tryProcess(0, entry(4L, 4L)));
    // process flushing WM
    assertTrue(stage1p1.tryProcessWatermark(wm(10)));
    assertTrue(stage1p2.tryProcessWatermark(wm(10)));
    processStage2(stage2p, stage1p1Outbox, stage1p2Outbox, inbox);
    assertTrue(stage2p.tryProcessWatermark(wm(10)));
    // Then
    assertEquals(collectionToString(asList(outboxFrame(2, 1), outboxFrame(3, 3), outboxFrame(4, 6), outboxFrame(5, 10), outboxFrame(6, 9), outboxFrame(7, 7), outboxFrame(8, 4), wm(10))), collectionToString(stage2Outbox.queue(0)));
}

Example 33 with TestOutbox

use of com.hazelcast.jet.core.test.TestOutbox in project hazelcast by hazelcast.

the class AsyncTransformUsingServicePTest method test_wmNotCountedToParallelOps.

@Test
public void test_wmNotCountedToParallelOps() throws Exception {
    Processor processor = getSupplier(2, (ctx, item) -> new CompletableFuture<>()).get(1).iterator().next();
    processor.init(new TestOutbox(128), new TestProcessorContext());
    TestInbox inbox = new TestInbox();
    inbox.add("foo");
    processor.process(0, inbox);
    assertTrue("inbox not empty", inbox.isEmpty());
    assertTrue("wm rejected", processor.tryProcessWatermark(wm(0)));
    inbox.add("bar");
    processor.process(0, inbox);
    assertTrue("2nd item rejected even though max parallel ops is 1", inbox.isEmpty());
}

Example 34 with TestOutbox

use of com.hazelcast.jet.core.test.TestOutbox in project hazelcast by hazelcast.

the class InsertWatermarksPTest method setUp.

@Before
public void setUp() {
    outbox = new TestOutbox(outboxCapacity);
    context = new TestProcessorContext();
}

Example 35 with TestOutbox

use of com.hazelcast.jet.core.test.TestOutbox in project hazelcast by hazelcast.

the class AbstractProcessorTest method before.

@Before
public void before() throws Exception {
    inbox = new TestInbox();
    inbox.add(MOCK_ITEM);
    int[] capacities = new int[OUTBOX_BUCKET_COUNT];
    Arrays.fill(capacities, 1);
    outbox = new TestOutbox(capacities);
    final Processor.Context ctx = new TestProcessorContext();
    p = new RegisteringMethodCallsP();
    p.init(outbox, ctx);
    tryProcessP = new SpecializedByOrdinalP();
    tryProcessP.init(outbox, ctx);
    nothingOverriddenP = new NothingOverriddenP();
    nothingOverriddenP.init(outbox, ctx);
}