Examples with Processor com.hazelcast.jet.core.Processor used on opensource projects

Example 26 with Processor

use of com.hazelcast.jet.core.Processor in project hazelcast by hazelcast.

the class TestSupport method runTest.

private void runTest(TestMode testMode) throws Exception {
    beforeEachRun.run();
    assert testMode.isSnapshotsEnabled() || testMode.snapshotRestoreInterval() == 0 : "Illegal combination: don't do snapshots, but do restore";
    boolean doSnapshots = testMode.doSnapshots;
    int doRestoreEvery = testMode.restoreInterval;
    IdleStrategy idler = new BackoffIdleStrategy(0, 0, MICROSECONDS.toNanos(1), MILLISECONDS.toNanos(1));
    int idleCount = 0;
    System.out.println("### Running the test, mode=" + testMode.toString());
    TestInbox inbox = new TestInbox();
    int inboxOrdinal = -1;
    Processor[] processor = { newProcessorFromSupplier() };
    boolean isCooperative = processor[0].isCooperative();
    // we'll use 1-capacity outbox to test outbox rejection
    TestOutbox[] outbox = { createOutbox() };
    List<List<Object>> actualOutputs = new ArrayList<>(outputOrdinalCount);
    for (int i = 0; i < outputOrdinalCount; i++) {
        actualOutputs.add(new ArrayList<>());
    }
    // create instance of your processor and call the init() method
    initProcessor(processor[0], outbox[0]);
    int[] restoreCount = { 0 };
    // do snapshot+restore before processing any item. This will test saveToSnapshot() in this edge case
    snapshotAndRestore(processor, outbox, actualOutputs, doSnapshots, doRestoreEvery, restoreCount);
    // call the process() method
    List<ObjectWithOrdinal> input = mixInputs(inputs, priorities);
    int inputPosition = 0;
    while (inputPosition < input.size() || !inbox.isEmpty()) {
        if (inbox.isEmpty() && inputPosition < input.size()) {
            inboxOrdinal = input.get(inputPosition).ordinal;
            for (int added = 0; inputPosition < input.size() && added < testMode.inboxLimit() && inboxOrdinal == input.get(inputPosition).ordinal && (added == 0 || !(input.get(inputPosition).item instanceof Watermark)); added++) {
                ObjectWithOrdinal objectWithOrdinal = input.get(inputPosition++);
                inbox.queue().add(objectWithOrdinal.item);
                inboxOrdinal = objectWithOrdinal.ordinal;
            }
            if (logInputOutput) {
                System.out.println(LocalTime.now() + " Input-" + inboxOrdinal + ": " + inbox);
            }
        }
        int lastInboxSize = inbox.size();
        String methodName;
        methodName = processInbox(inbox, inboxOrdinal, isCooperative, processor);
        boolean madeProgress = inbox.size() < lastInboxSize || (outbox[0].bucketCount() > 0 && !outbox[0].queue(0).isEmpty());
        assertTrue(methodName + "() call without progress", !assertProgress || madeProgress);
        idleCount = idle(idler, idleCount, madeProgress);
        if (outbox[0].bucketCount() > 0 && outbox[0].queue(0).size() == 1 && !inbox.isEmpty()) {
            // if the outbox is full, call the process() method again. Cooperative
            // processor must be able to cope with this situation and not try to put
            // more items to the outbox.
            outbox[0].reset();
            processInbox(inbox, inboxOrdinal, isCooperative, processor);
        }
        outbox[0].drainQueuesAndReset(actualOutputs, logInputOutput);
        if (inbox.isEmpty()) {
            snapshotAndRestore(processor, outbox, actualOutputs, doSnapshots, doRestoreEvery, restoreCount);
        }
    }
    if (logInputOutput && !inputs.isEmpty()) {
        System.out.println(LocalTime.now() + " Input processed, calling complete()");
    }
    // call the complete() method
    if (callComplete) {
        long completeStart = System.nanoTime();
        long outputMatchedAt = Long.MAX_VALUE;
        boolean[] done = { false };
        do {
            doCall("complete", isCooperative, () -> done[0] = processor[0].complete());
            boolean madeProgress = done[0] || (outbox[0].bucketCount() > 0 && !outbox[0].queue(0).isEmpty());
            assertTrue("complete() call without progress", !assertProgress || madeProgress);
            outbox[0].drainQueuesAndReset(actualOutputs, logInputOutput);
            if (outbox[0].hasUnfinishedItem()) {
                assertFalse("outbox has unfinished items, but complete() claims to be done", done[0]);
                outbox[0].block();
            } else {
                outbox[0].unblock();
                snapshotAndRestore(processor, outbox, actualOutputs, madeProgress && doSnapshots && !done[0], doRestoreEvery, restoreCount);
            }
            idleCount = idle(idler, idleCount, madeProgress);
            long now = System.nanoTime();
            if (runUntilOutputMatchesTimeoutMillis >= 0) {
                try {
                    assertOutputFn.accept(testMode, actualOutputs);
                    outputMatchedAt = Math.min(outputMatchedAt, now);
                } catch (AssertionError e) {
                    if (outputMatchedAt < Long.MAX_VALUE) {
                        throw new AssertionError("the output already matched, but doesn't match now", e);
                    }
                // ignore the failure otherwise and continue calling complete()
                }
                long elapsedSinceStart = NANOSECONDS.toMillis(now - completeStart);
                long elapsedSinceMatch = NANOSECONDS.toMillis(subtractClamped(now, outputMatchedAt));
                if (elapsedSinceStart > runUntilOutputMatchesTimeoutMillis || elapsedSinceMatch > runUntilOutputMatchesExtraTimeMillis) {
                    break;
                }
            }
        } while (!done[0]);
        assertTrue("complete returned true in a run-until-output-matches mode", !done[0] || runUntilOutputMatchesTimeoutMillis <= 0);
    }
    processor[0].close();
    assertOutputFn.accept(testMode, actualOutputs);
}

Example 27 with Processor

use of com.hazelcast.jet.core.Processor in project hazelcast by hazelcast.

the class WriteKafkaPTest method when_transactionRolledBackHeuristically_then_sinkIgnoresIt.

@Test
public void when_transactionRolledBackHeuristically_then_sinkIgnoresIt() throws Exception {
    /*
        Design of the test:
        We'll create a processor, process 1 item and do phase-1 of the snapshot and then throw
        it away. Then we'll create a new processor and will try to restore the snapshot. It should
        try to commit the transaction from the previous processor, but that transaction timed out,
        which should be logged and ignored.
         */
    int txnTimeout = 2000;
    properties.setProperty("transaction.timeout.ms", String.valueOf(txnTimeout));
    Processor processor = WriteKafkaP.supplier(properties, o -> new ProducerRecord<>(topic, o), true).get();
    TestOutbox outbox = new TestOutbox(new int[0], 1024);
    TestProcessorContext procContext = new TestProcessorContext().setProcessingGuarantee(ProcessingGuarantee.EXACTLY_ONCE);
    processor.init(outbox, procContext);
    TestInbox inbox = new TestInbox();
    inbox.add("foo");
    processor.process(0, inbox);
    assertEquals("inbox size", 0, inbox.size());
    assertTrue(processor.saveToSnapshot());
    processor.close();
    inbox.addAll(outbox.snapshotQueue());
    // transaction.abort.timed.out.transaction.cleanup.interval.ms is set to 200, allow it to kick in
    sleepMillis(txnTimeout + 1000);
    // create the 2nd processor
    processor = WriteKafkaP.supplier(properties, o -> new ProducerRecord<>(topic, o), true).get();
    processor.init(outbox, procContext);
    processor.restoreFromSnapshot(inbox);
    processor.finishSnapshotRestore();
}

Example 28 with Processor

use of com.hazelcast.jet.core.Processor in project hazelcast by hazelcast.

the class StreamKafkaPTest method when_snapshotSaved_then_offsetsRestored.

@Test
public void when_snapshotSaved_then_offsetsRestored() throws Exception {
    StreamKafkaP processor = createProcessor(properties(), 2, r -> entry(r.key(), r.value()), 10_000);
    TestOutbox outbox = new TestOutbox(new int[] { 10 }, 10);
    processor.init(outbox, new TestProcessorContext().setProcessingGuarantee(EXACTLY_ONCE));
    kafkaTestSupport.produce(topic1Name, 0, "0");
    assertEquals(entry(0, "0"), consumeEventually(processor, outbox));
    // create snapshot
    TestInbox snapshot = saveSnapshot(processor, outbox);
    Set<Entry<Object, Object>> snapshotItems = unwrapBroadcastKey(snapshot.queue());
    // consume one more item
    kafkaTestSupport.produce(topic1Name, 1, "1");
    assertEquals(entry(1, "1"), consumeEventually(processor, outbox));
    // create new processor and restore snapshot
    processor = createProcessor(properties(), 2, r -> entry(r.key(), r.value()), 10_000);
    outbox = new TestOutbox(new int[] { 10 }, 10);
    processor.init(outbox, new TestProcessorContext().setProcessingGuarantee(EXACTLY_ONCE));
    // restore snapshot
    processor.restoreFromSnapshot(snapshot);
    assertTrue("snapshot not fully processed", snapshot.isEmpty());
    TestInbox snapshot2 = saveSnapshot(processor, outbox);
    assertEquals("new snapshot not equal after restore", snapshotItems, unwrapBroadcastKey(snapshot2.queue()));
    // the second item should be produced one more time
    assertEquals(entry(1, "1"), consumeEventually(processor, outbox));
    assertNoMoreItems(processor, outbox);
}

Example 29 with Processor

use of com.hazelcast.jet.core.Processor in project hazelcast by hazelcast.

the class JoinByPrimitiveKeyProcessorSupplier method get.

@Nonnull
@Override
public Collection<? extends Processor> get(int count) {
    List<Processor> processors = new ArrayList<>(count);
    for (int i = 0; i < count; i++) {
        String mapName = this.mapName;
        KvRowProjector projector = rightRowProjectorSupplier.get(evalContext, extractors);
        Processor processor = new AsyncTransformUsingServiceOrderedP<>(ServiceFactories.nonSharedService(SecuredFunctions.iMapFn(mapName)), null, MAX_CONCURRENT_OPS, (IMap<Object, Object> map, JetSqlRow left) -> {
            Object key = left.get(leftEquiJoinIndex);
            if (key == null) {
                return inner ? null : completedFuture(null);
            }
            return map.getAsync(key).toCompletableFuture();
        }, (left, value) -> {
            JetSqlRow joined = join(left, left.get(leftEquiJoinIndex), value, projector, condition, evalContext);
            return joined != null ? singleton(joined) : inner ? null : singleton(left.extendedRow(projector.getColumnCount()));
        });
        processors.add(processor);
    }
    return processors;
}

Example 30 with Processor

use of com.hazelcast.jet.core.Processor in project hazelcast by hazelcast.

the class RowProjectorProcessorSupplier method get.

@Nonnull
@Override
public Collection<? extends Processor> get(int count) {
    List<Processor> processors = new ArrayList<>(count);
    for (int i = 0; i < count; i++) {
        ResettableSingletonTraverser<JetSqlRow> traverser = new ResettableSingletonTraverser<>();
        KvRowProjector projector = projectorSupplier.get(evalContext, extractors);
        Processor processor = new TransformP<LazyMapEntry<Object, Object>, JetSqlRow>(entry -> {
            traverser.accept(projector.project(entry.getKeyData(), entry.getValueData()));
            return traverser;
        });
        processors.add(processor);
    }
    return processors;
}