Examples with Processor com.hazelcast.jet.core.Processor used on opensource projects

Example 1 with Processor

use of com.hazelcast.jet.core.Processor in project hazelcast-jet by hazelcast.

the class SlidingWindowPTest method before.

@Before
public void before() {
    SlidingWindowPolicy windowDef = slidingWinPolicy(4, 1);
    AggregateOperation1<Entry<?, Long>, LongAccumulator, Long> operation = AggregateOperation.withCreate(LongAccumulator::new).andAccumulate((LongAccumulator acc, Entry<?, Long> item) -> acc.add(item.getValue())).andCombine(LongAccumulator::add).andDeduct(hasDeduct ? LongAccumulator::subtract : null).andFinish(LongAccumulator::get);
    DistributedFunction<?, Long> keyFn = t -> KEY;
    DistributedToLongFunction<Entry<Long, Long>> timestampFn = Entry::getKey;
    DistributedSupplier<Processor> procSupplier = singleStageProcessor ? aggregateToSlidingWindowP(singletonList(keyFn), singletonList(timestampFn), TimestampKind.EVENT, windowDef, operation, TimestampedEntry::new) : combineToSlidingWindowP(windowDef, operation, TimestampedEntry::new);
    // new supplier to save the last supplied instance
    supplier = () -> lastSuppliedProcessor = (SlidingWindowP) procSupplier.get();
}

Example 2 with Processor

use of com.hazelcast.jet.core.Processor in project hazelcast-jet by hazelcast.

the class SlidingWindowP_twoStageSnapshotTest method before.

@Before
public void before() {
    SlidingWindowPolicy windowDef = slidingWinPolicy(4, 1);
    AggregateOperation1<Entry<?, Long>, LongAccumulator, Long> aggrOp = AggregateOperation.withCreate(LongAccumulator::new).andAccumulate((LongAccumulator acc, Entry<?, Long> item) -> acc.add(item.getValue())).andCombine(LongAccumulator::add).andDeduct(LongAccumulator::subtract).andFinish(LongAccumulator::get);
    DistributedSupplier<Processor> procSupplier1 = Processors.accumulateByFrameP(singletonList((DistributedFunction<? super Entry<Long, Long>, ?>) t -> KEY), singletonList((DistributedToLongFunction<? super Entry<Long, Long>>) Entry::getKey), TimestampKind.EVENT, windowDef, ((AggregateOperation1<? super Entry<Long, Long>, LongAccumulator, ?>) aggrOp).withFinishFn(identity()));
    DistributedSupplier<Processor> procSupplier2 = combineToSlidingWindowP(windowDef, aggrOp, TimestampedEntry::new);
    // new supplier to save the last supplied instance
    stage1Supplier = () -> lastSuppliedStage1Processor = (SlidingWindowP<?, ?, ?, ?>) procSupplier1.get();
    stage2Supplier = () -> lastSuppliedStage2Processor = (SlidingWindowP<?, ?, ?, ?>) procSupplier2.get();
}

Example 3 with Processor

use of com.hazelcast.jet.core.Processor in project hazelcast-jet by hazelcast.

the class ExecutionContext method completeExecution.

/**
 * Complete local execution. If local execution was started, it should be
 * called after execution has completed.
 */
public void completeExecution(Throwable error) {
    assert executionFuture == null || executionFuture.isDone() : "If execution was begun, then completeExecution() should not be called before execution is done.";
    for (Processor processor : processors) {
        try {
            processor.close(error);
        } catch (Throwable e) {
            logger.severe(jobAndExecutionId(jobId, executionId) + " encountered an exception in Processor.close(), ignoring it", e);
        }
    }
    for (ProcessorSupplier s : procSuppliers) {
        try {
            s.close(error);
        } catch (Throwable e) {
            logger.severe(jobAndExecutionId(jobId, executionId) + " encountered an exception in ProcessorSupplier.complete(), ignoring it", e);
        }
    }
    MetricsRegistry metricsRegistry = ((NodeEngineImpl) nodeEngine).getMetricsRegistry();
    processors.forEach(metricsRegistry::deregister);
}

Example 4 with Processor

use of com.hazelcast.jet.core.Processor in project hazelcast-jet by hazelcast.

the class TestSupport method runTest.

private void runTest(boolean doSnapshots, int doRestoreEvery) throws Exception {
    assert doSnapshots || doRestoreEvery == 0 : "Illegal combination: don't do snapshots, but do restore";
    IdleStrategy idler = new BackoffIdleStrategy(0, 0, MICROSECONDS.toNanos(1), MILLISECONDS.toNanos(1));
    int idleCount = 0;
    if (doSnapshots && doRestoreEvery == 1) {
        // we do all 3 possible combinations: no snapshot, only snapshots and snapshots+restore
        runTest(false, 0);
        runTest(true, Integer.MAX_VALUE);
        runTest(true, 2);
    }
    System.out.println("### Running the test, mode=" + modeDescription(doSnapshots, doRestoreEvery));
    TestInbox inbox = new TestInbox();
    int inboxOrdinal = -1;
    Processor[] processor = { newProcessorFromSupplier() };
    boolean isCooperative = processor[0].isCooperative();
    // we'll use 1-capacity outbox to test outbox rejection
    TestOutbox[] outbox = { createOutbox() };
    List<List<Object>> actualOutputs = new ArrayList<>(expectedOutputs.size());
    for (int i = 0; i < expectedOutputs.size(); i++) {
        actualOutputs.add(new ArrayList());
    }
    // create instance of your processor and call the init() method
    initProcessor(processor[0], outbox[0]);
    int[] restoreCount = { 0 };
    // do snapshot+restore before processing any item. This will test saveToSnapshot() in this edge case
    snapshotAndRestore(processor, outbox, actualOutputs, doSnapshots, doRestoreEvery, restoreCount);
    // call the process() method
    List<ObjectWithOrdinal> input = mixInputs(inputs, priorities);
    Iterator<ObjectWithOrdinal> inputIterator = input.iterator();
    Watermark[] wmToProcess = { null };
    while (inputIterator.hasNext() || !inbox.isEmpty() || wmToProcess[0] != null) {
        if (inbox.isEmpty() && wmToProcess[0] == null && inputIterator.hasNext()) {
            ObjectWithOrdinal objectWithOrdinal = inputIterator.next();
            inbox.queue().add(objectWithOrdinal.item);
            inboxOrdinal = objectWithOrdinal.ordinal;
            if (logInputOutput) {
                System.out.println(LocalTime.now() + " Input-" + objectWithOrdinal.ordinal + ": " + inbox.peek());
            }
        }
        String methodName;
        if (wmToProcess[0] != null) {
            methodName = "offer";
            if (outbox[0].offer(wmToProcess[0])) {
                wmToProcess[0] = null;
            }
        } else {
            methodName = processInbox(inbox, inboxOrdinal, isCooperative, processor, wmToProcess);
        }
        boolean madeProgress = inbox.isEmpty() || !outbox[0].queue(0).isEmpty();
        assertTrue(methodName + "() call without progress", !assertProgress || madeProgress);
        idleCount = idle(idler, idleCount, madeProgress);
        if (outbox[0].queue(0).size() == 1 && !inbox.isEmpty()) {
            // if the outbox is full, call the process() method again. Cooperative
            // processor must be able to cope with this situation and not try to put
            // more items to the outbox.
            outbox[0].reset();
            processInbox(inbox, inboxOrdinal, isCooperative, processor, wmToProcess);
        }
        outbox[0].drainQueuesAndReset(actualOutputs, logInputOutput);
        if (inbox.isEmpty() && wmToProcess[0] == null) {
            snapshotAndRestore(processor, outbox, actualOutputs, doSnapshots, doRestoreEvery, restoreCount);
        }
    }
    if (logInputOutput && !inputs.isEmpty()) {
        System.out.println(LocalTime.now() + " Input processed, calling complete()");
    }
    // call the complete() method
    if (callComplete) {
        long completeStart = System.nanoTime();
        boolean[] done = { false };
        double elapsed;
        do {
            checkTime("complete", isCooperative, () -> done[0] = processor[0].complete());
            boolean madeProgress = done[0] || !outbox[0].queue(0).isEmpty();
            assertTrue("complete() call without progress", !assertProgress || madeProgress);
            outbox[0].drainQueuesAndReset(actualOutputs, logInputOutput);
            snapshotAndRestore(processor, outbox, actualOutputs, madeProgress && doSnapshots && !done[0], doRestoreEvery, restoreCount);
            idleCount = idle(idler, idleCount, madeProgress);
            if (runUntilCompletedTimeout > 0) {
                elapsed = toMillis(System.nanoTime() - completeStart);
                if (elapsed > runUntilCompletedTimeout) {
                    break;
                }
            }
        } while (!done[0]);
        assertTrue("complete returned true", !done[0] || runUntilCompletedTimeout <= 0);
    }
    processor[0].close(null);
    // assert the outbox
    for (int i = 0; i < expectedOutputs.size(); i++) {
        List<?> expectedOutput = expectedOutputs.get(i);
        List<?> actualOutput = actualOutputs.get(i);
        if (!outputChecker.test(expectedOutput, actualOutput)) {
            assertEquals("processor output in mode \"" + modeDescription(doSnapshots, doRestoreEvery) + "\" doesn't match", listToString(expectedOutput), listToString(actualOutput));
        }
    }
}

Example 5 with Processor

use of com.hazelcast.jet.core.Processor in project hazelcast-jet by hazelcast.

the class StreamSocketPTest method runTest.

private void runTest(byte[] inputBytes, int inputSplitAfter) throws Exception {
    try (ServerSocket serverSocket = new ServerSocket(0)) {
        Thread thread = new Thread(() -> uncheckRun(() -> {
            Socket socket = serverSocket.accept();
            OutputStream outputStream = socket.getOutputStream();
            outputStream.write(inputBytes, 0, inputSplitAfter);
            Thread.sleep(300);
            outputStream.write(inputBytes, inputSplitAfter, inputBytes.length - inputSplitAfter);
            outputStream.close();
            socket.close();
        }));
        thread.start();
        Processor processor = supplierFrom(streamSocketP("localhost", serverSocket.getLocalPort(), UTF_16)).get();
        processor.init(outbox, context);
        assertTrueEventually(() -> assertTrue(processor.complete()), 3);
        for (String s : output) {
            assertEquals(s, bucket.poll());
        }
        assertEquals(null, bucket.poll());
        assertTrueEventually(() -> assertFalse(thread.isAlive()));
    }
}