Examples with Processor com.hazelcast.jet.core.Processor used on opensource projects

Example 21 with Processor

use of com.hazelcast.jet.core.Processor in project hazelcast by hazelcast.

the class StreamEventJournalPTest method when_lostItems_afterRestore.

@Test
public void when_lostItems_afterRestore() throws Exception {
    TestOutbox outbox = new TestOutbox(new int[] { 16 }, 16);
    final Processor p = supplier.get();
    p.init(outbox, new TestProcessorContext().setHazelcastInstance(instance));
    List<Object> output = new ArrayList<>();
    assertTrueEventually(() -> {
        assertFalse("Processor should never complete", p.complete());
        outbox.drainQueueAndReset(0, output, true);
        assertTrue("consumed different number of items than expected", output.size() == 0);
    }, 3);
    assertTrueEventually(() -> {
        assertTrue("Processor did not finish snapshot", p.saveToSnapshot());
    }, 3);
    // overflow journal
    fillJournal(CAPACITY_PER_PARTITION + 1);
    List<Entry> snapshotItems = new ArrayList<>();
    outbox.drainSnapshotQueueAndReset(snapshotItems, false);
    logger.info("Restoring journal");
    // restore from snapshot
    assertRestore(snapshotItems);
}

Example 22 with Processor

use of com.hazelcast.jet.core.Processor in project hazelcast by hazelcast.

the class SlidingWindowPTest method before.

@Before
public void before() {
    SlidingWindowPolicy winPolicy = slidingWinPolicy(4, 1);
    AggregateOperation1<Entry<?, Long>, LongAccumulator, Long> operation = AggregateOperation.withCreate(LongAccumulator::new).andAccumulate((LongAccumulator acc, Entry<?, Long> item) -> acc.add(item.getValue())).andCombine(LongAccumulator::add).andDeduct(hasDeduct ? LongAccumulator::subtract : null).andExportFinish(LongAccumulator::get);
    FunctionEx<?, Long> keyFn = t -> KEY;
    ToLongFunctionEx<Entry<Long, Long>> timestampFn = Entry::getKey;
    SupplierEx<Processor> procSupplier = singleStageProcessor ? aggregateToSlidingWindowP(singletonList(keyFn), singletonList(timestampFn), TimestampKind.EVENT, winPolicy, 0L, operation, KeyedWindowResult::new) : combineToSlidingWindowP(winPolicy, operation, KeyedWindowResult::new);
    // new supplier to save the last supplied instance
    supplier = () -> lastSuppliedProcessor = (SlidingWindowP) procSupplier.get();
}

Example 23 with Processor

use of com.hazelcast.jet.core.Processor in project hazelcast by hazelcast.

the class SlidingWindowP_twoStageSnapshotTest method before.

@Before
public void before() {
    SlidingWindowPolicy windowDef = slidingWinPolicy(4, 1);
    AggregateOperation1<Entry<?, Long>, LongAccumulator, Long> aggrOp = AggregateOperation.withCreate(LongAccumulator::new).andAccumulate((LongAccumulator acc, Entry<?, Long> item) -> acc.add(item.getValue())).andCombine(LongAccumulator::add).andDeduct(LongAccumulator::subtract).andExportFinish(LongAccumulator::get);
    SupplierEx<Processor> procSupplier1 = Processors.accumulateByFrameP(singletonList((FunctionEx<? super Entry<Long, Long>, ?>) t -> KEY), singletonList((ToLongFunctionEx<? super Entry<Long, Long>>) Entry::getKey), TimestampKind.EVENT, windowDef, aggrOp.withIdentityFinish());
    SupplierEx<Processor> procSupplier2 = combineToSlidingWindowP(windowDef, aggrOp, KeyedWindowResult::new);
    // new supplier to save the last supplied instance
    stage1Supplier = () -> lastSuppliedStage1Processor = (SlidingWindowP<?, ?, ?, ?>) procSupplier1.get();
    stage2Supplier = () -> lastSuppliedStage2Processor = (SlidingWindowP<?, ?, ?, ?>) procSupplier2.get();
}

Example 24 with Processor

use of com.hazelcast.jet.core.Processor in project hazelcast by hazelcast.

the class CoAggregateOperationBuilder method build.

/**
 * Builds and returns the multi-input {@link AggregateOperation}. It will
 * call the supplied {@code exportFinishFn} to transform the {@link ItemsByTag}
 * it creates to the result type it emits as the actual result.
 *
 * @param exportFinishFn function to convert {@link ItemsByTag} to the
 *     target result type. It must be stateless and {@linkplain
 *     Processor#isCooperative() cooperative}.
 */
@Nonnull
@SuppressWarnings({ "unchecked", "ConstantConditions" })
public <R> AggregateOperation<Object[], R> build(@Nonnull FunctionEx<? super ItemsByTag, ? extends R> exportFinishFn) {
    checkSerializable(exportFinishFn, "exportFinishFn");
    Tag[] tags = opsByTag.keySet().stream().sorted().toArray(Tag[]::new);
    for (int i = 0; i < tags.length; i++) {
        Preconditions.checkTrue(tags[i].index() == i, "Registered tags' indices are " + stream(tags).map(Tag::index).collect(toList()) + ", but should be " + range(0, tags.length).boxed().collect(toList()));
    }
    // Variable `sorted` extracted due to type inference failure
    Stream<Entry<Tag, AggregateOperation1>> sorted = opsByTag.entrySet().stream().sorted(comparing(Entry::getKey));
    List<AggregateOperation1> ops = sorted.map(Entry::getValue).collect(toList());
    BiConsumerEx[] combineFns = ops.stream().map(AggregateOperation::combineFn).toArray(BiConsumerEx[]::new);
    BiConsumerEx[] deductFns = ops.stream().map(AggregateOperation::deductFn).toArray(BiConsumerEx[]::new);
    FunctionEx[] exportFns = ops.stream().map(AggregateOperation::exportFn).toArray(FunctionEx[]::new);
    FunctionEx[] finishFns = ops.stream().map(AggregateOperation::finishFn).toArray(FunctionEx[]::new);
    AggregateOperationBuilder.VarArity<Object[], Void> b = AggregateOperation.withCreate(() -> ops.stream().map(op -> op.createFn().get()).toArray()).varArity();
    opsByTag.forEach((tag, op) -> {
        int index = tag.index();
        b.andAccumulate(tag, (acc, item) -> op.accumulateFn().accept(acc[index], item));
    });
    return b.andCombine(stream(combineFns).anyMatch(Objects::isNull) ? null : (acc1, acc2) -> {
        for (int i = 0; i < combineFns.length; i++) {
            combineFns[i].accept(acc1[i], acc2[i]);
        }
    }).andDeduct(stream(deductFns).anyMatch(Objects::isNull) ? null : (acc1, acc2) -> {
        for (int i = 0; i < deductFns.length; i++) {
            deductFns[i].accept(acc1[i], acc2[i]);
        }
    }).<R>andExport(acc -> {
        ItemsByTag result = new ItemsByTag();
        for (int i = 0; i < exportFns.length; i++) {
            result.put(tags[i], exportFns[i].apply(acc[i]));
        }
        return exportFinishFn.apply(result);
    }).andFinish(acc -> {
        ItemsByTag result = new ItemsByTag();
        for (int i = 0; i < finishFns.length; i++) {
            result.put(tags[i], finishFns[i].apply(acc[i]));
        }
        return exportFinishFn.apply(result);
    });
}

Example 25 with Processor

use of com.hazelcast.jet.core.Processor in project hazelcast by hazelcast.

the class AllOfAggregationBuilder method build.

/**
 * Builds and returns the composite {@link AggregateOperation1}. It will
 * call the supplied {@code exportFinishFn} to transform the {@link ItemsByTag}
 * it creates to the result type it emits as the actual result.
 *
 * @param exportFinishFn function that converts the {@link ItemsByTag} to
 *     the target result type. It must be stateless and {@linkplain
 *     Processor#isCooperative() cooperative}.
 */
@Nonnull
@SuppressWarnings({ "unchecked", "ConstantConditions" })
public <R> AggregateOperation1<T, Object[], R> build(@Nonnull FunctionEx<ItemsByTag, R> exportFinishFn) {
    checkSerializable(exportFinishFn, "exportFinishFn");
    // Avoid capturing this builder in the lambdas:
    List<Tag> tags = this.tags;
    List<AggregateOperation1> operations = this.operations;
    return (AggregateOperation1<T, Object[], R>) AggregateOperation.withCreate(() -> {
        Object[] acc = new Object[tags.size()];
        Arrays.setAll(acc, i -> operations.get(i).createFn().get());
        return acc;
    }).andAccumulate((acc, item) -> {
        for (int i = 0; i < acc.length; i++) {
            operations.get(i).accumulateFn().accept(acc[i], item);
        }
    }).andCombine(operations.stream().anyMatch(o -> o.combineFn() == null) ? null : (acc1, acc2) -> {
        for (int i = 0; i < acc1.length; i++) {
            operations.get(i).combineFn().accept(acc1[i], acc2[i]);
        }
    }).andDeduct(operations.stream().anyMatch(o -> o.deductFn() == null) ? null : (acc1, acc2) -> {
        for (int i = 0; i < acc1.length; i++) {
            operations.get(i).deductFn().accept(acc1[i], acc2[i]);
        }
    }).andExport(acc -> {
        ItemsByTag result = new ItemsByTag();
        for (int i = 0; i < tags.size(); i++) {
            Object exportedVal = operations.get(i).exportFn().apply(acc[i]);
            result.put(tags.get(i), exportedVal);
        }
        return exportFinishFn.apply(result);
    }).andFinish(acc -> {
        ItemsByTag result = new ItemsByTag();
        for (int i = 0; i < tags.size(); i++) {
            Object finishedVal = operations.get(i).finishFn().apply(acc[i]);
            result.put(tags.get(i), finishedVal);
        }
        return exportFinishFn.apply(result);
    });
}