Examples with FunctionEx com.hazelcast.function.FunctionEx used on opensource projects

Example 31 with FunctionEx

use of com.hazelcast.function.FunctionEx in project hazelcast by hazelcast.

the class ComputeStageImplBase method attachMapUsingPartitionedServiceAsync.

@Nonnull
@SuppressWarnings({ "unchecked", "rawtypes" })
<S, K, R, RET> RET attachMapUsingPartitionedServiceAsync(@Nonnull ServiceFactory<?, S> serviceFactory, int maxConcurrentOps, boolean preserveOrder, @Nonnull FunctionEx<? super T, ? extends K> partitionKeyFn, @Nonnull BiFunctionEx<? super S, ? super T, ? extends CompletableFuture<R>> mapAsyncFn) {
    checkSerializable(mapAsyncFn, "mapAsyncFn");
    checkSerializable(partitionKeyFn, "partitionKeyFn");
    serviceFactory = moveAttachedFilesToPipeline(serviceFactory);
    BiFunctionEx<? super S, ? super T, ? extends CompletableFuture<Traverser<R>>> flatMapAsyncFn = (s, t) -> mapAsyncFn.apply(s, t).thenApply(Traversers::singleton);
    BiFunctionEx adaptedFlatMapFn = fnAdapter.adaptFlatMapUsingServiceAsyncFn(flatMapAsyncFn);
    FunctionEx adaptedPartitionKeyFn = fnAdapter.adaptKeyFn(partitionKeyFn);
    PartitionedProcessorTransform processorTransform = flatMapUsingServiceAsyncPartitionedTransform(transform, "map", serviceFactory, maxConcurrentOps, preserveOrder, adaptedFlatMapFn, adaptedPartitionKeyFn);
    return attach(processorTransform, fnAdapter);
}

Example 32 with FunctionEx

use of com.hazelcast.function.FunctionEx in project hazelcast by hazelcast.

the class ComputeStageImplBase method attachFlatMapUsingPartitionedService.

@Nonnull
@SuppressWarnings({ "unchecked", "rawtypes" })
<S, K, R, RET> RET attachFlatMapUsingPartitionedService(@Nonnull ServiceFactory<?, S> serviceFactory, @Nonnull FunctionEx<? super T, ? extends K> partitionKeyFn, @Nonnull BiFunctionEx<? super S, ? super T, ? extends Traverser<R>> flatMapFn) {
    checkSerializable(flatMapFn, "flatMapFn");
    checkSerializable(partitionKeyFn, "partitionKeyFn");
    serviceFactory = moveAttachedFilesToPipeline(serviceFactory);
    BiFunctionEx adaptedFlatMapFn = fnAdapter.adaptFlatMapUsingServiceFn(flatMapFn);
    FunctionEx adaptedPartitionKeyFn = fnAdapter.adaptKeyFn(partitionKeyFn);
    return (RET) attach(flatMapUsingServicePartitionedTransform(transform, serviceFactory, adaptedFlatMapFn, adaptedPartitionKeyFn), fnAdapter);
}

Example 33 with FunctionEx

use of com.hazelcast.function.FunctionEx in project hazelcast by hazelcast.

the class Planner method fuseFlatMapTransforms.

@SuppressWarnings("rawtypes")
private static Transform fuseFlatMapTransforms(List<Transform> chain) {
    assert chain.size() > 1 : "chain.size()=" + chain.size();
    assert chain.get(0).upstream().size() == 1;
    int lastFlatMap = 0;
    FunctionEx<Object, Traverser> flatMapFn = null;
    for (int i = 0; i < chain.size(); i++) {
        if (chain.get(i) instanceof FlatMapTransform) {
            FunctionEx<Object, Traverser> function = ((FlatMapTransform) chain.get(i)).flatMapFn();
            FunctionEx<Object, Object> inputMapFn = mergeMapFunctions(chain.subList(lastFlatMap, i));
            if (inputMapFn != null) {
                flatMapFn = flatMapFn == null ? (Object t) -> {
                    Object mappedValue = inputMapFn.apply(t);
                    return mappedValue != null ? function.apply(mappedValue) : Traversers.empty();
                } : flatMapFn.andThen(r -> r.map(inputMapFn).flatMap(function));
            } else {
                flatMapFn = flatMapFn == null ? function::apply : flatMapFn.andThen(r -> r.flatMap(function));
            }
            lastFlatMap = i + 1;
        }
    }
    FunctionEx trailingMapFn = mergeMapFunctions(chain.subList(lastFlatMap, chain.size()));
    String name = chain.stream().map(Transform::name).collect(Collectors.joining(", ", "fused(", ")"));
    Transform fused;
    if (flatMapFn == null) {
        fused = new MapTransform(name, chain.get(0).upstream().get(0), trailingMapFn);
    } else {
        if (trailingMapFn != null) {
            flatMapFn = flatMapFn.andThen(t -> t.map(trailingMapFn));
        }
        fused = new FlatMapTransform(name, chain.get(0).upstream().get(0), flatMapFn);
    }
    // if the first stage of the chain is rebalanced, then we set
    // the rebalance flag of the created fused stage. Only consider
    // the case when first element of the chain is rebalanced
    // because there isn't any other case. If any stage in the
    // middle includes rebalance, then those stages are not fused
    // by findFusableChain().
    fused.setRebalanceInput(0, chain.get(0).shouldRebalanceInput(0));
    return fused;
}

Example 34 with FunctionEx

use of com.hazelcast.function.FunctionEx in project hazelcast by hazelcast.

the class HashJoinTransform method addToDag.

// ---------           ----------           ----------
// | primary |         | joined-1 |         | joined-2 |
// ---------           ----------           ----------
// |                   |                     |
// |              distributed          distributed
// |               broadcast            broadcast
// |                   v                     v
// |             -------------         -------------
// |            | collector-1 |       | collector-2 |
// |            | localPara=1 |       | localPara=1 |
// |             -------------         -------------
// |                   |                     |
// |                 local                 local
// local             broadcast             broadcast
// unicast           prioritized           prioritized
// ordinal 0           ordinal 1             ordinal 2
// \                   |                     |
// ----------------\  |   /----------------/
// v  v  v
// --------
// | joiner |
// --------
@Override
@SuppressWarnings("unchecked")
public void addToDag(Planner p, Context context) {
    determineLocalParallelism(LOCAL_PARALLELISM_USE_DEFAULT, context, p.isPreserveOrder());
    PlannerVertex primary = p.xform2vertex.get(this.upstream().get(0));
    List keyFns = toList(this.clauses, JoinClause::leftKeyFn);
    List<Tag> tags = this.tags;
    BiFunctionEx mapToOutputBiFn = this.mapToOutputBiFn;
    TriFunction mapToOutputTriFn = this.mapToOutputTriFn;
    // must be extracted to variable, probably because of serialization bug
    BiFunctionEx<List<Tag>, Object[], ItemsByTag> tupleToItems = tupleToItemsByTag(whereNullsNotAllowed);
    Vertex joiner = p.addVertex(this, name() + "-joiner", determinedLocalParallelism(), () -> new HashJoinP<>(keyFns, tags, mapToOutputBiFn, mapToOutputTriFn, tupleToItems)).v;
    Edge edgeToJoiner = from(primary.v, primary.nextAvailableOrdinal()).to(joiner, 0);
    if (p.isPreserveOrder()) {
        edgeToJoiner.isolated();
    } else {
        applyRebalancing(edgeToJoiner, this);
    }
    p.dag.edge(edgeToJoiner);
    String collectorName = name() + "-collector";
    int collectorOrdinal = 1;
    for (Transform fromTransform : tailList(this.upstream())) {
        PlannerVertex fromPv = p.xform2vertex.get(fromTransform);
        JoinClause<?, ?, ?, ?> clause = this.clauses.get(collectorOrdinal - 1);
        FunctionEx<Object, Object> getKeyFn = (FunctionEx<Object, Object>) clause.rightKeyFn();
        FunctionEx<Object, Object> projectFn = (FunctionEx<Object, Object>) clause.rightProjectFn();
        Vertex collector = p.dag.newVertex(collectorName + collectorOrdinal, () -> new HashJoinCollectP(getKeyFn, projectFn));
        collector.localParallelism(1);
        p.dag.edge(from(fromPv.v, fromPv.nextAvailableOrdinal()).to(collector, 0).distributed().broadcast());
        p.dag.edge(from(collector, 0).to(joiner, collectorOrdinal).broadcast().priority(-1));
        collectorOrdinal++;
    }
}

Example 35 with FunctionEx

use of com.hazelcast.function.FunctionEx in project hazelcast by hazelcast.

the class CoAggregateOperationBuilder method build.

/**
 * Builds and returns the multi-input {@link AggregateOperation}. It will
 * call the supplied {@code exportFinishFn} to transform the {@link ItemsByTag}
 * it creates to the result type it emits as the actual result.
 *
 * @param exportFinishFn function to convert {@link ItemsByTag} to the
 *     target result type. It must be stateless and {@linkplain
 *     Processor#isCooperative() cooperative}.
 */
@Nonnull
@SuppressWarnings({ "unchecked", "ConstantConditions" })
public <R> AggregateOperation<Object[], R> build(@Nonnull FunctionEx<? super ItemsByTag, ? extends R> exportFinishFn) {
    checkSerializable(exportFinishFn, "exportFinishFn");
    Tag[] tags = opsByTag.keySet().stream().sorted().toArray(Tag[]::new);
    for (int i = 0; i < tags.length; i++) {
        Preconditions.checkTrue(tags[i].index() == i, "Registered tags' indices are " + stream(tags).map(Tag::index).collect(toList()) + ", but should be " + range(0, tags.length).boxed().collect(toList()));
    }
    // Variable `sorted` extracted due to type inference failure
    Stream<Entry<Tag, AggregateOperation1>> sorted = opsByTag.entrySet().stream().sorted(comparing(Entry::getKey));
    List<AggregateOperation1> ops = sorted.map(Entry::getValue).collect(toList());
    BiConsumerEx[] combineFns = ops.stream().map(AggregateOperation::combineFn).toArray(BiConsumerEx[]::new);
    BiConsumerEx[] deductFns = ops.stream().map(AggregateOperation::deductFn).toArray(BiConsumerEx[]::new);
    FunctionEx[] exportFns = ops.stream().map(AggregateOperation::exportFn).toArray(FunctionEx[]::new);
    FunctionEx[] finishFns = ops.stream().map(AggregateOperation::finishFn).toArray(FunctionEx[]::new);
    AggregateOperationBuilder.VarArity<Object[], Void> b = AggregateOperation.withCreate(() -> ops.stream().map(op -> op.createFn().get()).toArray()).varArity();
    opsByTag.forEach((tag, op) -> {
        int index = tag.index();
        b.andAccumulate(tag, (acc, item) -> op.accumulateFn().accept(acc[index], item));
    });
    return b.andCombine(stream(combineFns).anyMatch(Objects::isNull) ? null : (acc1, acc2) -> {
        for (int i = 0; i < combineFns.length; i++) {
            combineFns[i].accept(acc1[i], acc2[i]);
        }
    }).andDeduct(stream(deductFns).anyMatch(Objects::isNull) ? null : (acc1, acc2) -> {
        for (int i = 0; i < deductFns.length; i++) {
            deductFns[i].accept(acc1[i], acc2[i]);
        }
    }).<R>andExport(acc -> {
        ItemsByTag result = new ItemsByTag();
        for (int i = 0; i < exportFns.length; i++) {
            result.put(tags[i], exportFns[i].apply(acc[i]));
        }
        return exportFinishFn.apply(result);
    }).andFinish(acc -> {
        ItemsByTag result = new ItemsByTag();
        for (int i = 0; i < finishFns.length; i++) {
            result.put(tags[i], finishFns[i].apply(acc[i]));
        }
        return exportFinishFn.apply(result);
    });
}