Examples with FlatMapTransform com.hazelcast.jet.impl.pipeline.transform.FlatMapTransform used on opensource projects

Example 1 with FlatMapTransform

use of com.hazelcast.jet.impl.pipeline.transform.FlatMapTransform in project hazelcast by hazelcast.

the class Planner method findFusableChain.

private static List<Transform> findFusableChain(@Nonnull Transform transform, @Nonnull Map<Transform, List<Transform>> adjacencyMap) {
    ArrayList<Transform> chain = new ArrayList<>();
    for (; ; ) {
        if (!(transform instanceof MapTransform || transform instanceof FlatMapTransform)) {
            break;
        }
        chain.add(transform);
        List<Transform> downstream = adjacencyMap.get(transform);
        if (downstream.size() != 1) {
            break;
        }
        Transform nextTransform = downstream.get(0);
        if (nextTransform.localParallelism() != transform.localParallelism() || nextTransform.shouldRebalanceInput(0)) {
            break;
        }
        transform = nextTransform;
    }
    return chain.size() > 1 ? chain : null;
}

Example 2 with FlatMapTransform

use of com.hazelcast.jet.impl.pipeline.transform.FlatMapTransform in project hazelcast by hazelcast.

the class Planner method fuseFlatMapTransforms.

@SuppressWarnings("rawtypes")
private static Transform fuseFlatMapTransforms(List<Transform> chain) {
    assert chain.size() > 1 : "chain.size()=" + chain.size();
    assert chain.get(0).upstream().size() == 1;
    int lastFlatMap = 0;
    FunctionEx<Object, Traverser> flatMapFn = null;
    for (int i = 0; i < chain.size(); i++) {
        if (chain.get(i) instanceof FlatMapTransform) {
            FunctionEx<Object, Traverser> function = ((FlatMapTransform) chain.get(i)).flatMapFn();
            FunctionEx<Object, Object> inputMapFn = mergeMapFunctions(chain.subList(lastFlatMap, i));
            if (inputMapFn != null) {
                flatMapFn = flatMapFn == null ? (Object t) -> {
                    Object mappedValue = inputMapFn.apply(t);
                    return mappedValue != null ? function.apply(mappedValue) : Traversers.empty();
                } : flatMapFn.andThen(r -> r.map(inputMapFn).flatMap(function));
            } else {
                flatMapFn = flatMapFn == null ? function::apply : flatMapFn.andThen(r -> r.flatMap(function));
            }
            lastFlatMap = i + 1;
        }
    }
    FunctionEx trailingMapFn = mergeMapFunctions(chain.subList(lastFlatMap, chain.size()));
    String name = chain.stream().map(Transform::name).collect(Collectors.joining(", ", "fused(", ")"));
    Transform fused;
    if (flatMapFn == null) {
        fused = new MapTransform(name, chain.get(0).upstream().get(0), trailingMapFn);
    } else {
        if (trailingMapFn != null) {
            flatMapFn = flatMapFn.andThen(t -> t.map(trailingMapFn));
        }
        fused = new FlatMapTransform(name, chain.get(0).upstream().get(0), flatMapFn);
    }
    // if the first stage of the chain is rebalanced, then we set
    // the rebalance flag of the created fused stage. Only consider
    // the case when first element of the chain is rebalanced
    // because there isn't any other case. If any stage in the
    // middle includes rebalance, then those stages are not fused
    // by findFusableChain().
    fused.setRebalanceInput(0, chain.get(0).shouldRebalanceInput(0));
    return fused;
}