Examples with Transform com.hazelcast.jet.impl.pipeline.transform.Transform used on opensource projects

Example 16 with Transform

use of com.hazelcast.jet.impl.pipeline.transform.Transform in project hazelcast by hazelcast.

the class ComputeStageImplBase method attachMapUsingPartitionedServiceAsync.

@Nonnull
@SuppressWarnings({ "unchecked", "rawtypes" })
<S, K, R, RET> RET attachMapUsingPartitionedServiceAsync(@Nonnull ServiceFactory<?, S> serviceFactory, int maxConcurrentOps, boolean preserveOrder, @Nonnull FunctionEx<? super T, ? extends K> partitionKeyFn, @Nonnull BiFunctionEx<? super S, ? super T, ? extends CompletableFuture<R>> mapAsyncFn) {
    checkSerializable(mapAsyncFn, "mapAsyncFn");
    checkSerializable(partitionKeyFn, "partitionKeyFn");
    serviceFactory = moveAttachedFilesToPipeline(serviceFactory);
    BiFunctionEx<? super S, ? super T, ? extends CompletableFuture<Traverser<R>>> flatMapAsyncFn = (s, t) -> mapAsyncFn.apply(s, t).thenApply(Traversers::singleton);
    BiFunctionEx adaptedFlatMapFn = fnAdapter.adaptFlatMapUsingServiceAsyncFn(flatMapAsyncFn);
    FunctionEx adaptedPartitionKeyFn = fnAdapter.adaptKeyFn(partitionKeyFn);
    PartitionedProcessorTransform processorTransform = flatMapUsingServiceAsyncPartitionedTransform(transform, "map", serviceFactory, maxConcurrentOps, preserveOrder, adaptedFlatMapFn, adaptedPartitionKeyFn);
    return attach(processorTransform, fnAdapter);
}

Example 17 with Transform

use of com.hazelcast.jet.impl.pipeline.transform.Transform in project hazelcast by hazelcast.

the class Planner method addEdges.

public void addEdges(Transform transform, Vertex toVertex, ObjIntConsumer<Edge> configureEdgeFn) {
    int destOrdinal = 0;
    for (Transform fromTransform : transform.upstream()) {
        PlannerVertex fromPv = xform2vertex.get(fromTransform);
        Edge edge = from(fromPv.v, fromPv.nextAvailableOrdinal()).to(toVertex, destOrdinal);
        dag.edge(edge);
        configureEdgeFn.accept(edge, destOrdinal);
        applyRebalancing(edge, transform);
        destOrdinal++;
    }
}

Example 18 with Transform

use of com.hazelcast.jet.impl.pipeline.transform.Transform in project hazelcast by hazelcast.

the class Planner method fuseFlatMapTransforms.

@SuppressWarnings("rawtypes")
private static Transform fuseFlatMapTransforms(List<Transform> chain) {
    assert chain.size() > 1 : "chain.size()=" + chain.size();
    assert chain.get(0).upstream().size() == 1;
    int lastFlatMap = 0;
    FunctionEx<Object, Traverser> flatMapFn = null;
    for (int i = 0; i < chain.size(); i++) {
        if (chain.get(i) instanceof FlatMapTransform) {
            FunctionEx<Object, Traverser> function = ((FlatMapTransform) chain.get(i)).flatMapFn();
            FunctionEx<Object, Object> inputMapFn = mergeMapFunctions(chain.subList(lastFlatMap, i));
            if (inputMapFn != null) {
                flatMapFn = flatMapFn == null ? (Object t) -> {
                    Object mappedValue = inputMapFn.apply(t);
                    return mappedValue != null ? function.apply(mappedValue) : Traversers.empty();
                } : flatMapFn.andThen(r -> r.map(inputMapFn).flatMap(function));
            } else {
                flatMapFn = flatMapFn == null ? function::apply : flatMapFn.andThen(r -> r.flatMap(function));
            }
            lastFlatMap = i + 1;
        }
    }
    FunctionEx trailingMapFn = mergeMapFunctions(chain.subList(lastFlatMap, chain.size()));
    String name = chain.stream().map(Transform::name).collect(Collectors.joining(", ", "fused(", ")"));
    Transform fused;
    if (flatMapFn == null) {
        fused = new MapTransform(name, chain.get(0).upstream().get(0), trailingMapFn);
    } else {
        if (trailingMapFn != null) {
            flatMapFn = flatMapFn.andThen(t -> t.map(trailingMapFn));
        }
        fused = new FlatMapTransform(name, chain.get(0).upstream().get(0), flatMapFn);
    }
    // if the first stage of the chain is rebalanced, then we set
    // the rebalance flag of the created fused stage. Only consider
    // the case when first element of the chain is rebalanced
    // because there isn't any other case. If any stage in the
    // middle includes rebalance, then those stages are not fused
    // by findFusableChain().
    fused.setRebalanceInput(0, chain.get(0).shouldRebalanceInput(0));
    return fused;
}

Example 19 with Transform

use of com.hazelcast.jet.impl.pipeline.transform.Transform in project hazelcast by hazelcast.

the class Planner method createDag.

@SuppressWarnings("rawtypes")
DAG createDag(Context context) {
    pipeline.makeNamesUnique();
    Map<Transform, List<Transform>> adjacencyMap = pipeline.adjacencyMap();
    validateNoLeakage(adjacencyMap);
    checkTopologicalSort(adjacencyMap.entrySet());
    // Find the greatest common denominator of all frame lengths
    // appearing in the pipeline
    long frameSizeGcd = Util.gcd(adjacencyMap.keySet().stream().map(Transform::preferredWatermarkStride).filter(frameSize -> frameSize > 0).mapToLong(i -> i).toArray());
    if (frameSizeGcd == 0) {
        // even if there are no window aggregations, we want the watermarks for latency debugging
        frameSizeGcd = MAXIMUM_WATERMARK_GAP;
    }
    if (frameSizeGcd > MAXIMUM_WATERMARK_GAP) {
        frameSizeGcd = Util.gcd(frameSizeGcd, MAXIMUM_WATERMARK_GAP);
    }
    LoggingUtil.logFine(LOGGER, "Watermarks in the pipeline will be throttled to %d", frameSizeGcd);
    // Update watermark throttling frame length on all transforms with the determined length
    for (Transform transform : adjacencyMap.keySet()) {
        if (transform instanceof StreamSourceTransform) {
            StreamSourceTransform t = (StreamSourceTransform) transform;
            EventTimePolicy policy = t.getEventTimePolicy();
            if (policy != null) {
                t.setEventTimePolicy(withFrameSize(policy, frameSizeGcd));
            }
        } else if (transform instanceof TimestampTransform) {
            TimestampTransform t = (TimestampTransform) transform;
            t.setEventTimePolicy(withFrameSize(t.getEventTimePolicy(), frameSizeGcd));
        }
    }
    // fuse subsequent map/filter/flatMap transforms into one
    Map<Transform, List<Transform>> originalParents = new HashMap<>();
    List<Transform> transforms = new ArrayList<>(adjacencyMap.keySet());
    for (int i = 0; i < transforms.size(); i++) {
        Transform transform = transforms.get(i);
        List<Transform> chain = findFusableChain(transform, adjacencyMap);
        if (chain == null) {
            continue;
        }
        // remove transforms in the chain and replace the parent with a fused transform
        transforms.removeAll(chain.subList(1, chain.size()));
        Transform fused = fuseFlatMapTransforms(chain);
        transforms.set(i, fused);
        Transform lastInChain = chain.get(chain.size() - 1);
        for (Transform downstream : adjacencyMap.get(lastInChain)) {
            originalParents.put(downstream, new ArrayList<>(downstream.upstream()));
            downstream.upstream().replaceAll(p -> p == lastInChain ? fused : p);
        }
    }
    for (Transform transform : transforms) {
        transform.addToDag(this, context);
    }
    // restore original parents
    for (Entry<Transform, List<Transform>> en : originalParents.entrySet()) {
        List<Transform> upstream = en.getKey().upstream();
        for (int i = 0; i < upstream.size(); i++) {
            en.getKey().upstream().set(i, en.getValue().get(i));
        }
    }
    return dag;
}

Example 20 with Transform

use of com.hazelcast.jet.impl.pipeline.transform.Transform in project hazelcast by hazelcast.

the class AggBuilder method build.

@Nonnull
@SuppressWarnings({ "unchecked", "rawtypes" })
public <A, R, OUT, OUT_STAGE extends GeneralStage<OUT>> OUT_STAGE build(@Nonnull AggregateOperation<A, R> aggrOp, @Nonnull CreateOutStageFn<OUT, OUT_STAGE> createOutStageFn) {
    AggregateOperation adaptedAggrOp = wDef != null ? ADAPT_TO_JET_EVENT.adaptAggregateOperation(aggrOp) : aggrOp;
    List<Transform> upstreamTransforms = toList(upstreamStages, s -> ((AbstractStage) s).transform);
    final AbstractTransform transform;
    if (wDef != null) {
        transform = new WindowAggregateTransform<>(upstreamTransforms, wDef, adaptedAggrOp);
    } else {
        transform = new AggregateTransform<>(upstreamTransforms, adaptedAggrOp);
    }
    OUT_STAGE attached = createOutStageFn.get(transform, ADAPT_TO_JET_EVENT, pipelineImpl);
    pipelineImpl.connectGeneralStages(upstreamStages, transform);
    return attached;
}