Examples with PlannerVertex com.hazelcast.jet.impl.pipeline.Planner.PlannerVertex used on opensource projects

Example 1 with PlannerVertex

use of com.hazelcast.jet.impl.pipeline.Planner.PlannerVertex in project hazelcast-jet by hazelcast.

the class GroupTransform method addToDagSingleStage.

// ---------        ---------
// | source0 |  ... | sourceN |
// ---------        ---------
// |                  |
// distributed        distributed
// partitioned        partitioned
// |                  |
// \                  /
// ----\      /------
// v      v
// -----------------
// | aggregateByKeyP |
// -----------------
private void addToDagSingleStage(Planner p) {
    PlannerVertex pv = p.addVertex(this, p.uniqueVertexName(name(), ""), localParallelism(), aggregateByKeyP(groupKeyFns, aggrOp, mapToOutputFn));
    p.addEdges(this, pv.v, (e, ord) -> e.distributed().partitioned(groupKeyFns.get(ord)));
}

Example 2 with PlannerVertex

use of com.hazelcast.jet.impl.pipeline.Planner.PlannerVertex in project hazelcast-jet by hazelcast.

the class HashJoinTransform method addToDag.

// ---------           ----------           ----------
// | primary |         | joined-1 |         | joined-2 |
// ---------           ----------           ----------
// |                   |                     |
// |              distributed          distributed
// |               broadcast            broadcast
// |                   v                     v
// |             -------------         -------------
// |            | collector-1 |       | collector-2 |
// |             -------------         -------------
// |                   |                     |
// |                 local                 local
// local             broadcast             broadcast
// unicast           prioritized           prioritized
// ordinal 0           ordinal 1             ordinal 2
// \                   |                     |
// ----------------\  |   /----------------/
// v  v  v
// --------
// | joiner |
// --------
@Override
@SuppressWarnings("unchecked")
public void addToDag(Planner p) {
    String namePrefix = p.uniqueVertexName(this.name(), "");
    PlannerVertex primary = p.xform2vertex.get(this.upstream().get(0));
    List keyFns = this.clauses.stream().map(JoinClause::leftKeyFn).collect(toList());
    List<Tag> tags = this.tags;
    DistributedBiFunction mapToOutputBiFn = this.mapToOutputBiFn;
    DistributedTriFunction mapToOutputTriFn = this.mapToOutputTriFn;
    Vertex joiner = p.addVertex(this, namePrefix + "-joiner", localParallelism(), () -> new HashJoinP<>(keyFns, tags, mapToOutputBiFn, mapToOutputTriFn)).v;
    p.dag.edge(from(primary.v, primary.nextAvailableOrdinal()).to(joiner, 0));
    String collectorName = namePrefix + "-collector";
    int collectorOrdinal = 1;
    for (Transform fromTransform : tailList(this.upstream())) {
        PlannerVertex fromPv = p.xform2vertex.get(fromTransform);
        JoinClause<?, ?, ?, ?> clause = this.clauses.get(collectorOrdinal - 1);
        DistributedFunction<Object, Object> getKeyFn = (DistributedFunction<Object, Object>) clause.rightKeyFn();
        DistributedFunction<Object, Object> projectFn = (DistributedFunction<Object, Object>) clause.rightProjectFn();
        Vertex collector = p.dag.newVertex(collectorName + collectorOrdinal, () -> new HashJoinCollectP(getKeyFn, projectFn));
        collector.localParallelism(1);
        p.dag.edge(from(fromPv.v, fromPv.nextAvailableOrdinal()).to(collector, 0).distributed().broadcast());
        p.dag.edge(from(collector, 0).to(joiner, collectorOrdinal).broadcast().priority(-1));
        collectorOrdinal++;
    }
}

Example 3 with PlannerVertex

use of com.hazelcast.jet.impl.pipeline.Planner.PlannerVertex in project hazelcast-jet by hazelcast.

the class MapTransform method addToDag.

@Override
public void addToDag(Planner p) {
    PlannerVertex pv = p.addVertex(this, p.uniqueVertexName(name(), ""), localParallelism(), mapP(mapFn()));
    p.addEdges(this, pv.v);
}

Example 4 with PlannerVertex

use of com.hazelcast.jet.impl.pipeline.Planner.PlannerVertex in project hazelcast-jet by hazelcast.

the class MapUsingContextTransform method addToDag.

@Override
public void addToDag(Planner p) {
    PlannerVertex pv = p.addVertex(this, p.uniqueVertexName(name(), ""), localParallelism(), mapUsingContextP(contextFactory, mapFn));
    p.addEdges(this, pv.v);
}

Example 5 with PlannerVertex

use of com.hazelcast.jet.impl.pipeline.Planner.PlannerVertex in project hazelcast-jet by hazelcast.

the class ProcessorTransform method addToDag.

@Override
public void addToDag(Planner p) {
    PlannerVertex pv = p.addVertex(this, p.uniqueVertexName(name(), ""), localParallelism(), procSupplier);
    p.addEdges(this, pv.v);
}