Examples with PlannerVertex com.hazelcast.jet.impl.pipeline.Planner.PlannerVertex used on opensource projects

Example 11 with PlannerVertex

use of com.hazelcast.jet.impl.pipeline.Planner.PlannerVertex in project hazelcast-jet by hazelcast.

the class FlatMapTransform method addToDag.

@Override
public void addToDag(Planner p) {
    PlannerVertex pv = p.addVertex(this, p.uniqueVertexName(name(), ""), localParallelism(), flatMapP(flatMapFn()));
    p.addEdges(this, pv.v);
}

Example 12 with PlannerVertex

use of com.hazelcast.jet.impl.pipeline.Planner.PlannerVertex in project hazelcast-jet by hazelcast.

the class WindowGroupTransform method addSlidingWindowSingleStage.

// ---------       ---------
// | source0 | ... | sourceN |
// ---------       ---------
// |              |
// distributed    distributed
// partitioned    partitioned
// \              /
// ---\    /-----
// v  v
// ---------------------------
// | aggregateToSlidingWindowP |
// ---------------------------
private void addSlidingWindowSingleStage(Planner p, SlidingWindowDef wDef) {
    PlannerVertex pv = p.addVertex(this, p.uniqueVertexName(name(), ""), localParallelism(), aggregateToSlidingWindowP(keyFns, nCopies(keyFns.size(), (DistributedToLongFunction<JetEvent>) JetEvent::timestamp), TimestampKind.EVENT, wDef.toSlidingWindowPolicy(), aggrOp, mapToOutputFn));
    p.addEdges(this, pv.v, (e, ord) -> e.distributed().partitioned(keyFns.get(ord)));
}

Example 13 with PlannerVertex

use of com.hazelcast.jet.impl.pipeline.Planner.PlannerVertex in project hazelcast-jet by hazelcast.

the class WindowGroupTransform method addSlidingWindowTwoStage.

// ---------       ---------
// | source0 | ... | sourceN |
// ---------       ---------
// |               |
// local           local
// partitioned     partitioned
// v               v
// --------------------
// | accumulateByFrameP |
// --------------------
// |
// distributed
// partitioned
// v
// -------------------------
// | combineToSlidingWindowP |
// -------------------------
private void addSlidingWindowTwoStage(Planner p, SlidingWindowDef wDef) {
    String namePrefix = p.uniqueVertexName(name(), "-step");
    SlidingWindowPolicy winPolicy = wDef.toSlidingWindowPolicy();
    Vertex v1 = p.dag.newVertex(namePrefix + '1', accumulateByFrameP(keyFns, nCopies(keyFns.size(), (DistributedToLongFunction<JetEvent>) JetEvent::timestamp), TimestampKind.EVENT, winPolicy, aggrOp));
    v1.localParallelism(localParallelism());
    PlannerVertex pv2 = p.addVertex(this, namePrefix + '2', localParallelism(), combineToSlidingWindowP(winPolicy, aggrOp, mapToOutputFn));
    p.addEdges(this, v1, (e, ord) -> e.partitioned(keyFns.get(ord), HASH_CODE));
    p.dag.edge(between(v1, pv2.v).distributed().partitioned(entryKey()));
}

Example 14 with PlannerVertex

use of com.hazelcast.jet.impl.pipeline.Planner.PlannerVertex in project hazelcast by hazelcast.

the class AggregateTransform method addToDagTwoStage.

// WHEN PRESERVE ORDER IS NOT ACTIVE
// ---------       ---------
// | source0 | ... | sourceN |
// ---------       ---------
// |              |
// local          local
// unicast        unicast
// v              v
// -------------------
// |    accumulateP    |
// -------------------
// |
// distributed
// all-to-one
// v
// ----------------
// |    combineP    | local parallelism = 1
// ----------------
// WHEN PRESERVE ORDER IS ACTIVE
// ---------       ---------
// | source0 | ... | sourceN |
// ---------       ---------
// |              |
// isolated       isolated
// v              v
// -------------------
// |    accumulateP    |
// -------------------
// |
// distributed
// all-to-one
// v
// ----------------
// |    combineP    | local parallelism = 1
// ----------------
private void addToDagTwoStage(Planner p, Context context) {
    String vertexName = name();
    determineLocalParallelism(LOCAL_PARALLELISM_USE_DEFAULT, context, p.isPreserveOrder());
    Vertex v1 = p.dag.newVertex(vertexName + FIRST_STAGE_VERTEX_NAME_SUFFIX, accumulateP(aggrOp)).localParallelism(determinedLocalParallelism());
    if (p.isPreserveOrder()) {
        p.addEdges(this, v1, Edge::isolated);
    } else {
        p.addEdges(this, v1);
    }
    determinedLocalParallelism(1);
    PlannerVertex pv2 = p.addVertex(this, vertexName, determinedLocalParallelism(), ProcessorMetaSupplier.forceTotalParallelismOne(ProcessorSupplier.of(combineP(aggrOp)), vertexName));
    p.dag.edge(between(v1, pv2.v).distributed().allToOne(vertexName));
}

Example 15 with PlannerVertex

use of com.hazelcast.jet.impl.pipeline.Planner.PlannerVertex in project hazelcast by hazelcast.

the class GlobalMapStatefulTransform method addToDag.

@Override
public void addToDag(Planner p, Context context) {
    determinedLocalParallelism(1);
    ConstantFunctionEx<T, Integer> keyFn = new ConstantFunctionEx<>(name().hashCode());
    PlannerVertex pv = p.addVertex(this, name(), determinedLocalParallelism(), mapStatefulP(Long.MAX_VALUE, keyFn, timestampFn, createFn, statefulMapFn, null));
    p.addEdges(this, pv.v, edge -> edge.partitioned(keyFn).distributed());
}