Examples with PlannerVertex com.hazelcast.jet.impl.pipeline.Planner.PlannerVertex used on opensource projects

Example 21 with PlannerVertex

use of com.hazelcast.jet.impl.pipeline.Planner.PlannerVertex in project hazelcast by hazelcast.

the class TimestampTransform method addToDag.

@Override
public void addToDag(Planner p, Context context) {
    determineLocalParallelism(LOCAL_PARALLELISM_USE_DEFAULT, context, true);
    PlannerVertex pv = p.addVertex(this, name(), determinedLocalParallelism(), insertWatermarksP(eventTimePolicy));
    p.addEdges(this, pv.v, Edge::isolated);
}

Example 22 with PlannerVertex

use of com.hazelcast.jet.impl.pipeline.Planner.PlannerVertex in project hazelcast by hazelcast.

the class WindowAggregateTransform method addSessionWindow.

// ---------       ---------
// | source0 | ... | sourceN |
// ---------       ---------
// |              |
// distributed    distributed
// all-to-one      all-to-one
// \              /
// ---\    /-----
// v  v
// ---------------------------
// | aggregateToSessionWindowP | local parallelism = 1
// ---------------------------
private void addSessionWindow(Planner p, SessionWindowDefinition wDef) {
    determinedLocalParallelism(1);
    PlannerVertex pv = p.addVertex(this, name(), determinedLocalParallelism(), aggregateToSessionWindowP(wDef.sessionTimeout(), wDef.earlyResultsPeriod(), nCopies(aggrOp.arity(), (ToLongFunctionEx<JetEvent<?>>) JetEvent::timestamp), nCopies(aggrOp.arity(), new ConstantFunctionEx<>(name().hashCode())), aggrOp, jetEventOfWindowResultFn()));
    p.addEdges(this, pv.v, edge -> edge.distributed().allToOne(name().hashCode()));
}

Example 23 with PlannerVertex

use of com.hazelcast.jet.impl.pipeline.Planner.PlannerVertex in project hazelcast by hazelcast.

the class WindowAggregateTransform method addSlidingWindowSingleStage.

// ---------       ---------
// | source0 | ... | sourceN |
// ---------       ---------
// |              |
// distributed    distributed
// all-to-one      all-to-one
// \              /
// ---\    /-----
// v  v
// ---------------------------
// | aggregateToSlidingWindowP | local parallelism = 1
// ---------------------------
private void addSlidingWindowSingleStage(Planner p, SlidingWindowDefinition wDef) {
    determinedLocalParallelism(1);
    PlannerVertex pv = p.addVertex(this, name(), determinedLocalParallelism(), aggregateToSlidingWindowP(nCopies(aggrOp.arity(), new ConstantFunctionEx<>(name().hashCode())), nCopies(aggrOp.arity(), (ToLongFunctionEx<JetEvent<?>>) JetEvent::timestamp), TimestampKind.EVENT, slidingWinPolicy(wDef.windowSize(), wDef.slideBy()), wDef.earlyResultsPeriod(), aggrOp, jetEventOfWindowResultFn()));
    p.addEdges(this, pv.v, edge -> edge.distributed().allToOne(name().hashCode()));
}

Example 24 with PlannerVertex

use of com.hazelcast.jet.impl.pipeline.Planner.PlannerVertex in project hazelcast by hazelcast.

the class StreamSourceTransform method addToDag.

@Override
public void addToDag(Planner p, Context context) {
    if (emitsWatermarks || eventTimePolicy == null) {
        // Reached when the source either emits both JetEvents and watermarks
        // or neither. In these cases we don't have to insert watermarks.
        final ProcessorMetaSupplier metaSupplier = metaSupplierFn.apply(eventTimePolicy != null ? eventTimePolicy : noEventTime());
        determineLocalParallelism(metaSupplier.preferredLocalParallelism(), context, false);
        p.addVertex(this, name(), determinedLocalParallelism(), metaSupplier);
    } else {
        // ------------
        // |  sourceP   |
        // ------------
        // |
        // isolated
        // v
        // -------------
        // |  insertWmP  |
        // -------------
        String v1name = name();
        final ProcessorMetaSupplier metaSupplier = metaSupplierFn.apply(eventTimePolicy);
        determineLocalParallelism(metaSupplier.preferredLocalParallelism(), context, false);
        Vertex v1 = p.dag.newVertex(v1name, metaSupplier).localParallelism(determinedLocalParallelism());
        PlannerVertex pv2 = p.addVertex(this, v1name + "-add-timestamps", determinedLocalParallelism(), insertWatermarksP(eventTimePolicy));
        p.dag.edge(between(v1, pv2.v).isolated());
    }
}

Example 25 with PlannerVertex

use of com.hazelcast.jet.impl.pipeline.Planner.PlannerVertex in project hazelcast by hazelcast.

the class WindowGroupTransform method addSessionWindow.

// ---------       ---------
// | source0 | ... | sourceN |
// ---------       ---------
// |              |
// distributed    distributed
// partitioned    partitioned
// \              /
// ---\    /-----
// v  v
// ---------------------------
// | aggregateToSessionWindowP |
// ---------------------------
private void addSessionWindow(Planner p, SessionWindowDefinition wDef) {
    PlannerVertex pv = p.addVertex(this, name(), determinedLocalParallelism(), aggregateToSessionWindowP(wDef.sessionTimeout(), wDef.earlyResultsPeriod(), nCopies(keyFns.size(), (ToLongFunctionEx<JetEvent<?>>) JetEvent::timestamp), keyFns, aggrOp, jetEventOfKeyedWindowResultFn()));
    p.addEdges(this, pv.v, (e, ord) -> e.distributed().partitioned(keyFns.get(ord)));
}