Examples with Vertex com.hazelcast.jet.core.Vertex used on opensource projects

Example 96 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast by hazelcast.

the class DistinctTransform method addToDag.

@Override
public void addToDag(Planner p, Context context) {
    String vertexName = name();
    determineLocalParallelism(LOCAL_PARALLELISM_USE_DEFAULT, context, false);
    Vertex v1 = p.dag.newVertex(vertexName + FIRST_STAGE_VERTEX_NAME_SUFFIX, distinctP(keyFn)).localParallelism(determinedLocalParallelism());
    PlannerVertex pv2 = p.addVertex(this, vertexName, determinedLocalParallelism(), distinctP(keyFn));
    p.addEdges(this, v1, (e, ord) -> e.partitioned(keyFn, HASH_CODE));
    p.dag.edge(between(v1, pv2.v).distributed().partitioned(keyFn));
}

Example 97 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast by hazelcast.

the class HashJoinTransform method addToDag.

// ---------           ----------           ----------
// | primary |         | joined-1 |         | joined-2 |
// ---------           ----------           ----------
// |                   |                     |
// |              distributed          distributed
// |               broadcast            broadcast
// |                   v                     v
// |             -------------         -------------
// |            | collector-1 |       | collector-2 |
// |            | localPara=1 |       | localPara=1 |
// |             -------------         -------------
// |                   |                     |
// |                 local                 local
// local             broadcast             broadcast
// unicast           prioritized           prioritized
// ordinal 0           ordinal 1             ordinal 2
// \                   |                     |
// ----------------\  |   /----------------/
// v  v  v
// --------
// | joiner |
// --------
@Override
@SuppressWarnings("unchecked")
public void addToDag(Planner p, Context context) {
    determineLocalParallelism(LOCAL_PARALLELISM_USE_DEFAULT, context, p.isPreserveOrder());
    PlannerVertex primary = p.xform2vertex.get(this.upstream().get(0));
    List keyFns = toList(this.clauses, JoinClause::leftKeyFn);
    List<Tag> tags = this.tags;
    BiFunctionEx mapToOutputBiFn = this.mapToOutputBiFn;
    TriFunction mapToOutputTriFn = this.mapToOutputTriFn;
    // must be extracted to variable, probably because of serialization bug
    BiFunctionEx<List<Tag>, Object[], ItemsByTag> tupleToItems = tupleToItemsByTag(whereNullsNotAllowed);
    Vertex joiner = p.addVertex(this, name() + "-joiner", determinedLocalParallelism(), () -> new HashJoinP<>(keyFns, tags, mapToOutputBiFn, mapToOutputTriFn, tupleToItems)).v;
    Edge edgeToJoiner = from(primary.v, primary.nextAvailableOrdinal()).to(joiner, 0);
    if (p.isPreserveOrder()) {
        edgeToJoiner.isolated();
    } else {
        applyRebalancing(edgeToJoiner, this);
    }
    p.dag.edge(edgeToJoiner);
    String collectorName = name() + "-collector";
    int collectorOrdinal = 1;
    for (Transform fromTransform : tailList(this.upstream())) {
        PlannerVertex fromPv = p.xform2vertex.get(fromTransform);
        JoinClause<?, ?, ?, ?> clause = this.clauses.get(collectorOrdinal - 1);
        FunctionEx<Object, Object> getKeyFn = (FunctionEx<Object, Object>) clause.rightKeyFn();
        FunctionEx<Object, Object> projectFn = (FunctionEx<Object, Object>) clause.rightProjectFn();
        Vertex collector = p.dag.newVertex(collectorName + collectorOrdinal, () -> new HashJoinCollectP(getKeyFn, projectFn));
        collector.localParallelism(1);
        p.dag.edge(from(fromPv.v, fromPv.nextAvailableOrdinal()).to(collector, 0).distributed().broadcast());
        p.dag.edge(from(collector, 0).to(joiner, collectorOrdinal).broadcast().priority(-1));
        collectorOrdinal++;
    }
}

Example 98 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast by hazelcast.

the class SortTransform method addToDag.

@Override
public void addToDag(Planner p, Context context) {
    String vertexName = name();
    determineLocalParallelism(LOCAL_PARALLELISM_USE_DEFAULT, context, p.isPreserveOrder());
    Vertex v1 = p.dag.newVertex(vertexName, sortP(comparator)).localParallelism(determinedLocalParallelism());
    if (p.isPreserveOrder()) {
        p.addEdges(this, v1, Edge::isolated);
    } else {
        p.addEdges(this, v1);
    }
    determinedLocalParallelism(1);
    PlannerVertex pv2 = p.addVertex(this, vertexName + COLLECT_STAGE_SUFFIX, determinedLocalParallelism(), ProcessorMetaSupplier.forceTotalParallelismOne(ProcessorSupplier.of(mapP(identity())), vertexName));
    p.dag.edge(between(v1, pv2.v).distributed().allToOne(vertexName).ordered(comparator));
}

Example 99 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast by hazelcast.

the class WindowAggregateTransform method addSlidingWindowTwoStage.

// WHEN PRESERVE ORDER IS NOT ACTIVE
// --------        ---------
// | source0 | ... | sourceN |
// --------        ---------
// |               |
// local           local
// unicast         unicast
// v               v
// --------------------
// | accumulateByFrameP | keyFn = constantKey()
// --------------------
// |
// distributed
// all-to-one
// v
// -------------------------
// | combineToSlidingWindowP | local parallelism = 1
// -------------------------
// WHEN PRESERVE ORDER IS ACTIVE
// --------        ---------
// | source0 | ... | sourceN |
// --------        ---------
// |               |
// isolated        isolated
// v               v
// --------------------
// | accumulateByFrameP | keyFn = constantKey()
// --------------------
// |
// distributed
// all-to-one
// v
// -------------------------
// | combineToSlidingWindowP | local parallelism = 1
// -------------------------
private void addSlidingWindowTwoStage(Planner p, SlidingWindowDefinition wDef, Context context) {
    determineLocalParallelism(LOCAL_PARALLELISM_USE_DEFAULT, context, p.isPreserveOrder());
    SlidingWindowPolicy winPolicy = slidingWinPolicy(wDef.windowSize(), wDef.slideBy());
    Vertex v1 = p.dag.newVertex(name() + FIRST_STAGE_VERTEX_NAME_SUFFIX, accumulateByFrameP(nCopies(aggrOp.arity(), new ConstantFunctionEx<>(name().hashCode())), nCopies(aggrOp.arity(), (ToLongFunctionEx<JetEvent<?>>) JetEvent::timestamp), TimestampKind.EVENT, winPolicy, aggrOp));
    v1.localParallelism(determinedLocalParallelism());
    if (p.isPreserveOrder()) {
        p.addEdges(this, v1, Edge::isolated);
    } else {
        // when preserveOrder is false, we use requested parallelism
        // for 1st stage: edge to it is local-unicast, each processor
        // can process part of the input which will be combined into
        // one result in 2nd stage.
        p.addEdges(this, v1);
    }
    determinedLocalParallelism(1);
    PlannerVertex pv2 = p.addVertex(this, name(), determinedLocalParallelism(), combineToSlidingWindowP(winPolicy, aggrOp, jetEventOfWindowResultFn()));
    p.dag.edge(between(v1, pv2.v).distributed().allToOne(name().hashCode()));
}

Example 100 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast by hazelcast.

the class WindowGroupTransform method addSlidingWindowTwoStage.

// ---------       ---------
// | source0 | ... | sourceN |
// ---------       ---------
// |               |
// local           local
// partitioned     partitioned
// v               v
// --------------------
// | accumulateByFrameP |
// --------------------
// |
// distributed
// partitioned
// v
// -------------------------
// | combineToSlidingWindowP |
// -------------------------
private void addSlidingWindowTwoStage(Planner p, SlidingWindowDefinition wDef) {
    SlidingWindowPolicy winPolicy = slidingWinPolicy(wDef.windowSize(), wDef.slideBy());
    Vertex v1 = p.dag.newVertex(name() + FIRST_STAGE_VERTEX_NAME_SUFFIX, accumulateByFrameP(keyFns, nCopies(keyFns.size(), (ToLongFunctionEx<JetEvent<?>>) JetEvent::timestamp), TimestampKind.EVENT, winPolicy, aggrOp));
    v1.localParallelism(determinedLocalParallelism());
    PlannerVertex pv2 = p.addVertex(this, name(), determinedLocalParallelism(), combineToSlidingWindowP(winPolicy, aggrOp, jetEventOfKeyedWindowResultFn()));
    p.addEdges(this, v1, (e, ord) -> e.partitioned(keyFns.get(ord), HASH_CODE));
    p.dag.edge(between(v1, pv2.v).distributed().partitioned(entryKey()));
}