Examples with Vertex com.hazelcast.jet.core.Vertex used on opensource projects

Example 11 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast-jet by hazelcast.

the class GroupingSinkReducer method reduce.

@Override
public R reduce(StreamContext context, Pipe<? extends T> upstream) {
    DAG dag = new DAG();
    Vertex previous = upstream.buildDAG(dag);
    Vertex merger = dag.newVertex("group-and-accumulate", () -> new GroupAndAccumulateP<>(classifier, collector));
    Vertex combiner = dag.newVertex("combine-groups", () -> new CombineGroupsP<>(collector));
    Vertex writer = dag.newVertex(sinkName, metaSupplier);
    dag.edge(between(previous, merger).partitioned(classifier::apply, HASH_CODE)).edge(between(merger, combiner).distributed().partitioned(entryKey())).edge(between(combiner, writer));
    executeJob(context, dag);
    return toDistributedObject.apply(context.getJetInstance());
}

Example 12 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast-jet by hazelcast.

the class IListReducer method reduce.

@Override
public IListJet<T> reduce(StreamContext context, Pipe<? extends T> upstream) {
    IListJet<T> target = context.getJetInstance().getList(listName);
    DAG dag = new DAG();
    Vertex vertex = upstream.buildDAG(dag);
    Vertex writer = dag.newVertex("write-list-" + listName, SinkProcessors.writeListP(listName)).localParallelism(1);
    dag.edge(between(vertex, writer));
    executeJob(context, dag);
    return target;
}

Example 13 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast-jet by hazelcast.

the class DistinctPipe method orderedGraph.

@Nonnull
private Vertex orderedGraph(DAG dag) {
    Vertex previous = upstream.buildDAG(dag);
    Vertex distinct = dag.newVertex(uniqueVertexName("distinct"), DistinctP::new).localParallelism(1);
    dag.edge(between(previous, distinct));
    return distinct;
}

Example 14 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast-jet by hazelcast.

the class WindowGroupTransform method addSlidingWindowTwoStage.

// ---------       ---------
// | source0 | ... | sourceN |
// ---------       ---------
// |               |
// local           local
// partitioned     partitioned
// v               v
// --------------------
// | accumulateByFrameP |
// --------------------
// |
// distributed
// partitioned
// v
// -------------------------
// | combineToSlidingWindowP |
// -------------------------
private void addSlidingWindowTwoStage(Planner p, SlidingWindowDef wDef) {
    String namePrefix = p.uniqueVertexName(name(), "-step");
    SlidingWindowPolicy winPolicy = wDef.toSlidingWindowPolicy();
    Vertex v1 = p.dag.newVertex(namePrefix + '1', accumulateByFrameP(keyFns, nCopies(keyFns.size(), (DistributedToLongFunction<JetEvent>) JetEvent::timestamp), TimestampKind.EVENT, winPolicy, aggrOp));
    v1.localParallelism(localParallelism());
    PlannerVertex pv2 = p.addVertex(this, namePrefix + '2', localParallelism(), combineToSlidingWindowP(winPolicy, aggrOp, mapToOutputFn));
    p.addEdges(this, v1, (e, ord) -> e.partitioned(keyFns.get(ord), HASH_CODE));
    p.dag.edge(between(v1, pv2.v).distributed().partitioned(entryKey()));
}

Example 15 with Vertex

use of com.hazelcast.jet.core.Vertex in project hazelcast-jet by hazelcast.

the class SkipPipe method buildDAG.

@Override
public Vertex buildDAG(DAG dag) {
    Vertex previous = upstream.buildDAG(dag);
    // required final for lambda variable capture
    final long skip = this.skip;
    Vertex skipVertex = dag.newVertex(uniqueVertexName("skip"), () -> new SkipP(skip)).localParallelism(1);
    Edge edge = between(previous, skipVertex);
    // if upstream is not ordered, we need to shuffle data to one node
    if (!upstream.isOrdered()) {
        edge = edge.distributed().allToOne();
    }
    dag.edge(edge);
    return skipVertex;
}