Examples with Edge com.hazelcast.jet.core.Edge used on opensource projects

Example 1 with Edge

use of com.hazelcast.jet.core.Edge in project hazelcast-jet by hazelcast.

the class SkipPipe method buildDAG.

@Override
public Vertex buildDAG(DAG dag) {
    Vertex previous = upstream.buildDAG(dag);
    // required final for lambda variable capture
    final long skip = this.skip;
    Vertex skipVertex = dag.newVertex(uniqueVertexName("skip"), () -> new SkipP(skip)).localParallelism(1);
    Edge edge = between(previous, skipVertex);
    // if upstream is not ordered, we need to shuffle data to one node
    if (!upstream.isOrdered()) {
        edge = edge.distributed().allToOne();
    }
    dag.edge(edge);
    return skipVertex;
}

Example 2 with Edge

use of com.hazelcast.jet.core.Edge in project hazelcast by hazelcast.

the class CreateDagVisitor method connectInput.

/**
 * Converts the {@code inputRel} into a {@code Vertex} by visiting it and
 * create an edge from the input vertex into {@code thisVertex}.
 *
 * @param configureEdgeFn optional function to configure the edge
 * @return the input vertex
 */
private Vertex connectInput(RelNode inputRel, Vertex thisVertex, @Nullable Consumer<Edge> configureEdgeFn) {
    Vertex inputVertex = ((PhysicalRel) inputRel).accept(this);
    Edge edge = between(inputVertex, thisVertex);
    if (configureEdgeFn != null) {
        configureEdgeFn.accept(edge);
    }
    dag.edge(edge);
    return inputVertex;
}

Example 3 with Edge

use of com.hazelcast.jet.core.Edge in project hazelcast by hazelcast.

the class CreateDagVisitor method onSlidingWindowAggregate.

public Vertex onSlidingWindowAggregate(SlidingWindowAggregatePhysicalRel rel) {
    FunctionEx<JetSqlRow, ?> groupKeyFn = rel.groupKeyFn();
    AggregateOperation<?, JetSqlRow> aggregateOperation = rel.aggrOp();
    Expression<?> timestampExpression = rel.timestampExpression();
    ToLongFunctionEx<JetSqlRow> timestampFn = row -> WindowUtils.extractMillis(timestampExpression.eval(row.getRow(), MOCK_EEC));
    SlidingWindowPolicy windowPolicy = rel.windowPolicyProvider().apply(MOCK_EEC);
    KeyedWindowResultFunction<? super Object, ? super JetSqlRow, ?> resultMapping = rel.outputValueMapping();
    if (rel.numStages() == 1) {
        Vertex vertex = dag.newUniqueVertex("Sliding-Window-AggregateByKey", Processors.aggregateToSlidingWindowP(singletonList(groupKeyFn), singletonList(timestampFn), TimestampKind.EVENT, windowPolicy, 0, aggregateOperation, resultMapping));
        connectInput(rel.getInput(), vertex, edge -> edge.distributeTo(localMemberAddress).allToOne(""));
        return vertex;
    } else {
        assert rel.numStages() == 2;
        Vertex vertex1 = dag.newUniqueVertex("Sliding-Window-AccumulateByKey", Processors.accumulateByFrameP(singletonList(groupKeyFn), singletonList(timestampFn), TimestampKind.EVENT, windowPolicy, aggregateOperation));
        Vertex vertex2 = dag.newUniqueVertex("Sliding-Window-CombineByKey", Processors.combineToSlidingWindowP(windowPolicy, aggregateOperation, resultMapping));
        connectInput(rel.getInput(), vertex1, edge -> edge.partitioned(groupKeyFn));
        dag.edge(between(vertex1, vertex2).distributed().partitioned(entryKey()));
        return vertex2;
    }
}

Example 4 with Edge

use of com.hazelcast.jet.core.Edge in project hazelcast by hazelcast.

the class AggregateTransform method addToDagTwoStage.

// WHEN PRESERVE ORDER IS NOT ACTIVE
// ---------       ---------
// | source0 | ... | sourceN |
// ---------       ---------
// |              |
// local          local
// unicast        unicast
// v              v
// -------------------
// |    accumulateP    |
// -------------------
// |
// distributed
// all-to-one
// v
// ----------------
// |    combineP    | local parallelism = 1
// ----------------
// WHEN PRESERVE ORDER IS ACTIVE
// ---------       ---------
// | source0 | ... | sourceN |
// ---------       ---------
// |              |
// isolated       isolated
// v              v
// -------------------
// |    accumulateP    |
// -------------------
// |
// distributed
// all-to-one
// v
// ----------------
// |    combineP    | local parallelism = 1
// ----------------
private void addToDagTwoStage(Planner p, Context context) {
    String vertexName = name();
    determineLocalParallelism(LOCAL_PARALLELISM_USE_DEFAULT, context, p.isPreserveOrder());
    Vertex v1 = p.dag.newVertex(vertexName + FIRST_STAGE_VERTEX_NAME_SUFFIX, accumulateP(aggrOp)).localParallelism(determinedLocalParallelism());
    if (p.isPreserveOrder()) {
        p.addEdges(this, v1, Edge::isolated);
    } else {
        p.addEdges(this, v1);
    }
    determinedLocalParallelism(1);
    PlannerVertex pv2 = p.addVertex(this, vertexName, determinedLocalParallelism(), ProcessorMetaSupplier.forceTotalParallelismOne(ProcessorSupplier.of(combineP(aggrOp)), vertexName));
    p.dag.edge(between(v1, pv2.v).distributed().allToOne(vertexName));
}

Example 5 with Edge

use of com.hazelcast.jet.core.Edge in project hazelcast by hazelcast.

the class MapTransform method addToDag.

@Override
public void addToDag(Planner p, Context context) {
    determineLocalParallelism(LOCAL_PARALLELISM_USE_DEFAULT, context, p.isPreserveOrder());
    PlannerVertex pv = p.addVertex(this, name(), determinedLocalParallelism(), mapP(mapFn()));
    if (p.isPreserveOrder()) {
        p.addEdges(this, pv.v, Edge::isolated);
    } else {
        p.addEdges(this, pv.v);
    }
}