Examples with Edge com.hazelcast.jet.core.Edge used on opensource projects

Example 36 with Edge

use of com.hazelcast.jet.core.Edge in project hazelcast by hazelcast.

the class CreateDagVisitor method onUnion.

public Vertex onUnion(UnionPhysicalRel rel) {
    // UNION_TO_DISTINCT rule : Union[all=false] -> Union[all=true] + Aggregate.
    if (!rel.all) {
        throw new RuntimeException("Union[all=false] rel should never be produced");
    }
    Vertex merger = dag.newUniqueVertex("UnionMerger", ProcessorSupplier.of(mapP(FunctionEx.identity())));
    int ordinal = 0;
    for (RelNode input : rel.getInputs()) {
        Vertex inputVertex = ((PhysicalRel) input).accept(this);
        Edge edge = Edge.from(inputVertex).to(merger, ordinal++);
        dag.edge(edge);
    }
    return merger;
}

Example 37 with Edge

use of com.hazelcast.jet.core.Edge in project hazelcast by hazelcast.

the class CreateDagVisitor method connectJoinInput.

private void connectJoinInput(JetJoinInfo joinInfo, RelNode leftInputRel, RelNode rightInputRel, Vertex joinVertex) {
    Vertex leftInput = ((PhysicalRel) leftInputRel).accept(this);
    Vertex rightInput = ((PhysicalRel) rightInputRel).accept(this);
    Edge left = between(leftInput, joinVertex).priority(LOW_PRIORITY).broadcast().distributed();
    Edge right = from(rightInput).to(joinVertex, 1).priority(HIGH_PRIORITY).unicast().local();
    if (joinInfo.isLeftOuter()) {
        left = left.unicast().local();
        right = right.broadcast().distributed();
    }
    if (joinInfo.isEquiJoin()) {
        left = left.distributed().partitioned(ObjectArrayKey.projectFn(joinInfo.leftEquiJoinIndices()));
        right = right.distributed().partitioned(ObjectArrayKey.projectFn(joinInfo.rightEquiJoinIndices()));
    }
    dag.edge(left);
    dag.edge(right);
}

Example 38 with Edge

use of com.hazelcast.jet.core.Edge in project hazelcast by hazelcast.

the class RebalanceStreamStageTest method when_peekAndRebalanceAndMap_then_dagEdgeDistributed.

@Test
public void when_peekAndRebalanceAndMap_then_dagEdgeDistributed() {
    // Given
    List<Integer> input = sequence(itemCount);
    StreamStage<Integer> srcStage = streamStageFromList(input);
    FunctionEx<Integer, String> formatFn = i -> String.format("%04d-string", i);
    // When
    StreamStage<String> mapped = srcStage.peek().rebalance().map(formatFn);
    // Then
    mapped.writeTo(sink);
    DAG dag = p.toDag();
    Edge srcToMap = dag.getInboundEdges("map").get(0);
    assertTrue("Rebalancing should make the edge distributed", srcToMap.isDistributed());
    assertNull("Didn't rebalance by key, the edge must not be partitioned", srcToMap.getPartitioner());
    execute();
    assertEquals(streamToString(input.stream(), formatFn), streamToString(sinkStreamOf(String.class), identity()));
}

Example 39 with Edge

use of com.hazelcast.jet.core.Edge in project hazelcast by hazelcast.

the class RebalanceStreamStageTest method assertTwoStageAggregation.

private void assertTwoStageAggregation(boolean isRebalanceByKey) {
    DAG dag = p.toDag();
    try {
        Edge srcToAggregate = dag.getOutboundEdges("add-timestamps").get(0);
        assertEquals(isRebalanceByKey ? "Edge to aggregation after rebalancing by key must be partitioned" : "Rebalanced edge to global aggregation must be unicast", isRebalanceByKey, srcToAggregate.getPartitioner() != null);
        assertTrue("Outbound edge after rebalancing must be distributed", srcToAggregate.isDistributed());
        String accumulatingVertexName = srcToAggregate.getDestName();
        String aggregatePrepare = "sliding-window-prepare";
        assertEquals("Aggregation must be two-stage;", aggregatePrepare, accumulatingVertexName.substring(accumulatingVertexName.length() - aggregatePrepare.length()));
        Edge internalAggregationEdge = dag.getOutboundEdges(accumulatingVertexName).get(0);
        assertTrue("Internal aggregation edge must be distributed", internalAggregationEdge.isDistributed());
        assertNotNull("Internal aggregation edge must be partitioned", internalAggregationEdge.getPartitioner());
    } catch (AssertionError e) {
        System.err.println(dag.toDotString());
        throw e;
    }
}