Examples with Edge com.hazelcast.jet.core.Edge used on opensource projects

Example 21 with Edge

use of com.hazelcast.jet.core.Edge in project hazelcast by hazelcast.

the class RebalanceBatchStageTest method when_hashJoinRebalanceEnrichingStage_then_noEffect.

@Test
public void when_hashJoinRebalanceEnrichingStage_then_noEffect() {
    // Given
    List<Integer> input = sequence(itemCount);
    String prefix = "value-";
    BatchStage<Integer> mainStage = batchStageFromList(input);
    BatchStage<Entry<Integer, String>> enrichingStageRebalanced = batchStageFromList(input).map(i -> entry(i, prefix + i)).rebalance();
    // When
    BatchStage<Entry<Integer, String>> joined = mainStage.hashJoin(enrichingStageRebalanced, joinMapEntries(wholeItem()), Util::entry);
    // Then
    joined.writeTo(sink);
    DAG dag = p.toDag();
    Edge mapToJoin = dag.getInboundEdges("2-way hash-join-collector1").get(0);
    assertTrue("Edge into a hash-join collector vertex must be distributed", mapToJoin.isDistributed());
    assertNull("Didn't rebalance by key, the edge must not be partitioned", mapToJoin.getPartitioner());
    Edge stage0ToJoin = dag.getInboundEdges("2-way hash-join-joiner").get(0);
    assertFalse("Didn't rebalance this stage, why is its edge distributed?", stage0ToJoin.isDistributed());
    assertNull("Didn't rebalance by key, the edge must not be partitioned", stage0ToJoin.getPartitioner());
    execute();
    Function<Entry<Integer, String>, String> formatFn = e -> String.format("(%04d, %s)", e.getKey(), e.getValue());
    assertEquals(streamToString(input.stream().map(i -> tuple2(i, prefix + i)), formatFn), streamToString(sinkStreamOfEntry(), formatFn));
}

Example 22 with Edge

use of com.hazelcast.jet.core.Edge in project hazelcast-jet by hazelcast.

the class Planner method addEdges.

public void addEdges(Transform transform, Vertex toVertex, BiConsumer<Edge, Integer> configureEdgeFn) {
    int destOrdinal = 0;
    for (Transform fromTransform : transform.upstream()) {
        PlannerVertex fromPv = xform2vertex.get(fromTransform);
        Edge edge = from(fromPv.v, fromPv.nextAvailableOrdinal()).to(toVertex, destOrdinal);
        dag.edge(edge);
        configureEdgeFn.accept(edge, destOrdinal);
        destOrdinal++;
    }
}

Example 23 with Edge

use of com.hazelcast.jet.core.Edge in project hazelcast by hazelcast.

the class RebalanceBatchStageTest method when_mergeWithRebalancedByKey_thenOnlyRebalancedEdgePartitionedDistributed.

@Test
public void when_mergeWithRebalancedByKey_thenOnlyRebalancedEdgePartitionedDistributed() {
    // Given
    Iterator<Integer> sequence = IntStream.iterate(0, i -> i + 1).boxed().iterator();
    List<Integer> input0 = streamFromIterator(sequence).limit(itemCount).collect(toList());
    List<Integer> input1 = streamFromIterator(sequence).limit(itemCount).collect(toList());
    BatchStage<Integer> stage0 = batchStageFromList(input0);
    BatchStage<Integer> stage1 = batchStageFromList(input1).rebalance(i -> i);
    // When
    BatchStage<Integer> merged = stage0.merge(stage1);
    // Then
    merged.writeTo(assertAnyOrder(IntStream.range(0, 2 * itemCount).boxed().collect(toList())));
    DAG dag = p.toDag();
    List<Edge> intoMerge = dag.getInboundEdges("merge");
    assertFalse("Didn't rebalance this stage, why is its edge distributed?", intoMerge.get(0).isDistributed());
    assertNull("Didn't rebalance by key, the edge must not be partitioned", intoMerge.get(0).getPartitioner());
    assertTrue("Rebalancing should make the edge distributed", intoMerge.get(1).isDistributed());
    assertNotNull("Rebalancing by key, the edge must be partitioned", intoMerge.get(1).getPartitioner());
    execute();
}

Example 24 with Edge

use of com.hazelcast.jet.core.Edge in project hazelcast by hazelcast.

the class RebalanceBatchStageTest method when_mergeRebalancedWithNonRebalanced_thenOnlyRebalancedEdgeDistributed.

@Test
public void when_mergeRebalancedWithNonRebalanced_thenOnlyRebalancedEdgeDistributed() {
    // Given
    Iterator<Integer> sequence = IntStream.iterate(0, i -> i + 1).boxed().iterator();
    List<Integer> input0 = streamFromIterator(sequence).limit(itemCount).collect(toList());
    List<Integer> input1 = streamFromIterator(sequence).limit(itemCount).collect(toList());
    BatchStage<Integer> stage0 = batchStageFromList(input0).rebalance();
    BatchStage<Integer> stage1 = batchStageFromList(input1);
    // When
    BatchStage<Integer> merged = stage0.merge(stage1);
    // Then
    merged.writeTo(assertAnyOrder(IntStream.range(0, 2 * itemCount).boxed().collect(toList())));
    DAG dag = p.toDag();
    List<Edge> intoMerge = dag.getInboundEdges("merge");
    assertTrue("Rebalancing should make the edge distributed", intoMerge.get(0).isDistributed());
    assertNull("Didn't rebalance by key, the edge must not be partitioned", intoMerge.get(0).getPartitioner());
    assertFalse("Didn't rebalance this stage, why is its edge distributed?", intoMerge.get(1).isDistributed());
    execute();
}

Example 25 with Edge

use of com.hazelcast.jet.core.Edge in project hazelcast by hazelcast.

the class RebalanceBatchStageTest method when_hashJoinRebalanceMainStage_then_distributedEdge.

@Test
public void when_hashJoinRebalanceMainStage_then_distributedEdge() {
    // Given
    List<Integer> input = sequence(itemCount);
    String prefix = "value-";
    BatchStage<Integer> stage0Rebalanced = batchStageFromList(input).rebalance();
    BatchStage<Entry<Integer, String>> enrichingStage = batchStageFromList(input).map(i -> entry(i, prefix + i));
    // When
    BatchStage<Entry<Integer, String>> joined = stage0Rebalanced.hashJoin(enrichingStage, joinMapEntries(wholeItem()), // Method reference avoided due to JDK bug
    (k, v) -> entry(k, v));
    // Then
    joined.writeTo(sink);
    DAG dag = p.toDag();
    Edge stage0ToJoin = dag.getInboundEdges("2-way hash-join-joiner").get(0);
    assertTrue("Rebalancing should make the edge distributed", stage0ToJoin.isDistributed());
    assertNull("Didn't rebalance by key, the edge must not be partitioned", stage0ToJoin.getPartitioner());
    execute();
    Function<Entry<Integer, String>, String> formatFn = e -> String.format("(%04d, %s)", e.getKey(), e.getValue());
    assertEquals(streamToString(input.stream().map(i -> tuple2(i, prefix + i)), formatFn), streamToString(sinkStreamOfEntry(), formatFn));
}