Examples with Edge com.hazelcast.jet.core.Edge used on opensource projects

Example 16 with Edge

use of com.hazelcast.jet.core.Edge in project hazelcast by hazelcast.

the class RebalanceBatchStageTest method when_peekAndRebalanceByKeyAndMap_then_dagEdgePartitionedDistributed.

@Test
public void when_peekAndRebalanceByKeyAndMap_then_dagEdgePartitionedDistributed() {
    // Given
    List<Integer> input = sequence(itemCount);
    BatchStage<Integer> srcStage = batchStageFromList(input);
    FunctionEx<Integer, String> formatFn = i -> String.format("%04d-string", i);
    // When
    BatchStage<String> mapped = srcStage.peek().rebalance(i -> i).map(formatFn);
    // Then
    mapped.writeTo(sink);
    DAG dag = p.toDag();
    Edge srcToMap = dag.getInboundEdges("map").get(0);
    assertTrue("Rebalancing should make the edge distributed", srcToMap.isDistributed());
    assertNotNull("Rebalancing by key, the edge must be partitioned", srcToMap.getPartitioner());
    execute();
    assertEquals(streamToString(input.stream(), formatFn), streamToString(sinkStreamOf(String.class), identity()));
}

Example 17 with Edge

use of com.hazelcast.jet.core.Edge in project hazelcast by hazelcast.

the class RebalanceBatchStageTest method when_hashJoinBuilderRebalanceMainStage_then_distributedEdge.

@Test
public void when_hashJoinBuilderRebalanceMainStage_then_distributedEdge() {
    // Given
    List<Integer> input = sequence(itemCount);
    String prefix = "value-";
    BatchStage<Integer> stage0Rebalanced = batchStageFromList(input).rebalance();
    BatchStage<Entry<Integer, String>> enrichingStage = batchStageFromList(input).map(i -> entry(i, prefix + i));
    // When
    HashJoinBuilder<Integer> b = stage0Rebalanced.hashJoinBuilder();
    Tag<String> tag1 = b.add(enrichingStage, joinMapEntries(wholeItem()));
    // Method reference avoided due to JDK bug
    BatchStage<Entry<Integer, ItemsByTag>> joined = b.build((k, v) -> entry(k, v));
    // Then
    joined.writeTo(sink);
    DAG dag = p.toDag();
    Edge stage0ToJoin = dag.getInboundEdges("2-way hash-join-joiner").get(0);
    assertTrue("Rebalancing should make the edge distributed", stage0ToJoin.isDistributed());
    assertNull("Didn't rebalance by key, the edge must not be partitioned", stage0ToJoin.getPartitioner());
    execute();
    Function<Entry<Integer, String>, String> formatFn = e -> String.format("(%04d, %s)", e.getKey(), e.getValue());
    assertEquals(streamToString(input.stream().map(i -> tuple2(i, prefix + i)), formatFn), streamToString(this.<Integer, ItemsByTag>sinkStreamOfEntry(), e -> formatFn.apply(entry(e.getKey(), e.getValue().get(tag1)))));
}

Example 18 with Edge

use of com.hazelcast.jet.core.Edge in project hazelcast by hazelcast.

the class RebalanceBatchStageTest method when_rebalanceByKeyAndMap_then_dagEdgePartitionedDistributed.

@Test
public void when_rebalanceByKeyAndMap_then_dagEdgePartitionedDistributed() {
    // Given
    List<Integer> input = sequence(itemCount);
    BatchStage<Integer> srcStage = batchStageFromList(input);
    FunctionEx<Integer, String> formatFn = i -> String.format("%04d-string", i);
    // When
    BatchStage<String> mapped = srcStage.rebalance(i -> i).map(formatFn);
    // Then
    mapped.writeTo(sink);
    DAG dag = p.toDag();
    Edge srcToMap = dag.getInboundEdges("map").get(0);
    assertTrue("Rebalancing should make the edge distributed", srcToMap.isDistributed());
    assertNotNull("Rebalancing by key, the edge must be partitioned", srcToMap.getPartitioner());
    execute();
    assertEquals(streamToString(input.stream(), formatFn), streamToString(sinkStreamOf(String.class), identity()));
}

Example 19 with Edge

use of com.hazelcast.jet.core.Edge in project hazelcast by hazelcast.

the class RebalanceBatchStageTest method when_rebalanceAndMap_then_dagEdgeDistributed.

@Test
public void when_rebalanceAndMap_then_dagEdgeDistributed() {
    // Given
    List<Integer> input = sequence(itemCount);
    BatchStage<Integer> srcStage = batchStageFromList(input);
    FunctionEx<Integer, String> formatFn = i -> String.format("%04d-string", i);
    // When
    BatchStage<String> mapped = srcStage.rebalance().map(formatFn);
    // Then
    mapped.writeTo(sink);
    DAG dag = p.toDag();
    Edge srcToMap = dag.getInboundEdges("map").get(0);
    assertTrue("Rebalancing should make the edge distributed", srcToMap.isDistributed());
    assertNull("Didn't rebalance by key, the edge must not be partitioned", srcToMap.getPartitioner());
    execute();
    assertEquals(streamToString(input.stream(), formatFn), streamToString(sinkStreamOf(String.class), identity()));
}

Example 20 with Edge

use of com.hazelcast.jet.core.Edge in project hazelcast by hazelcast.

the class RebalanceBatchStageTest method when_mergeWithRebalanced_thenOnlyRebalancedEdgeDistributed.

@Test
public void when_mergeWithRebalanced_thenOnlyRebalancedEdgeDistributed() {
    // Given
    Iterator<Integer> sequence = IntStream.iterate(0, i -> i + 1).boxed().iterator();
    List<Integer> input0 = streamFromIterator(sequence).limit(itemCount).collect(toList());
    List<Integer> input1 = streamFromIterator(sequence).limit(itemCount).collect(toList());
    BatchStage<Integer> stage0 = batchStageFromList(input0);
    BatchStage<Integer> stage1 = batchStageFromList(input1).rebalance();
    // When
    BatchStage<Integer> merged = stage0.merge(stage1);
    // Then
    merged.writeTo(assertAnyOrder(IntStream.range(0, 2 * itemCount).boxed().collect(toList())));
    DAG dag = p.toDag();
    List<Edge> intoMerge = dag.getInboundEdges("merge");
    assertFalse("Didn't rebalance this stage, why is its edge distributed?", intoMerge.get(0).isDistributed());
    assertTrue("Rebalancing should make the edge distributed", intoMerge.get(1).isDistributed());
    assertNull("Didn't rebalance by key, the edge must not be partitioned", intoMerge.get(1).getPartitioner());
    execute();
}