Examples with Edge com.hazelcast.jet.core.Edge used on opensource projects

Example 11 with Edge

use of com.hazelcast.jet.core.Edge in project hazelcast by hazelcast.

the class RebalanceStreamStageTest method when_peekAndRebalanceByKeyAndMap_then_dagEdgePartitionedDistributed.

@Test
public void when_peekAndRebalanceByKeyAndMap_then_dagEdgePartitionedDistributed() {
    // Given
    List<Integer> input = sequence(itemCount);
    StreamStage<Integer> srcStage = streamStageFromList(input);
    FunctionEx<Integer, String> formatFn = i -> String.format("%04d-string", i);
    // When
    StreamStage<String> mapped = srcStage.peek().rebalance(i -> i).map(formatFn);
    // Then
    mapped.writeTo(sink);
    DAG dag = p.toDag();
    Edge srcToMap = dag.getInboundEdges("map").get(0);
    assertTrue("Rebalancing should make the edge distributed", srcToMap.isDistributed());
    assertNotNull("Rebalanced by key, the edge must be partitioned", srcToMap.getPartitioner());
}

Example 12 with Edge

use of com.hazelcast.jet.core.Edge in project hazelcast by hazelcast.

the class RebalanceStreamStageTest method when_rebalanceByKeyAndMap_then_dagEdgePartitionedDistributed.

@Test
public void when_rebalanceByKeyAndMap_then_dagEdgePartitionedDistributed() {
    // Given
    List<Integer> input = sequence(itemCount);
    StreamStage<Integer> srcStage = streamStageFromList(input);
    FunctionEx<Integer, String> formatFn = i -> String.format("%04d-string", i);
    // When
    StreamStage<String> mapped = srcStage.rebalance(i -> i).map(formatFn);
    // Then
    mapped.writeTo(sink);
    DAG dag = p.toDag();
    Edge srcToMap = dag.getInboundEdges("map").get(0);
    assertTrue("Rebalancing should make the edge distributed", srcToMap.isDistributed());
    assertNotNull("Rebalanced by key, the edge must be partitioned", srcToMap.getPartitioner());
}

Example 13 with Edge

use of com.hazelcast.jet.core.Edge in project hazelcast by hazelcast.

the class RebalanceBatchStageTest method assertSingleStageAggregation.

private void assertSingleStageAggregation() {
    DAG dag = p.toDag();
    try {
        Edge srcToAggregate = dag.getOutboundEdges("items").get(0);
        assertTrue("Outbound edge after rebalancing must be distributed", srcToAggregate.isDistributed());
        Edge aggregateToSink = dag.getOutboundEdges(srcToAggregate.getDestName()).get(0);
        List<Edge> sinkOutbound = dag.getOutboundEdges(aggregateToSink.getDestName());
        assertEquals("Aggregation after rebalancing must be single-stage", 0, sinkOutbound.size());
    } catch (AssertionError e) {
        System.err.println(dag.toDotString());
        throw e;
    }
}

Example 14 with Edge

use of com.hazelcast.jet.core.Edge in project hazelcast by hazelcast.

the class RebalanceBatchStageTest method assertTwoStageGlobalAggregation.

private void assertTwoStageGlobalAggregation(int i) {
    DAG dag = p.toDag();
    try {
        Edge srcToAggregate = dag.getOutboundEdges("items" + (i == 1 ? "" : "-" + i)).get(0);
        assertNull("Rebalanced edge to global aggregation must be unicast", srcToAggregate.getPartitioner());
        assertTrue("Outbound edge after rebalancing must be distributed", srcToAggregate.isDistributed());
        String accumulatingVertexName = srcToAggregate.getDestName();
        String aggregatePrepare = "aggregate-prepare";
        assertEquals("Aggregation must be two-stage", aggregatePrepare, accumulatingVertexName.substring(accumulatingVertexName.length() - aggregatePrepare.length()));
        Edge internalAggregationEdge = dag.getOutboundEdges(accumulatingVertexName).get(0);
        assertTrue("Internal aggregation edge must be distributed", internalAggregationEdge.isDistributed());
        assertNotNull("Internal aggregation edge must be partitioned", internalAggregationEdge.getPartitioner());
    } catch (AssertionError e) {
        System.err.println(dag.toDotString());
        throw e;
    }
}

Example 15 with Edge

use of com.hazelcast.jet.core.Edge in project hazelcast by hazelcast.

the class RebalanceBatchStageTest method when_rebalanceByKeyAndAggregate_then_distributedPartitionedEdgeAndTwoStageAggregation.

@Test
public void when_rebalanceByKeyAndAggregate_then_distributedPartitionedEdgeAndTwoStageAggregation() {
    List<Integer> input = sequence(itemCount);
    BatchStage<Integer> srcStage = batchStageFromList(input);
    // When
    BatchStage<Long> aggregated = srcStage.rebalance(i -> 2 * i).aggregate(SUMMING);
    // Then
    aggregated.writeTo(sink);
    DAG dag = p.toDag();
    try {
        Edge srcToAggregate = dag.getOutboundEdges("items").get(0);
        assertNotNull("Rebalanced edge to grouped aggregation must be partitioned", srcToAggregate.getPartitioner());
        assertTrue("Outbound edge after rebalancing must be distributed", srcToAggregate.isDistributed());
        String accumulatingVertexName = srcToAggregate.getDestName();
        assertEquals("Aggregation must be two-stage", "aggregate-prepare", accumulatingVertexName);
        Edge internalAggregationEdge = dag.getOutboundEdges(accumulatingVertexName).get(0);
        assertTrue("Internal aggregation edge must be distributed", internalAggregationEdge.isDistributed());
        assertNotNull("Internal aggregation edge must be partitioned", internalAggregationEdge.getPartitioner());
    } catch (AssertionError e) {
        System.err.println(dag.toDotString());
        throw e;
    }
    execute();
    assertEquals(singletonList(input.stream().mapToLong(i -> i).sum()), new ArrayList<>(sinkList));
}