use of com.hazelcast.jet.core.DAG in project hazelcast by hazelcast.
the class RebalanceBatchStageTest method assertSingleStageAggregation.
private void assertSingleStageAggregation() {
DAG dag = p.toDag();
try {
Edge srcToAggregate = dag.getOutboundEdges("items").get(0);
assertTrue("Outbound edge after rebalancing must be distributed", srcToAggregate.isDistributed());
Edge aggregateToSink = dag.getOutboundEdges(srcToAggregate.getDestName()).get(0);
List<Edge> sinkOutbound = dag.getOutboundEdges(aggregateToSink.getDestName());
assertEquals("Aggregation after rebalancing must be single-stage", 0, sinkOutbound.size());
} catch (AssertionError e) {
System.err.println(dag.toDotString());
throw e;
}
}
use of com.hazelcast.jet.core.DAG in project hazelcast by hazelcast.
the class RebalanceBatchStageTest method assertTwoStageGlobalAggregation.
private void assertTwoStageGlobalAggregation(int i) {
DAG dag = p.toDag();
try {
Edge srcToAggregate = dag.getOutboundEdges("items" + (i == 1 ? "" : "-" + i)).get(0);
assertNull("Rebalanced edge to global aggregation must be unicast", srcToAggregate.getPartitioner());
assertTrue("Outbound edge after rebalancing must be distributed", srcToAggregate.isDistributed());
String accumulatingVertexName = srcToAggregate.getDestName();
String aggregatePrepare = "aggregate-prepare";
assertEquals("Aggregation must be two-stage", aggregatePrepare, accumulatingVertexName.substring(accumulatingVertexName.length() - aggregatePrepare.length()));
Edge internalAggregationEdge = dag.getOutboundEdges(accumulatingVertexName).get(0);
assertTrue("Internal aggregation edge must be distributed", internalAggregationEdge.isDistributed());
assertNotNull("Internal aggregation edge must be partitioned", internalAggregationEdge.getPartitioner());
} catch (AssertionError e) {
System.err.println(dag.toDotString());
throw e;
}
}
use of com.hazelcast.jet.core.DAG in project hazelcast by hazelcast.
the class RebalanceBatchStageTest method when_rebalanceByKeyAndAggregate_then_distributedPartitionedEdgeAndTwoStageAggregation.
@Test
public void when_rebalanceByKeyAndAggregate_then_distributedPartitionedEdgeAndTwoStageAggregation() {
List<Integer> input = sequence(itemCount);
BatchStage<Integer> srcStage = batchStageFromList(input);
// When
BatchStage<Long> aggregated = srcStage.rebalance(i -> 2 * i).aggregate(SUMMING);
// Then
aggregated.writeTo(sink);
DAG dag = p.toDag();
try {
Edge srcToAggregate = dag.getOutboundEdges("items").get(0);
assertNotNull("Rebalanced edge to grouped aggregation must be partitioned", srcToAggregate.getPartitioner());
assertTrue("Outbound edge after rebalancing must be distributed", srcToAggregate.isDistributed());
String accumulatingVertexName = srcToAggregate.getDestName();
assertEquals("Aggregation must be two-stage", "aggregate-prepare", accumulatingVertexName);
Edge internalAggregationEdge = dag.getOutboundEdges(accumulatingVertexName).get(0);
assertTrue("Internal aggregation edge must be distributed", internalAggregationEdge.isDistributed());
assertNotNull("Internal aggregation edge must be partitioned", internalAggregationEdge.getPartitioner());
} catch (AssertionError e) {
System.err.println(dag.toDotString());
throw e;
}
execute();
assertEquals(singletonList(input.stream().mapToLong(i -> i).sum()), new ArrayList<>(sinkList));
}
use of com.hazelcast.jet.core.DAG in project hazelcast by hazelcast.
the class RebalanceBatchStageTest method when_peekAndRebalanceByKeyAndMap_then_dagEdgePartitionedDistributed.
@Test
public void when_peekAndRebalanceByKeyAndMap_then_dagEdgePartitionedDistributed() {
// Given
List<Integer> input = sequence(itemCount);
BatchStage<Integer> srcStage = batchStageFromList(input);
FunctionEx<Integer, String> formatFn = i -> String.format("%04d-string", i);
// When
BatchStage<String> mapped = srcStage.peek().rebalance(i -> i).map(formatFn);
// Then
mapped.writeTo(sink);
DAG dag = p.toDag();
Edge srcToMap = dag.getInboundEdges("map").get(0);
assertTrue("Rebalancing should make the edge distributed", srcToMap.isDistributed());
assertNotNull("Rebalancing by key, the edge must be partitioned", srcToMap.getPartitioner());
execute();
assertEquals(streamToString(input.stream(), formatFn), streamToString(sinkStreamOf(String.class), identity()));
}
use of com.hazelcast.jet.core.DAG in project hazelcast by hazelcast.
the class RebalanceBatchStageTest method when_hashJoinBuilderRebalanceMainStage_then_distributedEdge.
@Test
public void when_hashJoinBuilderRebalanceMainStage_then_distributedEdge() {
// Given
List<Integer> input = sequence(itemCount);
String prefix = "value-";
BatchStage<Integer> stage0Rebalanced = batchStageFromList(input).rebalance();
BatchStage<Entry<Integer, String>> enrichingStage = batchStageFromList(input).map(i -> entry(i, prefix + i));
// When
HashJoinBuilder<Integer> b = stage0Rebalanced.hashJoinBuilder();
Tag<String> tag1 = b.add(enrichingStage, joinMapEntries(wholeItem()));
// Method reference avoided due to JDK bug
BatchStage<Entry<Integer, ItemsByTag>> joined = b.build((k, v) -> entry(k, v));
// Then
joined.writeTo(sink);
DAG dag = p.toDag();
Edge stage0ToJoin = dag.getInboundEdges("2-way hash-join-joiner").get(0);
assertTrue("Rebalancing should make the edge distributed", stage0ToJoin.isDistributed());
assertNull("Didn't rebalance by key, the edge must not be partitioned", stage0ToJoin.getPartitioner());
execute();
Function<Entry<Integer, String>, String> formatFn = e -> String.format("(%04d, %s)", e.getKey(), e.getValue());
assertEquals(streamToString(input.stream().map(i -> tuple2(i, prefix + i)), formatFn), streamToString(this.<Integer, ItemsByTag>sinkStreamOfEntry(), e -> formatFn.apply(entry(e.getKey(), e.getValue().get(tag1)))));
}
Aggregations