use of com.hazelcast.jet.core.DAG in project hazelcast by hazelcast.
the class RebalanceBatchStageTest method when_rebalanceByKeyAndMap_then_dagEdgePartitionedDistributed.
@Test
public void when_rebalanceByKeyAndMap_then_dagEdgePartitionedDistributed() {
// Given
List<Integer> input = sequence(itemCount);
BatchStage<Integer> srcStage = batchStageFromList(input);
FunctionEx<Integer, String> formatFn = i -> String.format("%04d-string", i);
// When
BatchStage<String> mapped = srcStage.rebalance(i -> i).map(formatFn);
// Then
mapped.writeTo(sink);
DAG dag = p.toDag();
Edge srcToMap = dag.getInboundEdges("map").get(0);
assertTrue("Rebalancing should make the edge distributed", srcToMap.isDistributed());
assertNotNull("Rebalancing by key, the edge must be partitioned", srcToMap.getPartitioner());
execute();
assertEquals(streamToString(input.stream(), formatFn), streamToString(sinkStreamOf(String.class), identity()));
}
use of com.hazelcast.jet.core.DAG in project hazelcast by hazelcast.
the class RebalanceBatchStageTest method when_rebalanceAndMap_then_dagEdgeDistributed.
@Test
public void when_rebalanceAndMap_then_dagEdgeDistributed() {
// Given
List<Integer> input = sequence(itemCount);
BatchStage<Integer> srcStage = batchStageFromList(input);
FunctionEx<Integer, String> formatFn = i -> String.format("%04d-string", i);
// When
BatchStage<String> mapped = srcStage.rebalance().map(formatFn);
// Then
mapped.writeTo(sink);
DAG dag = p.toDag();
Edge srcToMap = dag.getInboundEdges("map").get(0);
assertTrue("Rebalancing should make the edge distributed", srcToMap.isDistributed());
assertNull("Didn't rebalance by key, the edge must not be partitioned", srcToMap.getPartitioner());
execute();
assertEquals(streamToString(input.stream(), formatFn), streamToString(sinkStreamOf(String.class), identity()));
}
use of com.hazelcast.jet.core.DAG in project hazelcast by hazelcast.
the class RebalanceBatchStageTest method when_mergeWithRebalanced_thenOnlyRebalancedEdgeDistributed.
@Test
public void when_mergeWithRebalanced_thenOnlyRebalancedEdgeDistributed() {
// Given
Iterator<Integer> sequence = IntStream.iterate(0, i -> i + 1).boxed().iterator();
List<Integer> input0 = streamFromIterator(sequence).limit(itemCount).collect(toList());
List<Integer> input1 = streamFromIterator(sequence).limit(itemCount).collect(toList());
BatchStage<Integer> stage0 = batchStageFromList(input0);
BatchStage<Integer> stage1 = batchStageFromList(input1).rebalance();
// When
BatchStage<Integer> merged = stage0.merge(stage1);
// Then
merged.writeTo(assertAnyOrder(IntStream.range(0, 2 * itemCount).boxed().collect(toList())));
DAG dag = p.toDag();
List<Edge> intoMerge = dag.getInboundEdges("merge");
assertFalse("Didn't rebalance this stage, why is its edge distributed?", intoMerge.get(0).isDistributed());
assertTrue("Rebalancing should make the edge distributed", intoMerge.get(1).isDistributed());
assertNull("Didn't rebalance by key, the edge must not be partitioned", intoMerge.get(1).getPartitioner());
execute();
}
use of com.hazelcast.jet.core.DAG in project hazelcast by hazelcast.
the class RebalanceBatchStageTest method when_hashJoinRebalanceEnrichingStage_then_noEffect.
@Test
public void when_hashJoinRebalanceEnrichingStage_then_noEffect() {
// Given
List<Integer> input = sequence(itemCount);
String prefix = "value-";
BatchStage<Integer> mainStage = batchStageFromList(input);
BatchStage<Entry<Integer, String>> enrichingStageRebalanced = batchStageFromList(input).map(i -> entry(i, prefix + i)).rebalance();
// When
BatchStage<Entry<Integer, String>> joined = mainStage.hashJoin(enrichingStageRebalanced, joinMapEntries(wholeItem()), Util::entry);
// Then
joined.writeTo(sink);
DAG dag = p.toDag();
Edge mapToJoin = dag.getInboundEdges("2-way hash-join-collector1").get(0);
assertTrue("Edge into a hash-join collector vertex must be distributed", mapToJoin.isDistributed());
assertNull("Didn't rebalance by key, the edge must not be partitioned", mapToJoin.getPartitioner());
Edge stage0ToJoin = dag.getInboundEdges("2-way hash-join-joiner").get(0);
assertFalse("Didn't rebalance this stage, why is its edge distributed?", stage0ToJoin.isDistributed());
assertNull("Didn't rebalance by key, the edge must not be partitioned", stage0ToJoin.getPartitioner());
execute();
Function<Entry<Integer, String>, String> formatFn = e -> String.format("(%04d, %s)", e.getKey(), e.getValue());
assertEquals(streamToString(input.stream().map(i -> tuple2(i, prefix + i)), formatFn), streamToString(sinkStreamOfEntry(), formatFn));
}
use of com.hazelcast.jet.core.DAG in project hazelcast by hazelcast.
the class JobRepositoryTest method createDagData.
private Data createDagData() {
DAG dag = new DAG();
dag.newVertex("v", () -> new TestProcessors.MockP().streaming());
return getNodeEngineImpl(instance).toData(dag);
}
Aggregations