use of com.hazelcast.jet.core.Edge in project hazelcast by hazelcast.
the class CreateDagVisitor method onUnion.
public Vertex onUnion(UnionPhysicalRel rel) {
// UNION_TO_DISTINCT rule : Union[all=false] -> Union[all=true] + Aggregate.
if (!rel.all) {
throw new RuntimeException("Union[all=false] rel should never be produced");
}
Vertex merger = dag.newUniqueVertex("UnionMerger", ProcessorSupplier.of(mapP(FunctionEx.identity())));
int ordinal = 0;
for (RelNode input : rel.getInputs()) {
Vertex inputVertex = ((PhysicalRel) input).accept(this);
Edge edge = Edge.from(inputVertex).to(merger, ordinal++);
dag.edge(edge);
}
return merger;
}
use of com.hazelcast.jet.core.Edge in project hazelcast by hazelcast.
the class CreateDagVisitor method connectJoinInput.
private void connectJoinInput(JetJoinInfo joinInfo, RelNode leftInputRel, RelNode rightInputRel, Vertex joinVertex) {
Vertex leftInput = ((PhysicalRel) leftInputRel).accept(this);
Vertex rightInput = ((PhysicalRel) rightInputRel).accept(this);
Edge left = between(leftInput, joinVertex).priority(LOW_PRIORITY).broadcast().distributed();
Edge right = from(rightInput).to(joinVertex, 1).priority(HIGH_PRIORITY).unicast().local();
if (joinInfo.isLeftOuter()) {
left = left.unicast().local();
right = right.broadcast().distributed();
}
if (joinInfo.isEquiJoin()) {
left = left.distributed().partitioned(ObjectArrayKey.projectFn(joinInfo.leftEquiJoinIndices()));
right = right.distributed().partitioned(ObjectArrayKey.projectFn(joinInfo.rightEquiJoinIndices()));
}
dag.edge(left);
dag.edge(right);
}
use of com.hazelcast.jet.core.Edge in project hazelcast by hazelcast.
the class RebalanceBatchStageTest method when_mergeWithRebalancedByKey_thenOnlyRebalancedEdgePartitionedDistributed.
@Test
public void when_mergeWithRebalancedByKey_thenOnlyRebalancedEdgePartitionedDistributed() {
// Given
Iterator<Integer> sequence = IntStream.iterate(0, i -> i + 1).boxed().iterator();
List<Integer> input0 = streamFromIterator(sequence).limit(itemCount).collect(toList());
List<Integer> input1 = streamFromIterator(sequence).limit(itemCount).collect(toList());
BatchStage<Integer> stage0 = batchStageFromList(input0);
BatchStage<Integer> stage1 = batchStageFromList(input1).rebalance(i -> i);
// When
BatchStage<Integer> merged = stage0.merge(stage1);
// Then
merged.writeTo(assertAnyOrder(IntStream.range(0, 2 * itemCount).boxed().collect(toList())));
DAG dag = p.toDag();
List<Edge> intoMerge = dag.getInboundEdges("merge");
assertFalse("Didn't rebalance this stage, why is its edge distributed?", intoMerge.get(0).isDistributed());
assertNull("Didn't rebalance by key, the edge must not be partitioned", intoMerge.get(0).getPartitioner());
assertTrue("Rebalancing should make the edge distributed", intoMerge.get(1).isDistributed());
assertNotNull("Rebalancing by key, the edge must be partitioned", intoMerge.get(1).getPartitioner());
execute();
}
use of com.hazelcast.jet.core.Edge in project hazelcast by hazelcast.
the class RebalanceBatchStageTest method when_mergeRebalancedWithNonRebalanced_thenOnlyRebalancedEdgeDistributed.
@Test
public void when_mergeRebalancedWithNonRebalanced_thenOnlyRebalancedEdgeDistributed() {
// Given
Iterator<Integer> sequence = IntStream.iterate(0, i -> i + 1).boxed().iterator();
List<Integer> input0 = streamFromIterator(sequence).limit(itemCount).collect(toList());
List<Integer> input1 = streamFromIterator(sequence).limit(itemCount).collect(toList());
BatchStage<Integer> stage0 = batchStageFromList(input0).rebalance();
BatchStage<Integer> stage1 = batchStageFromList(input1);
// When
BatchStage<Integer> merged = stage0.merge(stage1);
// Then
merged.writeTo(assertAnyOrder(IntStream.range(0, 2 * itemCount).boxed().collect(toList())));
DAG dag = p.toDag();
List<Edge> intoMerge = dag.getInboundEdges("merge");
assertTrue("Rebalancing should make the edge distributed", intoMerge.get(0).isDistributed());
assertNull("Didn't rebalance by key, the edge must not be partitioned", intoMerge.get(0).getPartitioner());
assertFalse("Didn't rebalance this stage, why is its edge distributed?", intoMerge.get(1).isDistributed());
execute();
}
use of com.hazelcast.jet.core.Edge in project hazelcast by hazelcast.
the class RebalanceBatchStageTest method when_hashJoinRebalanceMainStage_then_distributedEdge.
@Test
public void when_hashJoinRebalanceMainStage_then_distributedEdge() {
// Given
List<Integer> input = sequence(itemCount);
String prefix = "value-";
BatchStage<Integer> stage0Rebalanced = batchStageFromList(input).rebalance();
BatchStage<Entry<Integer, String>> enrichingStage = batchStageFromList(input).map(i -> entry(i, prefix + i));
// When
BatchStage<Entry<Integer, String>> joined = stage0Rebalanced.hashJoin(enrichingStage, joinMapEntries(wholeItem()), // Method reference avoided due to JDK bug
(k, v) -> entry(k, v));
// Then
joined.writeTo(sink);
DAG dag = p.toDag();
Edge stage0ToJoin = dag.getInboundEdges("2-way hash-join-joiner").get(0);
assertTrue("Rebalancing should make the edge distributed", stage0ToJoin.isDistributed());
assertNull("Didn't rebalance by key, the edge must not be partitioned", stage0ToJoin.getPartitioner());
execute();
Function<Entry<Integer, String>, String> formatFn = e -> String.format("(%04d, %s)", e.getKey(), e.getValue());
assertEquals(streamToString(input.stream().map(i -> tuple2(i, prefix + i)), formatFn), streamToString(sinkStreamOfEntry(), formatFn));
}
Aggregations