Examples with BatchExecExchange org.apache.flink.table.planner.plan.nodes.exec.batch.BatchExecExchange used on opensource projects

Example 1 with BatchExecExchange

use of org.apache.flink.table.planner.plan.nodes.exec.batch.BatchExecExchange in project flink by apache.

the class InputPriorityConflictResolverTest method testDetectAndResolve.

@Test
public void testDetectAndResolve() {
    // P = InputProperty.DamBehavior.PIPELINED, E = InputProperty.DamBehavior.END_INPUT
    // P100 = PIPELINED + priority 100
    // 
    // 0 --------(P0)----> 1 --(P0)-----------> 7
    // \                    \-(P0)-> 2 -(P0)--/
    // \-------(P0)----> 3 --(P1)-----------/
    // \------(P0)----> 4 --(P10)---------/
    // \              /                 /
    // \    8 -(P0)-<                 /
    // \            \               /
    // \--(E0)----> 5 --(P10)-----/
    // 6 ---------(P100)----------------/
    TestingBatchExecNode[] nodes = new TestingBatchExecNode[9];
    for (int i = 0; i < nodes.length; i++) {
        nodes[i] = new TestingBatchExecNode("TestingBatchExecNode" + i);
    }
    nodes[1].addInput(nodes[0], InputProperty.builder().priority(0).build());
    nodes[2].addInput(nodes[1], InputProperty.builder().priority(0).build());
    nodes[3].addInput(nodes[0], InputProperty.builder().priority(0).build());
    nodes[4].addInput(nodes[8], InputProperty.builder().priority(0).build());
    nodes[4].addInput(nodes[0], InputProperty.builder().priority(0).build());
    nodes[5].addInput(nodes[8], InputProperty.builder().priority(0).build());
    nodes[5].addInput(nodes[0], InputProperty.builder().damBehavior(InputProperty.DamBehavior.END_INPUT).priority(0).build());
    nodes[7].addInput(nodes[1], InputProperty.builder().priority(0).build());
    nodes[7].addInput(nodes[2], InputProperty.builder().priority(0).build());
    nodes[7].addInput(nodes[3], InputProperty.builder().priority(1).build());
    nodes[7].addInput(nodes[4], InputProperty.builder().priority(10).build());
    nodes[7].addInput(nodes[5], InputProperty.builder().priority(10).build());
    nodes[7].addInput(nodes[6], InputProperty.builder().priority(100).build());
    InputPriorityConflictResolver resolver = new InputPriorityConflictResolver(Collections.singletonList(nodes[7]), InputProperty.DamBehavior.END_INPUT, StreamExchangeMode.BATCH, new Configuration());
    resolver.detectAndResolve();
    Assert.assertEquals(nodes[1], nodes[7].getInputNodes().get(0));
    Assert.assertEquals(nodes[2], nodes[7].getInputNodes().get(1));
    Assert.assertTrue(nodes[7].getInputNodes().get(2) instanceof BatchExecExchange);
    Assert.assertEquals(Optional.of(StreamExchangeMode.BATCH), ((BatchExecExchange) nodes[7].getInputNodes().get(2)).getRequiredExchangeMode());
    Assert.assertEquals(nodes[3], nodes[7].getInputNodes().get(2).getInputEdges().get(0).getSource());
    Assert.assertTrue(nodes[7].getInputNodes().get(3) instanceof BatchExecExchange);
    Assert.assertEquals(Optional.of(StreamExchangeMode.BATCH), ((BatchExecExchange) nodes[7].getInputNodes().get(3)).getRequiredExchangeMode());
    Assert.assertEquals(nodes[4], nodes[7].getInputNodes().get(3).getInputEdges().get(0).getSource());
    Assert.assertEquals(nodes[5], nodes[7].getInputNodes().get(4));
    Assert.assertEquals(nodes[6], nodes[7].getInputNodes().get(5));
}

Example 2 with BatchExecExchange

use of org.apache.flink.table.planner.plan.nodes.exec.batch.BatchExecExchange in project flink by apache.

the class InputPriorityConflictResolver method resolveInputPriorityConflict.

@Override
protected void resolveInputPriorityConflict(ExecNode<?> node, int higherInput, int lowerInput) {
    ExecNode<?> higherNode = node.getInputEdges().get(higherInput).getSource();
    ExecNode<?> lowerNode = node.getInputEdges().get(lowerInput).getSource();
    final ExecNode<?> newNode;
    if (lowerNode instanceof BatchExecExchange) {
        BatchExecExchange exchange = (BatchExecExchange) lowerNode;
        InputProperty inputEdge = exchange.getInputProperties().get(0);
        InputProperty inputProperty = InputProperty.builder().requiredDistribution(inputEdge.getRequiredDistribution()).priority(inputEdge.getPriority()).damBehavior(getDamBehavior()).build();
        if (isConflictCausedByExchange(higherNode, exchange)) {
            // special case: if exchange is exactly the reuse node,
            // we should split it into two nodes
            BatchExecExchange newExchange = new BatchExecExchange(inputProperty, (RowType) exchange.getOutputType(), "Exchange");
            newExchange.setRequiredExchangeMode(exchangeMode);
            newExchange.setInputEdges(exchange.getInputEdges());
            newNode = newExchange;
        } else {
            // create new BatchExecExchange with new inputProperty
            BatchExecExchange newExchange = new BatchExecExchange(inputProperty, (RowType) exchange.getOutputType(), exchange.getDescription());
            newExchange.setRequiredExchangeMode(exchangeMode);
            newExchange.setInputEdges(exchange.getInputEdges());
            newNode = newExchange;
        }
    } else {
        newNode = createExchange(node, lowerInput);
    }
    ExecEdge newEdge = ExecEdge.builder().source(newNode).target(node).build();
    node.replaceInputEdge(lowerInput, newEdge);
}

Example 3 with BatchExecExchange

use of org.apache.flink.table.planner.plan.nodes.exec.batch.BatchExecExchange in project flink by apache.

the class ForwardHashExchangeProcessor method addExchangeAndReconnectEdge.

// TODO This implementation should be updated once FLINK-21224 is finished.
private ExecEdge addExchangeAndReconnectEdge(ExecEdge edge, InputProperty inputProperty, boolean strict) {
    ExecNode<?> target = edge.getTarget();
    ExecNode<?> source = edge.getSource();
    if (source instanceof CommonExecExchange) {
        return edge;
    }
    // only Calc, Correlate and Sort can propagate sort property and distribution property
    if (source instanceof BatchExecCalc || source instanceof BatchExecPythonCalc || source instanceof BatchExecSort || source instanceof BatchExecCorrelate || source instanceof BatchExecPythonCorrelate) {
        ExecEdge newEdge = addExchangeAndReconnectEdge(source.getInputEdges().get(0), inputProperty, strict);
        source.setInputEdges(Collections.singletonList(newEdge));
    }
    BatchExecExchange exchange = createExchangeWithKeepInputAsIsDistribution(inputProperty, strict, (RowType) edge.getOutputType());
    ExecEdge newEdge = new ExecEdge(source, exchange, edge.getShuffle(), edge.getExchangeMode());
    exchange.setInputEdges(Collections.singletonList(newEdge));
    return new ExecEdge(exchange, target, edge.getShuffle(), edge.getExchangeMode());
}

Example 4 with BatchExecExchange

use of org.apache.flink.table.planner.plan.nodes.exec.batch.BatchExecExchange in project flink by apache.

the class InputPriorityConflictResolver method createExchange.

private BatchExecExchange createExchange(ExecNode<?> node, int idx) {
    ExecNode<?> inputNode = node.getInputEdges().get(idx).getSource();
    InputProperty inputProperty = node.getInputProperties().get(idx);
    InputProperty.RequiredDistribution requiredDistribution = inputProperty.getRequiredDistribution();
    if (requiredDistribution.getType() == InputProperty.DistributionType.BROADCAST) {
        // should not occur
        throw new IllegalStateException("Trying to resolve input priority conflict on broadcast side. This is not expected.");
    }
    InputProperty newInputProperty = InputProperty.builder().requiredDistribution(requiredDistribution).priority(inputProperty.getPriority()).damBehavior(getDamBehavior()).build();
    BatchExecExchange exchange = new BatchExecExchange(newInputProperty, (RowType) inputNode.getOutputType(), "Exchange");
    exchange.setRequiredExchangeMode(exchangeMode);
    ExecEdge execEdge = ExecEdge.builder().source(inputNode).target(exchange).build();
    exchange.setInputEdges(Collections.singletonList(execEdge));
    return exchange;
}

Example 5 with BatchExecExchange

use of org.apache.flink.table.planner.plan.nodes.exec.batch.BatchExecExchange in project flink by apache.

the class InputPriorityConflictResolverTest method testDeadlockCausedByExchange.

@Test
public void testDeadlockCausedByExchange() {
    // P1 = PIPELINED + priority 1
    // 
    // 0 -(P0)-> exchange --(P0)-> 1
    // \-(P1)-/
    TestingBatchExecNode[] nodes = new TestingBatchExecNode[2];
    for (int i = 0; i < nodes.length; i++) {
        nodes[i] = new TestingBatchExecNode("TestingBatchExecNode" + i);
    }
    BatchExecExchange exchange = new BatchExecExchange(InputProperty.builder().requiredDistribution(InputProperty.ANY_DISTRIBUTION).build(), (RowType) nodes[0].getOutputType(), "Exchange");
    exchange.setRequiredExchangeMode(StreamExchangeMode.BATCH);
    ExecEdge execEdge = ExecEdge.builder().source(nodes[0]).target(exchange).build();
    exchange.setInputEdges(Collections.singletonList(execEdge));
    nodes[1].addInput(exchange, InputProperty.builder().priority(0).build());
    nodes[1].addInput(exchange, InputProperty.builder().priority(1).build());
    InputPriorityConflictResolver resolver = new InputPriorityConflictResolver(Collections.singletonList(nodes[1]), InputProperty.DamBehavior.END_INPUT, StreamExchangeMode.BATCH, new Configuration());
    resolver.detectAndResolve();
    ExecNode<?> input0 = nodes[1].getInputNodes().get(0);
    ExecNode<?> input1 = nodes[1].getInputNodes().get(1);
    Assert.assertNotSame(input0, input1);
    Consumer<ExecNode<?>> checkExchange = execNode -> {
        Assert.assertTrue(execNode instanceof BatchExecExchange);
        BatchExecExchange e = (BatchExecExchange) execNode;
        Assert.assertEquals(Optional.of(StreamExchangeMode.BATCH), e.getRequiredExchangeMode());
        Assert.assertEquals(nodes[0], e.getInputEdges().get(0).getSource());
    };
    checkExchange.accept(input0);
    checkExchange.accept(input1);
}