Examples with BlockingShuffleOutputFormat org.apache.flink.api.java.io.BlockingShuffleOutputFormat used on opensource projects

Example 1 with BlockingShuffleOutputFormat

use of org.apache.flink.api.java.io.BlockingShuffleOutputFormat in project flink by apache.

the class JobGraphGenerator method checkAndConfigurePersistentIntermediateResult.

private boolean checkAndConfigurePersistentIntermediateResult(PlanNode node) {
    if (!(node instanceof SinkPlanNode)) {
        return false;
    }
    final Object userCodeObject = node.getProgramOperator().getUserCodeWrapper().getUserCodeObject();
    if (!(userCodeObject instanceof BlockingShuffleOutputFormat)) {
        return false;
    }
    final Iterator<Channel> inputIterator = node.getInputs().iterator();
    checkState(inputIterator.hasNext(), "SinkPlanNode must have a input.");
    final PlanNode predecessorNode = inputIterator.next().getSource();
    final JobVertex predecessorVertex = (vertices.containsKey(predecessorNode)) ? vertices.get(predecessorNode) : chainedTasks.get(predecessorNode).getContainingVertex();
    checkState(predecessorVertex != null, "Bug: Chained task has not been assigned its containing vertex when connecting.");
    predecessorVertex.createAndAddResultDataSet(// use specified intermediateDataSetID
    new IntermediateDataSetID(((BlockingShuffleOutputFormat) userCodeObject).getIntermediateDataSetId()), ResultPartitionType.BLOCKING_PERSISTENT);
    // remove this node so the OutputFormatVertex will not shown in the final JobGraph.
    vertices.remove(node);
    return true;
}