use of org.apache.flink.table.planner.plan.nodes.exec.InputProperty.RequiredDistribution in project flink by apache.
the class BatchExecExchange method translateToPlanInternal.
@SuppressWarnings("unchecked")
@Override
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
final ExecEdge inputEdge = getInputEdges().get(0);
final Transformation<RowData> inputTransform = (Transformation<RowData>) inputEdge.translateToPlan(planner);
final RowType inputType = (RowType) inputEdge.getOutputType();
boolean requireUndefinedExchangeMode = false;
final StreamPartitioner<RowData> partitioner;
final int parallelism;
final InputProperty inputProperty = getInputProperties().get(0);
final RequiredDistribution requiredDistribution = inputProperty.getRequiredDistribution();
final InputProperty.DistributionType distributionType = requiredDistribution.getType();
switch(distributionType) {
case ANY:
partitioner = null;
parallelism = ExecutionConfig.PARALLELISM_DEFAULT;
break;
case BROADCAST:
partitioner = new BroadcastPartitioner<>();
parallelism = ExecutionConfig.PARALLELISM_DEFAULT;
break;
case SINGLETON:
partitioner = new GlobalPartitioner<>();
parallelism = 1;
break;
case HASH:
partitioner = createHashPartitioner(((HashDistribution) requiredDistribution), inputType, config);
parallelism = ExecutionConfig.PARALLELISM_DEFAULT;
break;
case KEEP_INPUT_AS_IS:
KeepInputAsIsDistribution keepInputAsIsDistribution = (KeepInputAsIsDistribution) requiredDistribution;
if (keepInputAsIsDistribution.isStrict()) {
// explicitly use ForwardPartitioner to guarantee the data distribution is
// exactly the same as input
partitioner = new ForwardPartitioner<>();
requireUndefinedExchangeMode = true;
} else {
RequiredDistribution inputDistribution = ((KeepInputAsIsDistribution) requiredDistribution).getInputDistribution();
checkArgument(inputDistribution instanceof HashDistribution, "Only HashDistribution is supported now");
partitioner = new ForwardForConsecutiveHashPartitioner<>(createHashPartitioner(((HashDistribution) inputDistribution), inputType, config));
}
parallelism = inputTransform.getParallelism();
break;
default:
throw new TableException(distributionType + "is not supported now!");
}
final StreamExchangeMode exchangeMode = requireUndefinedExchangeMode ? StreamExchangeMode.UNDEFINED : getBatchStreamExchangeMode(config, requiredExchangeMode);
final Transformation<RowData> transformation = new PartitionTransformation<>(inputTransform, partitioner, exchangeMode);
transformation.setParallelism(parallelism);
transformation.setOutputType(InternalTypeInfo.of(getOutputType()));
return transformation;
}
use of org.apache.flink.table.planner.plan.nodes.exec.InputProperty.RequiredDistribution in project flink by apache.
the class BatchExecExchange method getDescription.
@Override
public String getDescription() {
// make sure the description be consistent with before, update this once plan is stable
RequiredDistribution requiredDistribution = getInputProperties().get(0).getRequiredDistribution();
StringBuilder sb = new StringBuilder();
String type = requiredDistribution.getType().name().toLowerCase();
if (type.equals("singleton")) {
type = "single";
} else if (requiredDistribution instanceof KeepInputAsIsDistribution && ((KeepInputAsIsDistribution) requiredDistribution).isStrict()) {
type = "forward";
}
sb.append("distribution=[").append(type);
if (requiredDistribution instanceof HashDistribution) {
sb.append(getHashDistributionDescription((HashDistribution) requiredDistribution));
} else if (requiredDistribution instanceof KeepInputAsIsDistribution && !((KeepInputAsIsDistribution) requiredDistribution).isStrict()) {
KeepInputAsIsDistribution distribution = (KeepInputAsIsDistribution) requiredDistribution;
sb.append("[hash").append(getHashDistributionDescription((HashDistribution) distribution.getInputDistribution())).append("]");
}
sb.append("]");
if (requiredExchangeMode == StreamExchangeMode.BATCH) {
sb.append(", shuffle_mode=[BATCH]");
}
return String.format("Exchange(%s)", sb);
}
use of org.apache.flink.table.planner.plan.nodes.exec.InputProperty.RequiredDistribution in project flink by apache.
the class ForwardHashExchangeProcessor method process.
@Override
public ExecNodeGraph process(ExecNodeGraph execGraph, ProcessorContext context) {
if (execGraph.getRootNodes().get(0) instanceof StreamExecNode) {
throw new TableException("StreamExecNode is not supported yet");
}
if (!context.getPlanner().getExecEnv().getConfig().isDynamicGraph()) {
return execGraph;
}
ExecNodeVisitor visitor = new AbstractExecNodeExactlyOnceVisitor() {
@Override
protected void visitNode(ExecNode<?> node) {
visitInputs(node);
if (node instanceof CommonExecExchange) {
return;
}
boolean changed = false;
List<ExecEdge> newEdges = new ArrayList<>(node.getInputEdges());
for (int i = 0; i < node.getInputProperties().size(); ++i) {
InputProperty inputProperty = node.getInputProperties().get(i);
RequiredDistribution requiredDistribution = inputProperty.getRequiredDistribution();
ExecEdge edge = node.getInputEdges().get(i);
if (requiredDistribution.getType() == DistributionType.SINGLETON) {
if (!hasExchangeInput(edge) && isInputSortedNode(node)) {
// if operation chaining is disabled, this could mark sure the
// sort node and its output can also be connected by
// ForwardPartitioner
ExecEdge newEdge = addExchangeAndReconnectEdge(edge, inputProperty, true);
newEdges.set(i, newEdge);
changed = true;
}
continue;
}
if (requiredDistribution.getType() != DistributionType.HASH) {
continue;
}
if (!hasExchangeInput(edge)) {
ExecEdge newEdge;
if (isInputSortedNode(node)) {
if (hasSortInputForInputSortedNode(node)) {
// add Exchange with keep_input_as_is distribution as the
// input of Sort
ExecNode<?> sort = edge.getSource();
ExecEdge newEdgeOfSort = addExchangeAndReconnectEdge(sort.getInputEdges().get(0), inputProperty, false);
sort.setInputEdges(Collections.singletonList(newEdgeOfSort));
}
// if operation chaining is disabled, this could mark sure the
// sort node and its output can also be connected by
// ForwardPartitioner
newEdge = addExchangeAndReconnectEdge(edge, inputProperty, true);
} else {
// add Exchange with keep_input_as_is distribution as the input
// of the node
newEdge = addExchangeAndReconnectEdge(edge, inputProperty, false);
updateOriginalEdgeInMultipleInput(node, i, (BatchExecExchange) newEdge.getSource());
}
// update the edge
newEdges.set(i, newEdge);
changed = true;
} else if (hasSortInputForInputSortedNode(node)) {
// if operation chaining is disabled, this could mark sure the sort
// node and its output can also be connected by ForwardPartitioner
ExecEdge newEdge = addExchangeAndReconnectEdge(edge, inputProperty, true);
newEdges.set(i, newEdge);
changed = true;
}
}
if (changed) {
node.setInputEdges(newEdges);
}
}
};
execGraph.getRootNodes().forEach(s -> s.accept(visitor));
return execGraph;
}
Aggregations