Examples with DistributionPattern org.apache.flink.runtime.jobgraph.DistributionPattern used on opensource projects

Example 1 with DistributionPattern

use of org.apache.flink.runtime.jobgraph.DistributionPattern in project flink by apache.

the class ExecutionVertexLocalityTest method createTestGraph.

// ------------------------------------------------------------------------
//  Utilities
// ------------------------------------------------------------------------
/**
	 * Creates a simple 2 vertex graph with a parallel source and a parallel target.
	 */
private ExecutionGraph createTestGraph(int parallelism, boolean allToAll) throws Exception {
    JobVertex source = new JobVertex("source", sourceVertexId);
    source.setParallelism(parallelism);
    source.setInvokableClass(NoOpInvokable.class);
    JobVertex target = new JobVertex("source", targetVertexId);
    target.setParallelism(parallelism);
    target.setInvokableClass(NoOpInvokable.class);
    DistributionPattern connectionPattern = allToAll ? DistributionPattern.ALL_TO_ALL : DistributionPattern.POINTWISE;
    target.connectNewDataSetAsInput(source, connectionPattern, ResultPartitionType.PIPELINED);
    JobGraph testJob = new JobGraph(jobId, "test job", source, target);
    return ExecutionGraphBuilder.buildGraph(null, testJob, new Configuration(), TestingUtils.defaultExecutor(), TestingUtils.defaultExecutor(), mock(SlotProvider.class), getClass().getClassLoader(), new StandaloneCheckpointRecoveryFactory(), Time.of(10, TimeUnit.SECONDS), new FixedDelayRestartStrategy(10, 0L), new UnregisteredMetricsGroup(), 1, log);
}

Example 2 with DistributionPattern

use of org.apache.flink.runtime.jobgraph.DistributionPattern in project flink by apache.

the class ExecutionVertex method connectSource.

// --------------------------------------------------------------------------------------------
//  Graph building
// --------------------------------------------------------------------------------------------
public void connectSource(int inputNumber, IntermediateResult source, JobEdge edge, int consumerNumber) {
    final DistributionPattern pattern = edge.getDistributionPattern();
    final IntermediateResultPartition[] sourcePartitions = source.getPartitions();
    ExecutionEdge[] edges;
    switch(pattern) {
        case POINTWISE:
            edges = connectPointwise(sourcePartitions, inputNumber);
            break;
        case ALL_TO_ALL:
            edges = connectAllToAll(sourcePartitions, inputNumber);
            break;
        default:
            throw new RuntimeException("Unrecognized distribution pattern.");
    }
    this.inputEdges[inputNumber] = edges;
    // edges as the execution graph
    for (ExecutionEdge ee : edges) {
        ee.getSource().addConsumer(ee, consumerNumber);
    }
}

Example 3 with DistributionPattern

use of org.apache.flink.runtime.jobgraph.DistributionPattern in project flink by apache.

the class JobGraphGenerator method translateChannel.

private int translateChannel(Channel input, int inputIndex, JobVertex targetVertex, TaskConfig targetVertexConfig, boolean isBroadcast) throws Exception {
    final PlanNode inputPlanNode = input.getSource();
    final Iterator<Channel> allInChannels;
    if (inputPlanNode instanceof NAryUnionPlanNode) {
        allInChannels = ((NAryUnionPlanNode) inputPlanNode).getListOfInputs().iterator();
        // deadlocks when closing a branching flow at runtime.
        for (Channel in : inputPlanNode.getInputs()) {
            if (input.getDataExchangeMode().equals(DataExchangeMode.BATCH)) {
                in.setDataExchangeMode(DataExchangeMode.BATCH);
            }
            if (isBroadcast) {
                in.setShipStrategy(ShipStrategyType.BROADCAST, in.getDataExchangeMode());
            }
        }
        // The outgoing connection of an NAryUnion must be a forward connection.
        if (input.getShipStrategy() != ShipStrategyType.FORWARD && !isBroadcast) {
            throw new CompilerException("Optimized plan contains Union with non-forward outgoing ship strategy.");
        }
    } else if (inputPlanNode instanceof BulkPartialSolutionPlanNode) {
        if (this.vertices.get(inputPlanNode) == null) {
            // merged iteration head
            final BulkPartialSolutionPlanNode pspn = (BulkPartialSolutionPlanNode) inputPlanNode;
            final BulkIterationPlanNode iterationNode = pspn.getContainingIterationNode();
            // check if the iteration's input is a union
            if (iterationNode.getInput().getSource() instanceof NAryUnionPlanNode) {
                allInChannels = (iterationNode.getInput().getSource()).getInputs().iterator();
            } else {
                allInChannels = Collections.singletonList(iterationNode.getInput()).iterator();
            }
            // also, set the index of the gate with the partial solution
            targetVertexConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(inputIndex);
        } else {
            // standalone iteration head
            allInChannels = Collections.singletonList(input).iterator();
        }
    } else if (inputPlanNode instanceof WorksetPlanNode) {
        if (this.vertices.get(inputPlanNode) == null) {
            // merged iteration head
            final WorksetPlanNode wspn = (WorksetPlanNode) inputPlanNode;
            final WorksetIterationPlanNode iterationNode = wspn.getContainingIterationNode();
            // check if the iteration's input is a union
            if (iterationNode.getInput2().getSource() instanceof NAryUnionPlanNode) {
                allInChannels = (iterationNode.getInput2().getSource()).getInputs().iterator();
            } else {
                allInChannels = Collections.singletonList(iterationNode.getInput2()).iterator();
            }
            // also, set the index of the gate with the partial solution
            targetVertexConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(inputIndex);
        } else {
            // standalone iteration head
            allInChannels = Collections.singletonList(input).iterator();
        }
    } else if (inputPlanNode instanceof SolutionSetPlanNode) {
        // rather than a vertex connection
        return 0;
    } else {
        allInChannels = Collections.singletonList(input).iterator();
    }
    // check that the type serializer is consistent
    TypeSerializerFactory<?> typeSerFact = null;
    // accounting for channels on the dynamic path
    int numChannelsTotal = 0;
    int numChannelsDynamicPath = 0;
    int numDynamicSenderTasksTotal = 0;
    // source
    while (allInChannels.hasNext()) {
        final Channel inConn = allInChannels.next();
        // sanity check the common serializer
        if (typeSerFact == null) {
            typeSerFact = inConn.getSerializer();
        } else if (!typeSerFact.equals(inConn.getSerializer())) {
            throw new CompilerException("Conflicting types in union operator.");
        }
        final PlanNode sourceNode = inConn.getSource();
        JobVertex sourceVertex = this.vertices.get(sourceNode);
        TaskConfig sourceVertexConfig;
        if (sourceVertex == null) {
            // this predecessor is chained to another task or an iteration
            final TaskInChain chainedTask;
            final IterationDescriptor iteration;
            if ((chainedTask = this.chainedTasks.get(sourceNode)) != null) {
                // push chained task
                if (chainedTask.getContainingVertex() == null) {
                    throw new IllegalStateException("Bug: Chained task has not been assigned its containing vertex when connecting.");
                }
                sourceVertex = chainedTask.getContainingVertex();
                sourceVertexConfig = chainedTask.getTaskConfig();
            } else if ((iteration = this.iterations.get(sourceNode)) != null) {
                // predecessor is an iteration
                sourceVertex = iteration.getHeadTask();
                sourceVertexConfig = iteration.getHeadFinalResultConfig();
            } else {
                throw new CompilerException("Bug: Could not resolve source node for a channel.");
            }
        } else {
            // predecessor is its own vertex
            sourceVertexConfig = new TaskConfig(sourceVertex.getConfiguration());
        }
        DistributionPattern pattern = connectJobVertices(inConn, inputIndex, sourceVertex, sourceVertexConfig, targetVertex, targetVertexConfig, isBroadcast);
        // accounting on channels and senders
        numChannelsTotal++;
        if (inConn.isOnDynamicPath()) {
            numChannelsDynamicPath++;
            numDynamicSenderTasksTotal += getNumberOfSendersPerReceiver(pattern, sourceVertex.getParallelism(), targetVertex.getParallelism());
        }
    }
    // is a union between nodes on the static and nodes on the dynamic path
    if (numChannelsDynamicPath > 0 && numChannelsTotal != numChannelsDynamicPath) {
        throw new CompilerException("Error: It is currently not supported to union between dynamic and static path in an iteration.");
    }
    if (numDynamicSenderTasksTotal > 0) {
        if (isBroadcast) {
            targetVertexConfig.setBroadcastGateIterativeWithNumberOfEventsUntilInterrupt(inputIndex, numDynamicSenderTasksTotal);
        } else {
            targetVertexConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(inputIndex, numDynamicSenderTasksTotal);
        }
    }
    // the local strategy is added only once. in non-union case that is the actual edge,
    // in the union case, it is the edge between union and the target node
    addLocalInfoFromChannelToConfig(input, targetVertexConfig, inputIndex, isBroadcast);
    return 1;
}

Example 4 with DistributionPattern

use of org.apache.flink.runtime.jobgraph.DistributionPattern in project flink by apache.

the class JobGraphGenerator method connectJobVertices.

// ------------------------------------------------------------------------
// Connecting Vertices
// ------------------------------------------------------------------------
/**
 * NOTE: The channel for global and local strategies are different if we connect a union. The
 * global strategy channel is then the channel into the union node, the local strategy channel
 * the one from the union to the actual target operator.
 *
 * @throws CompilerException
 */
private DistributionPattern connectJobVertices(Channel channel, int inputNumber, final JobVertex sourceVertex, final TaskConfig sourceConfig, final JobVertex targetVertex, final TaskConfig targetConfig, boolean isBroadcast) throws CompilerException {
    // ------------ connect the vertices to the job graph --------------
    final DistributionPattern distributionPattern;
    switch(channel.getShipStrategy()) {
        case FORWARD:
            distributionPattern = DistributionPattern.POINTWISE;
            break;
        case PARTITION_RANDOM:
        case BROADCAST:
        case PARTITION_HASH:
        case PARTITION_CUSTOM:
        case PARTITION_RANGE:
        case PARTITION_FORCED_REBALANCE:
            distributionPattern = DistributionPattern.ALL_TO_ALL;
            break;
        default:
            throw new RuntimeException("Unknown runtime ship strategy: " + channel.getShipStrategy());
    }
    final ResultPartitionType resultType;
    switch(channel.getDataExchangeMode()) {
        case PIPELINED:
            resultType = ResultPartitionType.PIPELINED;
            break;
        case BATCH:
            // BLOCKING results are currently not supported in closed loop iterations
            // 
            // See https://issues.apache.org/jira/browse/FLINK-1713 for details
            resultType = channel.getSource().isOnDynamicPath() ? ResultPartitionType.PIPELINED : ResultPartitionType.BLOCKING;
            break;
        case PIPELINE_WITH_BATCH_FALLBACK:
            throw new UnsupportedOperationException("Data exchange mode " + channel.getDataExchangeMode() + " currently not supported.");
        default:
            throw new UnsupportedOperationException("Unknown data exchange mode.");
    }
    JobEdge edge = targetVertex.connectNewDataSetAsInput(sourceVertex, distributionPattern, resultType);
    // -------------- configure the source task's ship strategy strategies in task config
    // --------------
    final int outputIndex = sourceConfig.getNumOutputs();
    sourceConfig.addOutputShipStrategy(channel.getShipStrategy());
    if (outputIndex == 0) {
        sourceConfig.setOutputSerializer(channel.getSerializer());
    }
    if (channel.getShipStrategyComparator() != null) {
        sourceConfig.setOutputComparator(channel.getShipStrategyComparator(), outputIndex);
    }
    if (channel.getShipStrategy() == ShipStrategyType.PARTITION_RANGE) {
        final DataDistribution dataDistribution = channel.getDataDistribution();
        if (dataDistribution != null) {
            sourceConfig.setOutputDataDistribution(dataDistribution, outputIndex);
        } else {
            throw new RuntimeException("Range partitioning requires data distribution.");
        }
    }
    if (channel.getShipStrategy() == ShipStrategyType.PARTITION_CUSTOM) {
        if (channel.getPartitioner() != null) {
            sourceConfig.setOutputPartitioner(channel.getPartitioner(), outputIndex);
        } else {
            throw new CompilerException("The ship strategy was set to custom partitioning, but no partitioner was set.");
        }
    }
    // ---------------- configure the receiver -------------------
    if (isBroadcast) {
        targetConfig.addBroadcastInputToGroup(inputNumber);
    } else {
        targetConfig.addInputToGroup(inputNumber);
    }
    // ---------------- attach the additional infos to the job edge -------------------
    String shipStrategy = JsonMapper.getShipStrategyString(channel.getShipStrategy());
    if (channel.getShipStrategyKeys() != null && channel.getShipStrategyKeys().size() > 0) {
        shipStrategy += " on " + (channel.getShipStrategySortOrder() == null ? channel.getShipStrategyKeys().toString() : Utils.createOrdering(channel.getShipStrategyKeys(), channel.getShipStrategySortOrder()).toString());
    }
    String localStrategy;
    if (channel.getLocalStrategy() == null || channel.getLocalStrategy() == LocalStrategy.NONE) {
        localStrategy = null;
    } else {
        localStrategy = JsonMapper.getLocalStrategyString(channel.getLocalStrategy());
        if (localStrategy != null && channel.getLocalStrategyKeys() != null && channel.getLocalStrategyKeys().size() > 0) {
            localStrategy += " on " + (channel.getLocalStrategySortOrder() == null ? channel.getLocalStrategyKeys().toString() : Utils.createOrdering(channel.getLocalStrategyKeys(), channel.getLocalStrategySortOrder()).toString());
        }
    }
    String caching = channel.getTempMode() == TempMode.NONE ? null : channel.getTempMode().toString();
    edge.setShipStrategyName(shipStrategy);
    edge.setBroadcast(isBroadcast);
    edge.setForward(channel.getShipStrategy() == ShipStrategyType.FORWARD);
    edge.setPreProcessingOperationName(localStrategy);
    edge.setOperatorLevelCachingDescription(caching);
    return distributionPattern;
}

Example 5 with DistributionPattern

use of org.apache.flink.runtime.jobgraph.DistributionPattern in project flink by apache.

the class IntermediateResultPartition method computeNumberOfMaxPossiblePartitionConsumers.

private int computeNumberOfMaxPossiblePartitionConsumers() {
    final ExecutionJobVertex consumerJobVertex = getIntermediateResult().getConsumerExecutionJobVertex();
    final DistributionPattern distributionPattern = getIntermediateResult().getConsumingDistributionPattern();
    // decide the max possible consumer job vertex parallelism
    int maxConsumerJobVertexParallelism = consumerJobVertex.getParallelism();
    if (maxConsumerJobVertexParallelism <= 0) {
        checkState(consumerJobVertex.getMaxParallelism() > 0, "Neither the parallelism nor the max parallelism of a job vertex is set");
        maxConsumerJobVertexParallelism = consumerJobVertex.getMaxParallelism();
    }
    // compute number of subpartitions according to the distribution pattern
    if (distributionPattern == DistributionPattern.ALL_TO_ALL) {
        return maxConsumerJobVertexParallelism;
    } else {
        int numberOfPartitions = getIntermediateResult().getNumParallelProducers();
        return (int) Math.ceil(((double) maxConsumerJobVertexParallelism) / numberOfPartitions);
    }
}