Examples with JobVertex org.apache.flink.runtime.jobgraph.JobVertex used on opensource projects

Example 76 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class DefaultExecutionTopology method computeLogicalPipelinedRegionsByJobVertexId.

private static Map<JobVertexID, DefaultLogicalPipelinedRegion> computeLogicalPipelinedRegionsByJobVertexId(final ExecutionGraph executionGraph) {
    List<JobVertex> topologicallySortedJobVertices = IterableUtils.toStream(executionGraph.getVerticesTopologically()).map(ExecutionJobVertex::getJobVertex).collect(Collectors.toList());
    Iterable<DefaultLogicalPipelinedRegion> logicalPipelinedRegions = DefaultLogicalTopology.fromTopologicallySortedJobVertices(topologicallySortedJobVertices).getAllPipelinedRegions();
    Map<JobVertexID, DefaultLogicalPipelinedRegion> logicalPipelinedRegionsByJobVertexId = new HashMap<>();
    for (DefaultLogicalPipelinedRegion logicalPipelinedRegion : logicalPipelinedRegions) {
        for (LogicalVertex vertex : logicalPipelinedRegion.getVertices()) {
            logicalPipelinedRegionsByJobVertexId.put(vertex.getId(), logicalPipelinedRegion);
        }
    }
    return logicalPipelinedRegionsByJobVertexId;
}

Example 77 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class AdaptiveScheduler method createExecutionGraphWithAvailableResourcesAsync.

private CompletableFuture<CreatingExecutionGraph.ExecutionGraphWithVertexParallelism> createExecutionGraphWithAvailableResourcesAsync() {
    final VertexParallelism vertexParallelism;
    final VertexParallelismStore adjustedParallelismStore;
    try {
        vertexParallelism = determineParallelism(slotAllocator);
        JobGraph adjustedJobGraph = jobInformation.copyJobGraph();
        for (JobVertex vertex : adjustedJobGraph.getVertices()) {
            JobVertexID id = vertex.getID();
            // use the determined "available parallelism" to use
            // the resources we have access to
            vertex.setParallelism(vertexParallelism.getParallelism(id));
        }
        // use the originally configured max parallelism
        // as the default for consistent runs
        adjustedParallelismStore = computeVertexParallelismStoreForExecution(adjustedJobGraph, executionMode, (vertex) -> {
            VertexParallelismInformation vertexParallelismInfo = initialParallelismStore.getParallelismInfo(vertex.getID());
            return vertexParallelismInfo.getMaxParallelism();
        });
    } catch (Exception exception) {
        return FutureUtils.completedExceptionally(exception);
    }
    return createExecutionGraphAndRestoreStateAsync(adjustedParallelismStore).thenApply(executionGraph -> CreatingExecutionGraph.ExecutionGraphWithVertexParallelism.create(executionGraph, vertexParallelism));
}

Example 78 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class SchedulerBase method computeVertexParallelismStore.

/**
 * Compute the {@link VertexParallelismStore} for all given vertices, which will set defaults
 * and ensure that the returned store contains valid parallelisms, with a custom function for
 * default max parallelism calculation and a custom function for normalizing vertex parallelism.
 *
 * @param vertices the vertices to compute parallelism for
 * @param defaultMaxParallelismFunc a function for computing a default max parallelism if none
 *     is specified on a given vertex
 * @param normalizeParallelismFunc a function for normalizing vertex parallelism
 * @return the computed parallelism store
 */
public static VertexParallelismStore computeVertexParallelismStore(Iterable<JobVertex> vertices, Function<JobVertex, Integer> defaultMaxParallelismFunc, Function<Integer, Integer> normalizeParallelismFunc) {
    DefaultVertexParallelismStore store = new DefaultVertexParallelismStore();
    for (JobVertex vertex : vertices) {
        int parallelism = normalizeParallelismFunc.apply(vertex.getParallelism());
        int maxParallelism = vertex.getMaxParallelism();
        final boolean autoConfigured;
        // if no max parallelism was configured by the user, we calculate and set a default
        if (maxParallelism == JobVertex.MAX_PARALLELISM_DEFAULT) {
            maxParallelism = defaultMaxParallelismFunc.apply(vertex);
            autoConfigured = true;
        } else {
            autoConfigured = false;
        }
        VertexParallelismInformation parallelismInfo = new DefaultVertexParallelismInfo(parallelism, maxParallelism, // user
        (newMax) -> autoConfigured ? Optional.empty() : Optional.of("Cannot override a configured max parallelism."));
        store.setParallelismInfo(vertex.getID(), parallelismInfo);
    }
    return store;
}

Example 79 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class ForwardGroupComputeUtil method computeForwardGroups.

public static Map<JobVertexID, ForwardGroup> computeForwardGroups(final Iterable<JobVertex> topologicallySortedVertices, Function<JobVertexID, ExecutionJobVertex> executionJobVertexRetriever) {
    final Map<JobVertex, Set<JobVertex>> vertexToGroup = new IdentityHashMap<>();
    // iterate all the vertices which are topologically sorted
    for (JobVertex vertex : topologicallySortedVertices) {
        Set<JobVertex> currentGroup = new HashSet<>();
        currentGroup.add(vertex);
        vertexToGroup.put(vertex, currentGroup);
        for (JobEdge input : getForwardInputs(vertex)) {
            final JobVertex producerVertex = input.getSource().getProducer();
            final Set<JobVertex> producerGroup = vertexToGroup.get(producerVertex);
            if (producerGroup == null) {
                throw new IllegalStateException("Producer task " + producerVertex.getID() + " forward group is null" + " while calculating forward group for the consumer task " + vertex.getID() + ". This should be a forward group building bug.");
            }
            if (currentGroup != producerGroup) {
                currentGroup = VertexGroupComputeUtil.mergeVertexGroups(currentGroup, producerGroup, vertexToGroup);
            }
        }
    }
    final Map<JobVertexID, ForwardGroup> ret = new HashMap<>();
    for (Set<JobVertex> vertexGroup : VertexGroupComputeUtil.uniqueVertexGroups(vertexToGroup)) {
        if (vertexGroup.size() > 1) {
            ForwardGroup forwardGroup = new ForwardGroup(vertexGroup.stream().map(vertex -> executionJobVertexRetriever.apply(vertex.getID())).collect(Collectors.toSet()));
            for (JobVertexID jobVertexId : forwardGroup.getJobVertexIds()) {
                ret.put(jobVertexId, forwardGroup);
            }
        }
    }
    return ret;
}

Example 80 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class SsgNetworkMemoryCalculationUtils method buildTaskInputsOutputsDescriptor.

private static TaskInputsOutputsDescriptor buildTaskInputsOutputsDescriptor(ExecutionJobVertex ejv, Function<JobVertexID, ExecutionJobVertex> ejvs) {
    Map<IntermediateDataSetID, Integer> maxInputChannelNums;
    Map<IntermediateDataSetID, Integer> maxSubpartitionNums;
    if (ejv.getGraph().isDynamic()) {
        maxInputChannelNums = getMaxInputChannelNumsForDynamicGraph(ejv);
        maxSubpartitionNums = getMaxSubpartitionNumsForDynamicGraph(ejv);
    } else {
        maxInputChannelNums = getMaxInputChannelNums(ejv);
        maxSubpartitionNums = getMaxSubpartitionNums(ejv, ejvs);
    }
    JobVertex jv = ejv.getJobVertex();
    Map<IntermediateDataSetID, ResultPartitionType> partitionTypes = getPartitionTypes(jv);
    return TaskInputsOutputsDescriptor.from(maxInputChannelNums, maxSubpartitionNums, partitionTypes);
}