Examples with VertexParallelismInformation org.apache.flink.runtime.scheduler.VertexParallelismInformation used on opensource projects

Example 1 with VertexParallelismInformation

use of org.apache.flink.runtime.scheduler.VertexParallelismInformation in project flink by apache.

the class ExecutionJobVertexTest method createDynamicExecutionJobVertex.

public static ExecutionJobVertex createDynamicExecutionJobVertex(int parallelism, int maxParallelism, int defaultMaxParallelism) throws Exception {
    JobVertex jobVertex = new JobVertex("testVertex");
    jobVertex.setInvokableClass(AbstractInvokable.class);
    jobVertex.createAndAddResultDataSet(new IntermediateDataSetID(), ResultPartitionType.BLOCKING);
    if (maxParallelism > 0) {
        jobVertex.setMaxParallelism(maxParallelism);
    }
    if (parallelism > 0) {
        jobVertex.setParallelism(parallelism);
    }
    final DefaultExecutionGraph eg = TestingDefaultExecutionGraphBuilder.newBuilder().build();
    final VertexParallelismStore vertexParallelismStore = AdaptiveBatchScheduler.computeVertexParallelismStoreForDynamicGraph(Collections.singletonList(jobVertex), defaultMaxParallelism);
    final VertexParallelismInformation vertexParallelismInfo = vertexParallelismStore.getParallelismInfo(jobVertex.getID());
    return new ExecutionJobVertex(eg, jobVertex, vertexParallelismInfo);
}

Example 2 with VertexParallelismInformation

use of org.apache.flink.runtime.scheduler.VertexParallelismInformation in project flink by apache.

the class AdaptiveScheduler method createExecutionGraphWithAvailableResourcesAsync.

private CompletableFuture<CreatingExecutionGraph.ExecutionGraphWithVertexParallelism> createExecutionGraphWithAvailableResourcesAsync() {
    final VertexParallelism vertexParallelism;
    final VertexParallelismStore adjustedParallelismStore;
    try {
        vertexParallelism = determineParallelism(slotAllocator);
        JobGraph adjustedJobGraph = jobInformation.copyJobGraph();
        for (JobVertex vertex : adjustedJobGraph.getVertices()) {
            JobVertexID id = vertex.getID();
            // use the determined "available parallelism" to use
            // the resources we have access to
            vertex.setParallelism(vertexParallelism.getParallelism(id));
        }
        // use the originally configured max parallelism
        // as the default for consistent runs
        adjustedParallelismStore = computeVertexParallelismStoreForExecution(adjustedJobGraph, executionMode, (vertex) -> {
            VertexParallelismInformation vertexParallelismInfo = initialParallelismStore.getParallelismInfo(vertex.getID());
            return vertexParallelismInfo.getMaxParallelism();
        });
    } catch (Exception exception) {
        return FutureUtils.completedExceptionally(exception);
    }
    return createExecutionGraphAndRestoreStateAsync(adjustedParallelismStore).thenApply(executionGraph -> CreatingExecutionGraph.ExecutionGraphWithVertexParallelism.create(executionGraph, vertexParallelism));
}

Example 3 with VertexParallelismInformation

use of org.apache.flink.runtime.scheduler.VertexParallelismInformation in project flink by apache.

the class AdaptiveSchedulerComputeReactiveModeVertexParallelismTest method testCreateStoreWithoutAdjustedParallelism.

@Test
public void testCreateStoreWithoutAdjustedParallelism() {
    JobVertex jobVertex = createNoOpVertex("test", parallelism, maxParallelism);
    VertexParallelismStore store = AdaptiveScheduler.computeReactiveModeVertexParallelismStore(Collections.singleton(jobVertex), SchedulerBase::getDefaultMaxParallelism, false);
    VertexParallelismInformation info = store.getParallelismInfo(jobVertex.getID());
    Assert.assertEquals("parallelism is not adjusted", parallelism, info.getParallelism());
    Assert.assertEquals("expected max", expectedMaxParallelism, info.getMaxParallelism());
    Assert.assertEquals("can rescale max", expectedCanRescaleTo, info.canRescaleMaxParallelism(maxToScaleTo));
}

Example 4 with VertexParallelismInformation

use of org.apache.flink.runtime.scheduler.VertexParallelismInformation in project flink by apache.

the class DefaultExecutionGraph method attachJobVertices.

/**
 * Attach job vertices without initializing them.
 */
private void attachJobVertices(List<JobVertex> topologicallySorted) throws JobException {
    for (JobVertex jobVertex : topologicallySorted) {
        if (jobVertex.isInputVertex() && !jobVertex.isStoppable()) {
            this.isStoppable = false;
        }
        VertexParallelismInformation parallelismInfo = parallelismStore.getParallelismInfo(jobVertex.getID());
        // create the execution job vertex and attach it to the graph
        ExecutionJobVertex ejv = new ExecutionJobVertex(this, jobVertex, parallelismInfo);
        ExecutionJobVertex previousTask = this.tasks.putIfAbsent(jobVertex.getID(), ejv);
        if (previousTask != null) {
            throw new JobException(String.format("Encountered two job vertices with ID %s : previous=[%s] / new=[%s]", jobVertex.getID(), ejv, previousTask));
        }
        this.verticesInCreationOrder.add(ejv);
        this.numJobVerticesTotal++;
    }
}

Example 5 with VertexParallelismInformation

use of org.apache.flink.runtime.scheduler.VertexParallelismInformation in project flink by apache.

the class AdaptiveSchedulerComputeReactiveModeVertexParallelismTest method testCreateStoreWithAdjustedParallelism.

@Test
public void testCreateStoreWithAdjustedParallelism() {
    JobVertex jobVertex = createNoOpVertex("test", parallelism, maxParallelism);
    VertexParallelismStore store = AdaptiveScheduler.computeReactiveModeVertexParallelismStore(Collections.singleton(jobVertex), SchedulerBase::getDefaultMaxParallelism, true);
    VertexParallelismInformation info = store.getParallelismInfo(jobVertex.getID());
    Assert.assertEquals("parallelism is adjusted to max", expectedMaxParallelism, info.getParallelism());
    Assert.assertEquals("expected max", expectedMaxParallelism, info.getMaxParallelism());
    Assert.assertEquals("can rescale max", expectedCanRescaleTo, info.canRescaleMaxParallelism(maxToScaleTo));
}