Examples with SchedulerBase org.apache.flink.runtime.scheduler.SchedulerBase used on opensource projects

Example 11 with SchedulerBase

use of org.apache.flink.runtime.scheduler.SchedulerBase in project flink by apache.

the class AdaptiveBatchSchedulerTest method testAdaptiveBatchScheduler.

@Test
public void testAdaptiveBatchScheduler() throws Exception {
    JobGraph jobGraph = createJobGraph(false);
    Iterator<JobVertex> jobVertexIterator = jobGraph.getVertices().iterator();
    JobVertex source1 = jobVertexIterator.next();
    JobVertex source2 = jobVertexIterator.next();
    JobVertex sink = jobVertexIterator.next();
    SchedulerBase scheduler = createScheduler(jobGraph);
    final DefaultExecutionGraph graph = (DefaultExecutionGraph) scheduler.getExecutionGraph();
    final ExecutionJobVertex sinkExecutionJobVertex = graph.getJobVertex(sink.getID());
    scheduler.startScheduling();
    assertThat(sinkExecutionJobVertex.getParallelism(), is(-1));
    // trigger source1 finished.
    transitionExecutionsState(scheduler, ExecutionState.FINISHED, source1);
    assertThat(sinkExecutionJobVertex.getParallelism(), is(-1));
    // trigger source2 finished.
    transitionExecutionsState(scheduler, ExecutionState.FINISHED, source2);
    assertThat(sinkExecutionJobVertex.getParallelism(), is(10));
    // check that the jobGraph is updated
    assertThat(sink.getParallelism(), is(10));
}

Example 12 with SchedulerBase

use of org.apache.flink.runtime.scheduler.SchedulerBase in project flink by apache.

the class EdgeManagerTest method testGetConsumedPartitionGroup.

@Test
public void testGetConsumedPartitionGroup() throws Exception {
    JobVertex v1 = new JobVertex("source");
    JobVertex v2 = new JobVertex("sink");
    v1.setParallelism(2);
    v2.setParallelism(2);
    v1.setInvokableClass(NoOpInvokable.class);
    v2.setInvokableClass(NoOpInvokable.class);
    v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
    JobGraph jobGraph = JobGraphTestUtils.batchJobGraph(v1, v2);
    SchedulerBase scheduler = SchedulerTestingUtils.createScheduler(jobGraph, ComponentMainThreadExecutorServiceAdapter.forMainThread());
    ExecutionGraph eg = scheduler.getExecutionGraph();
    ConsumedPartitionGroup groupRetrievedByDownstreamVertex = Objects.requireNonNull(eg.getJobVertex(v2.getID())).getTaskVertices()[0].getAllConsumedPartitionGroups().get(0);
    IntermediateResultPartition consumedPartition = Objects.requireNonNull(eg.getJobVertex(v1.getID())).getProducedDataSets()[0].getPartitions()[0];
    ConsumedPartitionGroup groupRetrievedByIntermediateResultPartition = consumedPartition.getConsumedPartitionGroups().get(0);
    assertEquals(groupRetrievedByDownstreamVertex, groupRetrievedByIntermediateResultPartition);
    ConsumedPartitionGroup groupRetrievedByScheduledResultPartition = scheduler.getExecutionGraph().getSchedulingTopology().getResultPartition(consumedPartition.getPartitionId()).getConsumedPartitionGroups().get(0);
    assertEquals(groupRetrievedByDownstreamVertex, groupRetrievedByScheduledResultPartition);
}

Example 13 with SchedulerBase

use of org.apache.flink.runtime.scheduler.SchedulerBase in project flink by apache.

the class ExecutionGraphFinishTest method testJobFinishes.

@Test
public void testJobFinishes() throws Exception {
    JobGraph jobGraph = JobGraphTestUtils.streamingJobGraph(ExecutionGraphTestUtils.createJobVertex("Task1", 2, NoOpInvokable.class), ExecutionGraphTestUtils.createJobVertex("Task2", 2, NoOpInvokable.class));
    SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(jobGraph, ComponentMainThreadExecutorServiceAdapter.forMainThread()).build();
    ExecutionGraph eg = scheduler.getExecutionGraph();
    scheduler.startScheduling();
    ExecutionGraphTestUtils.switchAllVerticesToRunning(eg);
    Iterator<ExecutionJobVertex> jobVertices = eg.getVerticesTopologically().iterator();
    ExecutionJobVertex sender = jobVertices.next();
    ExecutionJobVertex receiver = jobVertices.next();
    List<ExecutionVertex> senderVertices = Arrays.asList(sender.getTaskVertices());
    List<ExecutionVertex> receiverVertices = Arrays.asList(receiver.getTaskVertices());
    // test getNumExecutionVertexFinished
    senderVertices.get(0).getCurrentExecutionAttempt().markFinished();
    assertEquals(1, sender.getNumExecutionVertexFinished());
    assertEquals(JobStatus.RUNNING, eg.getState());
    senderVertices.get(1).getCurrentExecutionAttempt().markFinished();
    assertEquals(2, sender.getNumExecutionVertexFinished());
    assertEquals(JobStatus.RUNNING, eg.getState());
    // test job finishes
    receiverVertices.get(0).getCurrentExecutionAttempt().markFinished();
    receiverVertices.get(1).getCurrentExecutionAttempt().markFinished();
    assertEquals(4, eg.getNumFinishedVertices());
    assertEquals(JobStatus.FINISHED, eg.getState());
}

Example 14 with SchedulerBase

use of org.apache.flink.runtime.scheduler.SchedulerBase in project flink by apache.

the class ExecutionGraphRestartTest method testCancelWhileFailing.

@Test
public void testCancelWhileFailing() throws Exception {
    try (SlotPool slotPool = SlotPoolUtils.createDeclarativeSlotPoolBridge()) {
        SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(createJobGraph(), mainThreadExecutor).setExecutionSlotAllocatorFactory(createExecutionSlotAllocatorFactory(slotPool)).setRestartBackoffTimeStrategy(new TestRestartBackoffTimeStrategy(false, Long.MAX_VALUE)).build();
        ExecutionGraph graph = scheduler.getExecutionGraph();
        startScheduling(scheduler);
        offerSlots(slotPool, NUM_TASKS);
        assertEquals(JobStatus.RUNNING, graph.getState());
        switchAllTasksToRunning(graph);
        scheduler.handleGlobalFailure(new Exception("test"));
        assertEquals(JobStatus.FAILING, graph.getState());
        scheduler.cancel();
        assertEquals(JobStatus.CANCELLING, graph.getState());
        // let all tasks finish cancelling
        completeCanceling(graph);
        assertEquals(JobStatus.CANCELED, graph.getState());
    }
}

Example 15 with SchedulerBase

use of org.apache.flink.runtime.scheduler.SchedulerBase in project flink by apache.

the class ExecutionGraphRestartTest method testCancelWhileRestarting.

@Test
public void testCancelWhileRestarting() throws Exception {
    // We want to manually control the restart and delay
    try (SlotPool slotPool = SlotPoolUtils.createDeclarativeSlotPoolBridge()) {
        SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(createJobGraph(), mainThreadExecutor).setExecutionSlotAllocatorFactory(createExecutionSlotAllocatorFactory(slotPool)).setRestartBackoffTimeStrategy(new TestRestartBackoffTimeStrategy(true, Long.MAX_VALUE)).setDelayExecutor(taskRestartExecutor).build();
        ExecutionGraph executionGraph = scheduler.getExecutionGraph();
        startScheduling(scheduler);
        final ResourceID taskManagerResourceId = offerSlots(slotPool, NUM_TASKS);
        // Release the TaskManager and wait for the job to restart
        slotPool.releaseTaskManager(taskManagerResourceId, new Exception("Test Exception"));
        assertEquals(JobStatus.RESTARTING, executionGraph.getState());
        // Canceling needs to abort the restart
        scheduler.cancel();
        assertEquals(JobStatus.CANCELED, executionGraph.getState());
        taskRestartExecutor.triggerScheduledTasks();
        assertEquals(JobStatus.CANCELED, executionGraph.getState());
        for (ExecutionVertex vertex : executionGraph.getAllExecutionVertices()) {
            assertEquals(ExecutionState.FAILED, vertex.getExecutionState());
        }
    }
}