Examples with SchedulerBase org.apache.flink.runtime.scheduler.SchedulerBase used on opensource projects

Example 46 with SchedulerBase

use of org.apache.flink.runtime.scheduler.SchedulerBase in project flink by apache.

the class AdaptiveSchedulerComputeReactiveModeVertexParallelismTest method testCreateStoreWithAdjustedParallelism.

@Test
public void testCreateStoreWithAdjustedParallelism() {
    JobVertex jobVertex = createNoOpVertex("test", parallelism, maxParallelism);
    VertexParallelismStore store = AdaptiveScheduler.computeReactiveModeVertexParallelismStore(Collections.singleton(jobVertex), SchedulerBase::getDefaultMaxParallelism, true);
    VertexParallelismInformation info = store.getParallelismInfo(jobVertex.getID());
    Assert.assertEquals("parallelism is adjusted to max", expectedMaxParallelism, info.getParallelism());
    Assert.assertEquals("expected max", expectedMaxParallelism, info.getMaxParallelism());
    Assert.assertEquals("can rescale max", expectedCanRescaleTo, info.canRescaleMaxParallelism(maxToScaleTo));
}

Example 47 with SchedulerBase

use of org.apache.flink.runtime.scheduler.SchedulerBase in project flink by apache.

the class DefaultSchedulerCheckpointCoordinatorTest method testClosingSchedulerShutsDownCheckpointCoordinatorOnFinishedExecutionGraph.

/**
 * Tests that the checkpoint coordinator is shut down if the execution graph is finished.
 */
@Test
public void testClosingSchedulerShutsDownCheckpointCoordinatorOnFinishedExecutionGraph() throws Exception {
    final CompletableFuture<JobStatus> counterShutdownFuture = new CompletableFuture<>();
    CheckpointIDCounter counter = TestingCheckpointIDCounter.createStoreWithShutdownCheckAndNoStartAction(counterShutdownFuture);
    final CompletableFuture<JobStatus> storeShutdownFuture = new CompletableFuture<>();
    CompletedCheckpointStore store = TestingCompletedCheckpointStore.createStoreWithShutdownCheckAndNoCompletedCheckpoints(storeShutdownFuture);
    final SchedulerBase scheduler = createSchedulerAndEnableCheckpointing(counter, store);
    final ExecutionGraph graph = scheduler.getExecutionGraph();
    final CheckpointCoordinator checkpointCoordinator = graph.getCheckpointCoordinator();
    assertThat(checkpointCoordinator, Matchers.notNullValue());
    assertThat(checkpointCoordinator.isShutdown(), is(false));
    scheduler.startScheduling();
    for (ExecutionVertex executionVertex : graph.getAllExecutionVertices()) {
        final Execution currentExecutionAttempt = executionVertex.getCurrentExecutionAttempt();
        scheduler.updateTaskExecutionState(new TaskExecutionState(currentExecutionAttempt.getAttemptId(), ExecutionState.FINISHED));
    }
    assertThat(graph.getTerminationFuture().get(), is(JobStatus.FINISHED));
    scheduler.closeAsync().get();
    assertThat(checkpointCoordinator.isShutdown(), is(true));
    assertThat(counterShutdownFuture.get(), is(JobStatus.FINISHED));
    assertThat(storeShutdownFuture.get(), is(JobStatus.FINISHED));
}

Example 48 with SchedulerBase

use of org.apache.flink.runtime.scheduler.SchedulerBase in project flink by apache.

the class DefaultSchedulerCheckpointCoordinatorTest method testClosingSchedulerShutsDownCheckpointCoordinatorOnSuspendedExecutionGraph.

/**
 * Tests that the checkpoint coordinator is shut down if the execution graph is suspended.
 */
@Test
public void testClosingSchedulerShutsDownCheckpointCoordinatorOnSuspendedExecutionGraph() throws Exception {
    final CompletableFuture<JobStatus> counterShutdownFuture = new CompletableFuture<>();
    CheckpointIDCounter counter = TestingCheckpointIDCounter.createStoreWithShutdownCheckAndNoStartAction(counterShutdownFuture);
    final CompletableFuture<JobStatus> storeShutdownFuture = new CompletableFuture<>();
    CompletedCheckpointStore store = TestingCompletedCheckpointStore.createStoreWithShutdownCheckAndNoCompletedCheckpoints(storeShutdownFuture);
    final SchedulerBase scheduler = createSchedulerAndEnableCheckpointing(counter, store);
    final ExecutionGraph graph = scheduler.getExecutionGraph();
    final CheckpointCoordinator checkpointCoordinator = graph.getCheckpointCoordinator();
    assertThat(checkpointCoordinator, Matchers.notNullValue());
    assertThat(checkpointCoordinator.isShutdown(), is(false));
    graph.suspend(new Exception("Test Exception"));
    scheduler.closeAsync().get();
    assertThat(checkpointCoordinator.isShutdown(), is(true));
    assertThat(counterShutdownFuture.get(), is(JobStatus.SUSPENDED));
    assertThat(storeShutdownFuture.get(), is(JobStatus.SUSPENDED));
}

Example 49 with SchedulerBase

use of org.apache.flink.runtime.scheduler.SchedulerBase in project flink by apache.

the class ArchivedExecutionGraphTest method setupExecutionGraph.

@BeforeClass
public static void setupExecutionGraph() throws Exception {
    // -------------------------------------------------------------------------------------------------------------
    // Setup
    // -------------------------------------------------------------------------------------------------------------
    JobVertexID v1ID = new JobVertexID();
    JobVertexID v2ID = new JobVertexID();
    JobVertex v1 = new JobVertex("v1", v1ID);
    JobVertex v2 = new JobVertex("v2", v2ID);
    v1.setParallelism(1);
    v2.setParallelism(2);
    v1.setInvokableClass(AbstractInvokable.class);
    v2.setInvokableClass(AbstractInvokable.class);
    ExecutionConfig config = new ExecutionConfig();
    config.setRestartStrategy(new RestartStrategies.NoRestartStrategyConfiguration());
    config.setParallelism(4);
    config.enableObjectReuse();
    config.setGlobalJobParameters(new TestJobParameters());
    CheckpointCoordinatorConfiguration chkConfig = new CheckpointCoordinatorConfiguration(100, 100, 100, 1, CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION, true, false, 0, 0);
    JobCheckpointingSettings checkpointingSettings = new JobCheckpointingSettings(chkConfig, null);
    final JobGraph jobGraph = JobGraphBuilder.newStreamingJobGraphBuilder().addJobVertices(asList(v1, v2)).setJobCheckpointingSettings(checkpointingSettings).setExecutionConfig(config).build();
    SchedulerBase scheduler = SchedulerTestingUtils.createScheduler(jobGraph, ComponentMainThreadExecutorServiceAdapter.forMainThread());
    runtimeGraph = scheduler.getExecutionGraph();
    scheduler.startScheduling();
    scheduler.updateTaskExecutionState(new TaskExecutionState(runtimeGraph.getAllExecutionVertices().iterator().next().getCurrentExecutionAttempt().getAttemptId(), ExecutionState.FAILED, new RuntimeException("Local failure")));
}

Example 50 with SchedulerBase

use of org.apache.flink.runtime.scheduler.SchedulerBase in project flink by apache.

the class DefaultExecutionGraphDeploymentTest method testRegistrationOfExecutionsFailedExternally.

@Test
public void testRegistrationOfExecutionsFailedExternally() {
    try {
        final JobVertexID jid1 = new JobVertexID();
        final JobVertexID jid2 = new JobVertexID();
        JobVertex v1 = new JobVertex("v1", jid1);
        JobVertex v2 = new JobVertex("v2", jid2);
        SchedulerBase scheduler = setupScheduler(v1, 7, v2, 6);
        Collection<Execution> executions = new ArrayList<>(scheduler.getExecutionGraph().getRegisteredExecutions().values());
        for (Execution e : executions) {
            e.fail(null);
        }
        assertEquals(0, scheduler.getExecutionGraph().getRegisteredExecutions().size());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}