Examples with DefaultScheduler org.apache.flink.runtime.scheduler.DefaultScheduler used on opensource projects

Example 36 with DefaultScheduler

use of org.apache.flink.runtime.scheduler.DefaultScheduler in project flink by apache.

the class OperatorCoordinatorSchedulerTest method setupTestJobAndScheduler.

private DefaultScheduler setupTestJobAndScheduler(OperatorCoordinator.Provider provider, @Nullable TaskExecutorOperatorEventGateway taskExecutorOperatorEventGateway, @Nullable Consumer<JobGraph> jobGraphPreProcessing, boolean restartAllOnFailover) throws Exception {
    final OperatorIDPair opIds = OperatorIDPair.of(new OperatorID(), provider.getOperatorId());
    final JobVertex vertex = new JobVertex("Vertex with OperatorCoordinator", testVertexId, Collections.singletonList(opIds));
    vertex.setInvokableClass(NoOpInvokable.class);
    vertex.addOperatorCoordinator(new SerializedValue<>(provider));
    vertex.setParallelism(2);
    final JobGraph jobGraph = JobGraphBuilder.newStreamingJobGraphBuilder().addJobVertex(vertex).build();
    SchedulerTestingUtils.enableCheckpointing(jobGraph);
    if (jobGraphPreProcessing != null) {
        jobGraphPreProcessing.accept(jobGraph);
    }
    final ComponentMainThreadExecutor mainThreadExecutor = new ComponentMainThreadExecutorServiceAdapter((ScheduledExecutorService) executor, Thread.currentThread());
    final SchedulerTestingUtils.DefaultSchedulerBuilder schedulerBuilder = taskExecutorOperatorEventGateway == null ? SchedulerTestingUtils.createSchedulerBuilder(jobGraph, mainThreadExecutor) : SchedulerTestingUtils.createSchedulerBuilder(jobGraph, mainThreadExecutor, taskExecutorOperatorEventGateway);
    if (restartAllOnFailover) {
        schedulerBuilder.setFailoverStrategyFactory(new RestartAllFailoverStrategy.Factory());
    }
    final DefaultScheduler scheduler = schedulerBuilder.setFutureExecutor(executor).setDelayExecutor(executor).build();
    this.createdScheduler = scheduler;
    return scheduler;
}

Example 37 with DefaultScheduler

use of org.apache.flink.runtime.scheduler.DefaultScheduler in project flink by apache.

the class OperatorCoordinatorSchedulerTest method getCoordinator.

private TestingOperatorCoordinator getCoordinator(DefaultScheduler scheduler) {
    final ExecutionJobVertex vertexWithCoordinator = getJobVertex(scheduler, testVertexId);
    assertNotNull("vertex for coordinator not found", vertexWithCoordinator);
    final Optional<OperatorCoordinatorHolder> coordinatorOptional = vertexWithCoordinator.getOperatorCoordinators().stream().filter((holder) -> holder.operatorId().equals(testOperatorId)).findFirst();
    assertTrue("vertex does not contain coordinator", coordinatorOptional.isPresent());
    final OperatorCoordinator coordinator = coordinatorOptional.get().coordinator();
    assertThat(coordinator, instanceOf(TestingOperatorCoordinator.class));
    return (TestingOperatorCoordinator) coordinator;
}

Example 38 with DefaultScheduler

use of org.apache.flink.runtime.scheduler.DefaultScheduler in project flink by apache.

the class OperatorCoordinatorSchedulerTest method createSchedulerWithAllRestartOnFailureAndDeployTasks.

private DefaultScheduler createSchedulerWithAllRestartOnFailureAndDeployTasks() throws Exception {
    final DefaultScheduler scheduler = setupTestJobAndScheduler(new TestingOperatorCoordinator.Provider(testOperatorId), null, null, true);
    scheduleAllTasksToRunning(scheduler);
    return scheduler;
}

Example 39 with DefaultScheduler

use of org.apache.flink.runtime.scheduler.DefaultScheduler in project flink by apache.

the class OperatorCoordinatorSchedulerTest method testConfirmCheckpointComplete.

@Test
public void testConfirmCheckpointComplete() throws Exception {
    final DefaultScheduler scheduler = createSchedulerAndDeployTasks();
    final TestingOperatorCoordinator coordinator = getCoordinator(scheduler);
    final long checkpointId = takeCompleteCheckpoint(scheduler, coordinator, new byte[] { 37, 11, 83, 4 });
    assertEquals("coordinator should be notified of completed checkpoint", checkpointId, coordinator.getLastCheckpointComplete());
}

Example 40 with DefaultScheduler

use of org.apache.flink.runtime.scheduler.DefaultScheduler in project flink by apache.

the class OperatorCoordinatorSchedulerTest method runningTaskFailureNotifiesCoordinator.

@Test
public void runningTaskFailureNotifiesCoordinator() throws Exception {
    final DefaultScheduler scheduler = createSchedulerAndDeployTasks();
    final TestingOperatorCoordinator coordinator = getCoordinator(scheduler);
    failTask(scheduler, 1);
    assertEquals(1, coordinator.getFailedTasks().size());
    assertThat(coordinator.getFailedTasks(), contains(1));
    assertThat(coordinator.getFailedTasks(), not(contains(0)));
}