Examples with TestRestartBackoffTimeStrategy org.apache.flink.runtime.executiongraph.failover.flip1.TestRestartBackoffTimeStrategy used on opensource projects

Example 1 with TestRestartBackoffTimeStrategy

use of org.apache.flink.runtime.executiongraph.failover.flip1.TestRestartBackoffTimeStrategy in project flink by apache.

the class DefaultSchedulerTest method setUp.

@Before
public void setUp() throws Exception {
    executor = Executors.newSingleThreadExecutor();
    scheduledExecutorService = new DirectScheduledExecutorService();
    configuration = new Configuration();
    testRestartBackoffTimeStrategy = new TestRestartBackoffTimeStrategy(true, 0);
    testExecutionVertexOperations = new TestExecutionVertexOperationsDecorator(new DefaultExecutionVertexOperations());
    executionVertexVersioner = new ExecutionVertexVersioner();
    executionSlotAllocatorFactory = new TestExecutionSlotAllocatorFactory();
    testExecutionSlotAllocator = executionSlotAllocatorFactory.getTestExecutionSlotAllocator();
    shuffleMaster = new TestingShuffleMaster();
    partitionTracker = new TestingJobMasterPartitionTracker();
    timeout = Time.seconds(60);
}

Example 2 with TestRestartBackoffTimeStrategy

use of org.apache.flink.runtime.executiongraph.failover.flip1.TestRestartBackoffTimeStrategy in project flink by apache.

the class ExecutionGraphRestartTest method testCancelWhileFailing.

@Test
public void testCancelWhileFailing() throws Exception {
    try (SlotPool slotPool = SlotPoolUtils.createDeclarativeSlotPoolBridge()) {
        SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(createJobGraph(), mainThreadExecutor).setExecutionSlotAllocatorFactory(createExecutionSlotAllocatorFactory(slotPool)).setRestartBackoffTimeStrategy(new TestRestartBackoffTimeStrategy(false, Long.MAX_VALUE)).build();
        ExecutionGraph graph = scheduler.getExecutionGraph();
        startScheduling(scheduler);
        offerSlots(slotPool, NUM_TASKS);
        assertEquals(JobStatus.RUNNING, graph.getState());
        switchAllTasksToRunning(graph);
        scheduler.handleGlobalFailure(new Exception("test"));
        assertEquals(JobStatus.FAILING, graph.getState());
        scheduler.cancel();
        assertEquals(JobStatus.CANCELLING, graph.getState());
        // let all tasks finish cancelling
        completeCanceling(graph);
        assertEquals(JobStatus.CANCELED, graph.getState());
    }
}

Example 3 with TestRestartBackoffTimeStrategy

use of org.apache.flink.runtime.executiongraph.failover.flip1.TestRestartBackoffTimeStrategy in project flink by apache.

the class ExecutionGraphRestartTest method testCancelWhileRestarting.

@Test
public void testCancelWhileRestarting() throws Exception {
    // We want to manually control the restart and delay
    try (SlotPool slotPool = SlotPoolUtils.createDeclarativeSlotPoolBridge()) {
        SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(createJobGraph(), mainThreadExecutor).setExecutionSlotAllocatorFactory(createExecutionSlotAllocatorFactory(slotPool)).setRestartBackoffTimeStrategy(new TestRestartBackoffTimeStrategy(true, Long.MAX_VALUE)).setDelayExecutor(taskRestartExecutor).build();
        ExecutionGraph executionGraph = scheduler.getExecutionGraph();
        startScheduling(scheduler);
        final ResourceID taskManagerResourceId = offerSlots(slotPool, NUM_TASKS);
        // Release the TaskManager and wait for the job to restart
        slotPool.releaseTaskManager(taskManagerResourceId, new Exception("Test Exception"));
        assertEquals(JobStatus.RESTARTING, executionGraph.getState());
        // Canceling needs to abort the restart
        scheduler.cancel();
        assertEquals(JobStatus.CANCELED, executionGraph.getState());
        taskRestartExecutor.triggerScheduledTasks();
        assertEquals(JobStatus.CANCELED, executionGraph.getState());
        for (ExecutionVertex vertex : executionGraph.getAllExecutionVertices()) {
            assertEquals(ExecutionState.FAILED, vertex.getExecutionState());
        }
    }
}

Example 4 with TestRestartBackoffTimeStrategy

use of org.apache.flink.runtime.executiongraph.failover.flip1.TestRestartBackoffTimeStrategy in project flink by apache.

the class ExecutionGraphRestartTest method testFailingExecutionAfterRestart.

/**
 * Tests that a failing execution does not affect a restarted job. This is important if a
 * callback handler fails an execution after it has already reached a final state and the job
 * has been restarted.
 */
@Test
public void testFailingExecutionAfterRestart() throws Exception {
    JobVertex sender = ExecutionGraphTestUtils.createJobVertex("Task1", 1, NoOpInvokable.class);
    JobVertex receiver = ExecutionGraphTestUtils.createJobVertex("Task2", 1, NoOpInvokable.class);
    JobGraph jobGraph = JobGraphTestUtils.streamingJobGraph(sender, receiver);
    try (SlotPool slotPool = SlotPoolUtils.createDeclarativeSlotPoolBridge()) {
        SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(jobGraph, mainThreadExecutor).setExecutionSlotAllocatorFactory(createExecutionSlotAllocatorFactory(slotPool)).setRestartBackoffTimeStrategy(new TestRestartBackoffTimeStrategy(true, Long.MAX_VALUE)).setDelayExecutor(taskRestartExecutor).build();
        ExecutionGraph eg = scheduler.getExecutionGraph();
        startScheduling(scheduler);
        offerSlots(slotPool, 2);
        Iterator<ExecutionVertex> executionVertices = eg.getAllExecutionVertices().iterator();
        Execution finishedExecution = executionVertices.next().getCurrentExecutionAttempt();
        Execution failedExecution = executionVertices.next().getCurrentExecutionAttempt();
        finishedExecution.markFinished();
        failedExecution.fail(new Exception("Test Exception"));
        failedExecution.completeCancelling();
        taskRestartExecutor.triggerScheduledTasks();
        assertEquals(JobStatus.RUNNING, eg.getState());
        // At this point all resources have been assigned
        for (ExecutionVertex vertex : eg.getAllExecutionVertices()) {
            assertNotNull("No assigned resource (test instability).", vertex.getCurrentAssignedResource());
            vertex.getCurrentExecutionAttempt().switchToRecovering();
            vertex.getCurrentExecutionAttempt().switchToRunning();
        }
        // fail old finished execution, this should not affect the execution
        finishedExecution.fail(new Exception("This should have no effect"));
        for (ExecutionVertex vertex : eg.getAllExecutionVertices()) {
            vertex.getCurrentExecutionAttempt().markFinished();
        }
        // the state of the finished execution should have not changed since it is terminal
        assertEquals(ExecutionState.FINISHED, finishedExecution.getState());
        assertEquals(JobStatus.FINISHED, eg.getState());
    }
}

Example 5 with TestRestartBackoffTimeStrategy

use of org.apache.flink.runtime.executiongraph.failover.flip1.TestRestartBackoffTimeStrategy in project flink by apache.

the class ExecutionGraphRestartTest method testFailWhileCanceling.

@Test
public void testFailWhileCanceling() throws Exception {
    try (SlotPool slotPool = SlotPoolUtils.createDeclarativeSlotPoolBridge()) {
        SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(createJobGraph(), mainThreadExecutor).setExecutionSlotAllocatorFactory(createExecutionSlotAllocatorFactory(slotPool)).setRestartBackoffTimeStrategy(new TestRestartBackoffTimeStrategy(false, Long.MAX_VALUE)).build();
        ExecutionGraph graph = scheduler.getExecutionGraph();
        startScheduling(scheduler);
        offerSlots(slotPool, NUM_TASKS);
        assertEquals(JobStatus.RUNNING, graph.getState());
        switchAllTasksToRunning(graph);
        scheduler.cancel();
        assertEquals(JobStatus.CANCELLING, graph.getState());
        scheduler.handleGlobalFailure(new Exception("test"));
        assertEquals(JobStatus.FAILING, graph.getState());
        // let all tasks finish cancelling
        completeCanceling(graph);
        assertEquals(JobStatus.FAILED, graph.getState());
    }
}