Search in sources :

Example 11 with TaskExecutionStateTransition

use of org.apache.flink.runtime.executiongraph.TaskExecutionStateTransition in project flink by apache.

the class AdaptiveSchedulerTest method testUpdateTaskExecutionStateReturnsFalseInIllegalState.

@Test
public void testUpdateTaskExecutionStateReturnsFalseInIllegalState() throws Exception {
    final JobGraph jobGraph = createJobGraph();
    final AdaptiveScheduler scheduler = new AdaptiveSchedulerBuilder(jobGraph, mainThreadExecutor).build();
    assertThat(scheduler.updateTaskExecutionState(new TaskExecutionStateTransition(new TaskExecutionState(new ExecutionAttemptID(), ExecutionState.FAILED)))).isFalse();
}
Also used : JobGraph(org.apache.flink.runtime.jobgraph.JobGraph) JobGraphTestUtils.streamingJobGraph(org.apache.flink.runtime.jobgraph.JobGraphTestUtils.streamingJobGraph) ExecutionAttemptID(org.apache.flink.runtime.executiongraph.ExecutionAttemptID) TaskExecutionStateTransition(org.apache.flink.runtime.executiongraph.TaskExecutionStateTransition) TaskExecutionState(org.apache.flink.runtime.taskmanager.TaskExecutionState) Test(org.junit.Test) ArchivedExecutionGraphTest(org.apache.flink.runtime.executiongraph.ArchivedExecutionGraphTest) DefaultSchedulerTest(org.apache.flink.runtime.scheduler.DefaultSchedulerTest)

Example 12 with TaskExecutionStateTransition

use of org.apache.flink.runtime.executiongraph.TaskExecutionStateTransition in project flink by apache.

the class AdaptiveSchedulerTest method testExceptionHistoryWithTaskFailureWithRestart.

@Test
public void testExceptionHistoryWithTaskFailureWithRestart() throws Exception {
    final Exception expectedException = new Exception("Expected Local Exception");
    Consumer<AdaptiveSchedulerBuilder> setupScheduler = builder -> builder.setRestartBackoffTimeStrategy(new FixedDelayRestartBackoffTimeStrategy.FixedDelayRestartBackoffTimeStrategyFactory(1, 100).create());
    BiConsumer<AdaptiveScheduler, List<ExecutionAttemptID>> testLogic = (scheduler, attemptIds) -> {
        final ExecutionAttemptID attemptId = attemptIds.get(1);
        scheduler.updateTaskExecutionState(new TaskExecutionStateTransition(new TaskExecutionState(attemptId, ExecutionState.FAILED, expectedException)));
    };
    final Iterable<RootExceptionHistoryEntry> actualExceptionHistory = runExceptionHistoryTests(testLogic, setupScheduler);
    assertThat(actualExceptionHistory).hasSize(1);
    final RootExceptionHistoryEntry failure = actualExceptionHistory.iterator().next();
    assertThat(failure.getException().deserializeError(classLoader)).isEqualTo(expectedException);
}
Also used : Arrays(java.util.Arrays) ResourceRequirement(org.apache.flink.runtime.slots.ResourceRequirement) TaskNotRunningException(org.apache.flink.runtime.operators.coordination.TaskNotRunningException) ArchivedExecution(org.apache.flink.runtime.executiongraph.ArchivedExecution) TestingSlotAllocator(org.apache.flink.runtime.scheduler.adaptive.allocator.TestingSlotAllocator) CheckpointException(org.apache.flink.runtime.checkpoint.CheckpointException) ResultPartitionID(org.apache.flink.runtime.io.network.partition.ResultPartitionID) TestingFatalErrorHandler(org.apache.flink.runtime.util.TestingFatalErrorHandler) MetricRegistry(org.apache.flink.runtime.metrics.MetricRegistry) Duration(java.time.Duration) ClassRule(org.junit.ClassRule) TestingCheckpointRecoveryFactory(org.apache.flink.runtime.checkpoint.TestingCheckpointRecoveryFactory) ManuallyTriggeredComponentMainThreadExecutor(org.apache.flink.runtime.concurrent.ManuallyTriggeredComponentMainThreadExecutor) BlockingQueue(java.util.concurrent.BlockingQueue) MetricOptions(org.apache.flink.configuration.MetricOptions) JobManagerOptions(org.apache.flink.configuration.JobManagerOptions) Executors(java.util.concurrent.Executors) MetricNames(org.apache.flink.runtime.metrics.MetricNames) ArrayBlockingQueue(java.util.concurrent.ArrayBlockingQueue) VertexParallelismStore(org.apache.flink.runtime.scheduler.VertexParallelismStore) Time(org.apache.flink.api.common.time.Time) RootExceptionHistoryEntry(org.apache.flink.runtime.scheduler.exceptionhistory.RootExceptionHistoryEntry) FlinkException(org.apache.flink.util.FlinkException) ComponentMainThreadExecutor(org.apache.flink.runtime.concurrent.ComponentMainThreadExecutor) UpTimeGauge(org.apache.flink.runtime.executiongraph.metrics.UpTimeGauge) ExceptionHistoryEntry(org.apache.flink.runtime.scheduler.exceptionhistory.ExceptionHistoryEntry) LocalTaskManagerLocation(org.apache.flink.runtime.taskmanager.LocalTaskManagerLocation) ResourceCounter(org.apache.flink.runtime.util.ResourceCounter) JobStatus(org.apache.flink.api.common.JobStatus) DefaultAllocatedSlotPool(org.apache.flink.runtime.jobmaster.slotpool.DefaultAllocatedSlotPool) ArrayList(java.util.ArrayList) SchedulerNG(org.apache.flink.runtime.scheduler.SchedulerNG) DownTimeGauge(org.apache.flink.runtime.executiongraph.metrics.DownTimeGauge) PartitionProducerDisposedException(org.apache.flink.runtime.jobmanager.PartitionProducerDisposedException) Gauge(org.apache.flink.metrics.Gauge) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) BiConsumer(java.util.function.BiConsumer) FixedDelayRestartBackoffTimeStrategy(org.apache.flink.runtime.executiongraph.failover.flip1.FixedDelayRestartBackoffTimeStrategy) Nullable(javax.annotation.Nullable) ArchivedExecutionVertex(org.apache.flink.runtime.executiongraph.ArchivedExecutionVertex) TestExecutorResource(org.apache.flink.testutils.executor.TestExecutorResource) ExecutionState(org.apache.flink.runtime.execution.ExecutionState) CheckpointsCleaner(org.apache.flink.runtime.checkpoint.CheckpointsCleaner) TestOperatorEvent(org.apache.flink.runtime.operators.coordination.TestOperatorEvent) Test(org.junit.Test) IOException(java.io.IOException) IterableUtils(org.apache.flink.util.IterableUtils) SimpleAckingTaskManagerGateway(org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway) ExecutionException(java.util.concurrent.ExecutionException) TaskExecutionStateTransition(org.apache.flink.runtime.executiongraph.TaskExecutionStateTransition) JobID(org.apache.flink.api.common.JobID) NoRestartBackoffTimeStrategy(org.apache.flink.runtime.executiongraph.failover.flip1.NoRestartBackoffTimeStrategy) SlotPoolTestUtils.offerSlots(org.apache.flink.runtime.jobmaster.slotpool.SlotPoolTestUtils.offerSlots) JobManagerMetricGroup(org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup) ComponentMainThreadExecutorServiceAdapter(org.apache.flink.runtime.concurrent.ComponentMainThreadExecutorServiceAdapter) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) JobGraph(org.apache.flink.runtime.jobgraph.JobGraph) CheckpointCoordinatorConfiguration(org.apache.flink.runtime.jobgraph.tasks.CheckpointCoordinatorConfiguration) SuppressRestartsException(org.apache.flink.runtime.execution.SuppressRestartsException) ArchivedExecutionGraphTest(org.apache.flink.runtime.executiongraph.ArchivedExecutionGraphTest) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) TestLogger(org.apache.flink.util.TestLogger) JobCheckpointingSettings(org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings) CheckpointIDCounter(org.apache.flink.runtime.checkpoint.CheckpointIDCounter) ArchivedExecutionJobVertex(org.apache.flink.runtime.executiongraph.ArchivedExecutionJobVertex) TestingCompletedCheckpointStore(org.apache.flink.runtime.checkpoint.TestingCompletedCheckpointStore) TestRestartBackoffTimeStrategy(org.apache.flink.runtime.executiongraph.failover.flip1.TestRestartBackoffTimeStrategy) IntermediateDataSetID(org.apache.flink.runtime.jobgraph.IntermediateDataSetID) Collectors(java.util.stream.Collectors) ResourceProfile(org.apache.flink.runtime.clusterframework.types.ResourceProfile) DefaultDeclarativeSlotPool(org.apache.flink.runtime.jobmaster.slotpool.DefaultDeclarativeSlotPool) List(java.util.List) StandaloneCheckpointIDCounter(org.apache.flink.runtime.checkpoint.StandaloneCheckpointIDCounter) CoordinationRequest(org.apache.flink.runtime.operators.coordination.CoordinationRequest) OperatorID(org.apache.flink.runtime.jobgraph.OperatorID) Optional(java.util.Optional) SchedulerExecutionMode(org.apache.flink.configuration.SchedulerExecutionMode) KeyGroupRangeAssignment(org.apache.flink.runtime.state.KeyGroupRangeAssignment) OneShotLatch(org.apache.flink.core.testutils.OneShotLatch) SchedulerBase(org.apache.flink.runtime.scheduler.SchedulerBase) SavepointFormatType(org.apache.flink.core.execution.SavepointFormatType) JobVertex(org.apache.flink.runtime.jobgraph.JobVertex) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) CompletableFuture(java.util.concurrent.CompletableFuture) VertexParallelismInformation(org.apache.flink.runtime.scheduler.VertexParallelismInformation) TaskDeploymentDescriptor(org.apache.flink.runtime.deployment.TaskDeploymentDescriptor) TestingCheckpointIDCounter(org.apache.flink.runtime.checkpoint.TestingCheckpointIDCounter) TestingMetricRegistry(org.apache.flink.runtime.metrics.util.TestingMetricRegistry) Nonnull(javax.annotation.Nonnull) StandaloneCompletedCheckpointStore(org.apache.flink.runtime.checkpoint.StandaloneCompletedCheckpointStore) DefaultSchedulerTest(org.apache.flink.runtime.scheduler.DefaultSchedulerTest) JobGraphTestUtils.streamingJobGraph(org.apache.flink.runtime.jobgraph.JobGraphTestUtils.streamingJobGraph) ArchivedExecutionGraph(org.apache.flink.runtime.executiongraph.ArchivedExecutionGraph) ArchivedExecutionGraphBuilder(org.apache.flink.runtime.rest.handler.legacy.utils.ArchivedExecutionGraphBuilder) Logger(org.slf4j.Logger) Configuration(org.apache.flink.configuration.Configuration) CompletedCheckpointStore(org.apache.flink.runtime.checkpoint.CompletedCheckpointStore) JobStatusListener(org.apache.flink.runtime.executiongraph.JobStatusListener) DefaultDeclarativeSlotPoolTest.createSlotOffersForResourceRequirements(org.apache.flink.runtime.jobmaster.slotpool.DefaultDeclarativeSlotPoolTest.createSlotOffersForResourceRequirements) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) ExecutionAttemptID(org.apache.flink.runtime.executiongraph.ExecutionAttemptID) ExecutionGraphTestUtils.createNoOpVertex(org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils.createNoOpVertex) SchedulerTestingUtils.enableCheckpointing(org.apache.flink.runtime.scheduler.SchedulerTestingUtils.enableCheckpointing) TaskExecutionState(org.apache.flink.runtime.taskmanager.TaskExecutionState) TemporaryFolder(org.junit.rules.TemporaryFolder) ExecutionAttemptID(org.apache.flink.runtime.executiongraph.ExecutionAttemptID) TaskExecutionStateTransition(org.apache.flink.runtime.executiongraph.TaskExecutionStateTransition) RootExceptionHistoryEntry(org.apache.flink.runtime.scheduler.exceptionhistory.RootExceptionHistoryEntry) TaskNotRunningException(org.apache.flink.runtime.operators.coordination.TaskNotRunningException) CheckpointException(org.apache.flink.runtime.checkpoint.CheckpointException) FlinkException(org.apache.flink.util.FlinkException) PartitionProducerDisposedException(org.apache.flink.runtime.jobmanager.PartitionProducerDisposedException) IOException(java.io.IOException) ExecutionException(java.util.concurrent.ExecutionException) SuppressRestartsException(org.apache.flink.runtime.execution.SuppressRestartsException) TaskExecutionState(org.apache.flink.runtime.taskmanager.TaskExecutionState) ArrayList(java.util.ArrayList) List(java.util.List) Test(org.junit.Test) ArchivedExecutionGraphTest(org.apache.flink.runtime.executiongraph.ArchivedExecutionGraphTest) DefaultSchedulerTest(org.apache.flink.runtime.scheduler.DefaultSchedulerTest)

Example 13 with TaskExecutionStateTransition

use of org.apache.flink.runtime.executiongraph.TaskExecutionStateTransition in project flink by apache.

the class FailingTest method testTaskFailuresAreIgnored.

@Test
public void testTaskFailuresAreIgnored() throws Exception {
    try (MockFailingContext ctx = new MockFailingContext()) {
        StateTrackingMockExecutionGraph meg = new StateTrackingMockExecutionGraph();
        Failing failing = createFailingState(ctx, meg);
        // register execution at EG
        Exception exception = new RuntimeException();
        TestingAccessExecution execution = TestingAccessExecution.newBuilder().withExecutionState(ExecutionState.FAILED).withErrorInfo(new ErrorInfo(exception, System.currentTimeMillis())).build();
        meg.registerExecution(execution);
        TaskExecutionStateTransition update = ExecutingTest.createFailingStateTransition(execution.getAttemptId(), exception);
        failing.updateTaskExecutionState(update);
        ctx.assertNoStateTransition();
    }
}
Also used : TestingAccessExecution(org.apache.flink.runtime.scheduler.exceptionhistory.TestingAccessExecution) TaskExecutionStateTransition(org.apache.flink.runtime.executiongraph.TaskExecutionStateTransition) ErrorInfo(org.apache.flink.runtime.executiongraph.ErrorInfo) Test(org.junit.Test)

Example 14 with TaskExecutionStateTransition

use of org.apache.flink.runtime.executiongraph.TaskExecutionStateTransition in project flink by apache.

the class StopWithSavepointTest method testFailingOnUpdateTaskExecutionStateWithNoRestart.

@Test
public void testFailingOnUpdateTaskExecutionStateWithNoRestart() throws Exception {
    try (MockStopWithSavepointContext ctx = new MockStopWithSavepointContext()) {
        StateTrackingMockExecutionGraph executionGraph = new StateTrackingMockExecutionGraph();
        StopWithSavepoint sws = createStopWithSavepoint(ctx, executionGraph);
        ctx.setStopWithSavepoint(sws);
        ctx.setHowToHandleFailure(FailureResult::canNotRestart);
        ctx.setExpectFailing(failingArguments -> {
            assertThat(failingArguments.getFailureCause(), containsCause(RuntimeException.class));
        });
        Exception exception = new RuntimeException();
        TestingAccessExecution execution = TestingAccessExecution.newBuilder().withExecutionState(ExecutionState.FAILED).withErrorInfo(new ErrorInfo(exception, System.currentTimeMillis())).build();
        executionGraph.registerExecution(execution);
        TaskExecutionStateTransition taskExecutionStateTransition = ExecutingTest.createFailingStateTransition(execution.getAttemptId(), exception);
        assertThat(sws.updateTaskExecutionState(taskExecutionStateTransition), is(true));
    }
}
Also used : TestingAccessExecution(org.apache.flink.runtime.scheduler.exceptionhistory.TestingAccessExecution) TaskExecutionStateTransition(org.apache.flink.runtime.executiongraph.TaskExecutionStateTransition) ErrorInfo(org.apache.flink.runtime.executiongraph.ErrorInfo) FlinkException(org.apache.flink.util.FlinkException) Test(org.junit.Test)

Example 15 with TaskExecutionStateTransition

use of org.apache.flink.runtime.executiongraph.TaskExecutionStateTransition in project flink by apache.

the class ExecutingTest method testFailureReportedViaUpdateTaskExecutionStateCausesFailingOnNoRestart.

@Test
public void testFailureReportedViaUpdateTaskExecutionStateCausesFailingOnNoRestart() throws Exception {
    try (MockExecutingContext ctx = new MockExecutingContext()) {
        StateTrackingMockExecutionGraph returnsFailedStateExecutionGraph = new StateTrackingMockExecutionGraph();
        Executing exec = new ExecutingStateBuilder().setExecutionGraph(returnsFailedStateExecutionGraph).build(ctx);
        ctx.setHowToHandleFailure(FailureResult::canNotRestart);
        ctx.setExpectFailing(assertNonNull());
        Exception exception = new RuntimeException();
        TestingAccessExecution execution = TestingAccessExecution.newBuilder().withExecutionState(ExecutionState.FAILED).withErrorInfo(new ErrorInfo(exception, System.currentTimeMillis())).build();
        returnsFailedStateExecutionGraph.registerExecution(execution);
        TaskExecutionStateTransition taskExecutionStateTransition = createFailingStateTransition(execution.getAttemptId(), exception);
        exec.updateTaskExecutionState(taskExecutionStateTransition);
    }
}
Also used : TestingAccessExecution(org.apache.flink.runtime.scheduler.exceptionhistory.TestingAccessExecution) TaskExecutionStateTransition(org.apache.flink.runtime.executiongraph.TaskExecutionStateTransition) ErrorInfo(org.apache.flink.runtime.executiongraph.ErrorInfo) JobException(org.apache.flink.runtime.JobException) JobExecutionException(org.apache.flink.runtime.client.JobExecutionException) Test(org.junit.Test)

Aggregations

TaskExecutionStateTransition (org.apache.flink.runtime.executiongraph.TaskExecutionStateTransition)15 Test (org.junit.Test)14 FlinkException (org.apache.flink.util.FlinkException)8 CompletableFuture (java.util.concurrent.CompletableFuture)6 IOException (java.io.IOException)5 Duration (java.time.Duration)5 ArrayList (java.util.ArrayList)5 Arrays (java.util.Arrays)5 List (java.util.List)5 Optional (java.util.Optional)5 ArrayBlockingQueue (java.util.concurrent.ArrayBlockingQueue)5 BlockingQueue (java.util.concurrent.BlockingQueue)5 ExecutionException (java.util.concurrent.ExecutionException)5 Executors (java.util.concurrent.Executors)5 ScheduledExecutorService (java.util.concurrent.ScheduledExecutorService)5 TimeUnit (java.util.concurrent.TimeUnit)5 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)5 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)5 BiConsumer (java.util.function.BiConsumer)5 Consumer (java.util.function.Consumer)5