Search in sources :

Example 1 with NoOpTaskManagerActions

use of org.apache.flink.runtime.taskmanager.NoOpTaskManagerActions in project flink by apache.

the class StreamTaskTest method testEarlyCanceling.

/**
 * This test checks that cancel calls that are issued before the operator is instantiated still
 * lead to proper canceling.
 */
@Test
public void testEarlyCanceling() throws Exception {
    final StreamConfig cfg = new StreamConfig(new Configuration());
    cfg.setOperatorID(new OperatorID(4711L, 42L));
    cfg.setStreamOperator(new SlowlyDeserializingOperator());
    cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);
    final TaskManagerActions taskManagerActions = spy(new NoOpTaskManagerActions());
    try (NettyShuffleEnvironment shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build()) {
        final Task task = new TestTaskBuilder(shuffleEnvironment).setInvokable(SourceStreamTask.class).setTaskConfig(cfg.getConfiguration()).setTaskManagerActions(taskManagerActions).build();
        final TaskExecutionState state = new TaskExecutionState(task.getExecutionId(), ExecutionState.RUNNING);
        task.startTaskThread();
        verify(taskManagerActions, timeout(2000L)).updateTaskExecutionState(eq(state));
        // send a cancel. because the operator takes a long time to deserialize, this should
        // hit the task before the operator is deserialized
        task.cancelExecution();
        task.getExecutingThread().join();
        assertFalse("Task did not cancel", task.getExecutingThread().isAlive());
        assertEquals(ExecutionState.CANCELED, task.getExecutionState());
    }
}
Also used : NoOpTaskManagerActions(org.apache.flink.runtime.taskmanager.NoOpTaskManagerActions) Task(org.apache.flink.runtime.taskmanager.Task) Configuration(org.apache.flink.configuration.Configuration) TestTaskBuilder(org.apache.flink.runtime.taskmanager.TestTaskBuilder) MockStreamConfig(org.apache.flink.streaming.util.MockStreamConfig) StreamConfig(org.apache.flink.streaming.api.graph.StreamConfig) NettyShuffleEnvironmentBuilder(org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder) OperatorID(org.apache.flink.runtime.jobgraph.OperatorID) NettyShuffleEnvironment(org.apache.flink.runtime.io.network.NettyShuffleEnvironment) TaskExecutionState(org.apache.flink.runtime.taskmanager.TaskExecutionState) TaskManagerActions(org.apache.flink.runtime.taskmanager.TaskManagerActions) NoOpTaskManagerActions(org.apache.flink.runtime.taskmanager.NoOpTaskManagerActions) Test(org.junit.Test)

Example 2 with NoOpTaskManagerActions

use of org.apache.flink.runtime.taskmanager.NoOpTaskManagerActions in project flink by apache.

the class StreamTaskTest method testRecordWriterClosedOnTransitStateError.

private void testRecordWriterClosedOnTransitStateError(ExecutionState executionState) throws Exception {
    // Throw the exception when the state updating to the expected one.
    NoOpTaskManagerActions taskManagerActions = new NoOpTaskManagerActions() {

        @Override
        public void updateTaskExecutionState(TaskExecutionState taskExecutionState) {
            if (taskExecutionState.getExecutionState() == executionState) {
                throw new ExpectedTestException();
            }
        }
    };
    testRecordWriterClosedOnError(env -> taskBuilderWithConfiguredRecordWriter(env).setTaskManagerActions(taskManagerActions).build());
}
Also used : NoOpTaskManagerActions(org.apache.flink.runtime.taskmanager.NoOpTaskManagerActions) ExpectedTestException(org.apache.flink.runtime.operators.testutils.ExpectedTestException) TaskExecutionState(org.apache.flink.runtime.taskmanager.TaskExecutionState)

Example 3 with NoOpTaskManagerActions

use of org.apache.flink.runtime.taskmanager.NoOpTaskManagerActions in project flink by apache.

the class TaskExecutorPartitionLifecycleTest method testJobMasterConnectionTerminationAfterExternalReleaseOrPromotion.

private void testJobMasterConnectionTerminationAfterExternalReleaseOrPromotion(TriConsumer<TaskExecutorGateway, JobID, ResultPartitionID> releaseOrPromoteCall) throws Exception {
    final CompletableFuture<Void> disconnectFuture = new CompletableFuture<>();
    final JobMasterGateway jobMasterGateway = new TestingJobMasterGatewayBuilder().setDisconnectTaskManagerFunction(resourceID -> {
        disconnectFuture.complete(null);
        return CompletableFuture.completedFuture(Acknowledge.get());
    }).build();
    final DefaultJobTable jobTable = DefaultJobTable.create();
    final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder().setJobTable(jobTable).setShuffleEnvironment(new NettyShuffleEnvironmentBuilder().build()).setTaskSlotTable(createTaskSlotTable()).build();
    final TestingTaskExecutorPartitionTracker partitionTracker = new TestingTaskExecutorPartitionTracker();
    final AtomicBoolean trackerIsTrackingPartitions = new AtomicBoolean(false);
    partitionTracker.setIsTrackingPartitionsForFunction(jobId -> trackerIsTrackingPartitions.get());
    final CompletableFuture<Collection<ResultPartitionID>> firstReleasePartitionsCallFuture = new CompletableFuture<>();
    partitionTracker.setStopTrackingAndReleasePartitionsConsumer(firstReleasePartitionsCallFuture::complete);
    final ResultPartitionDeploymentDescriptor resultPartitionDeploymentDescriptor = PartitionTestUtils.createPartitionDeploymentDescriptor(ResultPartitionType.BLOCKING);
    final ResultPartitionID resultPartitionId = resultPartitionDeploymentDescriptor.getShuffleDescriptor().getResultPartitionID();
    final TestingTaskExecutor taskExecutor = createTestingTaskExecutor(taskManagerServices, partitionTracker);
    try {
        taskExecutor.start();
        taskExecutor.waitUntilStarted();
        TaskSubmissionTestEnvironment.registerJobMasterConnection(jobTable, jobId, rpc, jobMasterGateway, new NoOpTaskManagerActions(), timeout, taskExecutor.getMainThreadExecutableForTesting());
        final TaskExecutorGateway taskExecutorGateway = taskExecutor.getSelfGateway(TaskExecutorGateway.class);
        trackerIsTrackingPartitions.set(true);
        assertThat(firstReleasePartitionsCallFuture.isDone(), is(false));
        taskExecutorGateway.releaseOrPromotePartitions(jobId, Collections.singleton(new ResultPartitionID()), Collections.emptySet());
        // at this point we only know that the TE has entered releasePartitions; we cannot be
        // certain whether it
        // has already checked whether it should disconnect or not
        firstReleasePartitionsCallFuture.get();
        // connection should be kept alive since the table still contains partitions
        assertThat(disconnectFuture.isDone(), is(false));
        trackerIsTrackingPartitions.set(false);
        // the TM should check whether partitions are still stored, and afterwards terminate the
        // connection
        releaseOrPromoteCall.accept(taskExecutorGateway, jobId, resultPartitionId);
        disconnectFuture.get();
    } finally {
        RpcUtils.terminateRpcEndpoint(taskExecutor, timeout);
    }
}
Also used : CoreMatchers.is(org.hamcrest.CoreMatchers.is) TestingRpcService(org.apache.flink.runtime.rpc.TestingRpcService) ShuffleEnvironment(org.apache.flink.runtime.shuffle.ShuffleEnvironment) InetAddress(java.net.InetAddress) ResultPartitionID(org.apache.flink.runtime.io.network.partition.ResultPartitionID) SettableLeaderRetrievalService(org.apache.flink.runtime.leaderretrieval.SettableLeaderRetrievalService) TestingFatalErrorHandler(org.apache.flink.runtime.util.TestingFatalErrorHandler) TaskExecutorPartitionTracker(org.apache.flink.runtime.io.network.partition.TaskExecutorPartitionTracker) After(org.junit.After) TestLogger(org.apache.flink.util.TestLogger) TestingJobMasterGatewayBuilder(org.apache.flink.runtime.jobmaster.utils.TestingJobMasterGatewayBuilder) ClassRule(org.junit.ClassRule) AfterClass(org.junit.AfterClass) Collection(java.util.Collection) AbstractInvokable(org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable) NoOpTaskExecutorBlobService(org.apache.flink.runtime.blob.NoOpTaskExecutorBlobService) TestingTaskExecutorPartitionTracker(org.apache.flink.runtime.io.network.partition.TestingTaskExecutorPartitionTracker) UUID(java.util.UUID) NettyShuffleEnvironment(org.apache.flink.runtime.io.network.NettyShuffleEnvironment) IntermediateDataSetID(org.apache.flink.runtime.jobgraph.IntermediateDataSetID) Acknowledge(org.apache.flink.runtime.messages.Acknowledge) ResourceProfile(org.apache.flink.runtime.clusterframework.types.ResourceProfile) HeartbeatServices(org.apache.flink.runtime.heartbeat.HeartbeatServices) TaskExecutorPartitionInfo(org.apache.flink.runtime.io.network.partition.TaskExecutorPartitionInfo) SerializedValue(org.apache.flink.util.SerializedValue) ResultPartitionDeploymentDescriptor(org.apache.flink.runtime.deployment.ResultPartitionDeploymentDescriptor) Matchers.equalTo(org.hamcrest.Matchers.equalTo) ExecutionConfig(org.apache.flink.api.common.ExecutionConfig) Optional(java.util.Optional) ResultPartitionManager(org.apache.flink.runtime.io.network.partition.ResultPartitionManager) Time(org.apache.flink.api.common.time.Time) Environment(org.apache.flink.runtime.execution.Environment) OneShotLatch(org.apache.flink.core.testutils.OneShotLatch) TaskExecutorPartitionTrackerImpl(org.apache.flink.runtime.io.network.partition.TaskExecutorPartitionTrackerImpl) NoOpTaskManagerActions(org.apache.flink.runtime.taskmanager.NoOpTaskManagerActions) TaskExecutorLocalStateStoresManager(org.apache.flink.runtime.state.TaskExecutorLocalStateStoresManager) BeforeClass(org.junit.BeforeClass) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ResultPartitionType(org.apache.flink.runtime.io.network.partition.ResultPartitionType) JMTMRegistrationSuccess(org.apache.flink.runtime.jobmaster.JMTMRegistrationSuccess) CompletableFuture(java.util.concurrent.CompletableFuture) TestingJobMasterGateway(org.apache.flink.runtime.jobmaster.utils.TestingJobMasterGateway) TaskDeploymentDescriptor(org.apache.flink.runtime.deployment.TaskDeploymentDescriptor) JobMasterGateway(org.apache.flink.runtime.jobmaster.JobMasterGateway) BlockerSync(org.apache.flink.core.testutils.BlockerSync) TaskSlotTable(org.apache.flink.runtime.taskexecutor.slot.TaskSlotTable) ExternalResourceInfoProvider(org.apache.flink.runtime.externalresource.ExternalResourceInfoProvider) ClusterInformation(org.apache.flink.runtime.entrypoint.ClusterInformation) TriConsumer(org.apache.flink.util.function.TriConsumer) StreamSupport(java.util.stream.StreamSupport) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) ResourceID(org.apache.flink.runtime.clusterframework.types.ResourceID) TestFileUtils(org.apache.flink.testutils.TestFileUtils) Before(org.junit.Before) TaskSlotUtils(org.apache.flink.runtime.taskexecutor.slot.TaskSlotUtils) TestExecutorResource(org.apache.flink.testutils.executor.TestExecutorResource) NettyShuffleEnvironmentBuilder(org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder) CoreMatchers.hasItems(org.hamcrest.CoreMatchers.hasItems) Configuration(org.apache.flink.configuration.Configuration) ExecutionState(org.apache.flink.runtime.execution.ExecutionState) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) IOException(java.io.IOException) InstanceID(org.apache.flink.runtime.instance.InstanceID) Reference(org.apache.flink.util.Reference) RpcUtils(org.apache.flink.runtime.rpc.RpcUtils) File(java.io.File) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) Executors(org.apache.flink.util.concurrent.Executors) ExecutionAttemptID(org.apache.flink.runtime.executiongraph.ExecutionAttemptID) TestingResourceManagerGateway(org.apache.flink.runtime.resourcemanager.utils.TestingResourceManagerGateway) JobID(org.apache.flink.api.common.JobID) UnregisteredMetricGroups(org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups) Task(org.apache.flink.runtime.taskmanager.Task) Rule(org.junit.Rule) PartitionTestUtils(org.apache.flink.runtime.io.network.partition.PartitionTestUtils) TestingHighAvailabilityServices(org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices) Collections(java.util.Collections) TemporaryFolder(org.junit.rules.TemporaryFolder) ResultPartitionDeploymentDescriptor(org.apache.flink.runtime.deployment.ResultPartitionDeploymentDescriptor) NettyShuffleEnvironmentBuilder(org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder) TestingJobMasterGateway(org.apache.flink.runtime.jobmaster.utils.TestingJobMasterGateway) JobMasterGateway(org.apache.flink.runtime.jobmaster.JobMasterGateway) NoOpTaskManagerActions(org.apache.flink.runtime.taskmanager.NoOpTaskManagerActions) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) CompletableFuture(java.util.concurrent.CompletableFuture) TestingTaskExecutorPartitionTracker(org.apache.flink.runtime.io.network.partition.TestingTaskExecutorPartitionTracker) TestingJobMasterGatewayBuilder(org.apache.flink.runtime.jobmaster.utils.TestingJobMasterGatewayBuilder) Collection(java.util.Collection) ResultPartitionID(org.apache.flink.runtime.io.network.partition.ResultPartitionID)

Aggregations

NoOpTaskManagerActions (org.apache.flink.runtime.taskmanager.NoOpTaskManagerActions)3 Configuration (org.apache.flink.configuration.Configuration)2 NettyShuffleEnvironment (org.apache.flink.runtime.io.network.NettyShuffleEnvironment)2 NettyShuffleEnvironmentBuilder (org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder)2 Task (org.apache.flink.runtime.taskmanager.Task)2 TaskExecutionState (org.apache.flink.runtime.taskmanager.TaskExecutionState)2 Test (org.junit.Test)2 File (java.io.File)1 IOException (java.io.IOException)1 InetAddress (java.net.InetAddress)1 Collection (java.util.Collection)1 Collections (java.util.Collections)1 Optional (java.util.Optional)1 UUID (java.util.UUID)1 CompletableFuture (java.util.concurrent.CompletableFuture)1 ExecutionException (java.util.concurrent.ExecutionException)1 TimeUnit (java.util.concurrent.TimeUnit)1 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)1 StreamSupport (java.util.stream.StreamSupport)1 ExecutionConfig (org.apache.flink.api.common.ExecutionConfig)1