Examples with TaskManagerRuntimeInfo org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo used on opensource projects

Example 1 with TaskManagerRuntimeInfo

use of org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo in project flink by apache.

the class AsyncWaitOperatorTest method testAsyncTimeout.

@Test
public void testAsyncTimeout() throws Exception {
    final long timeout = 10L;
    final AsyncWaitOperator<Integer, Integer> operator = new AsyncWaitOperator<>(new LazyAsyncFunction(), timeout, 2, AsyncDataStream.OutputMode.ORDERED);
    final Environment mockEnvironment = mock(Environment.class);
    final Configuration taskConfiguration = new Configuration();
    final ExecutionConfig executionConfig = new ExecutionConfig();
    final TaskMetricGroup metricGroup = new UnregisteredTaskMetricsGroup();
    final TaskManagerRuntimeInfo taskManagerRuntimeInfo = new TestingTaskManagerRuntimeInfo();
    final TaskInfo taskInfo = new TaskInfo("foobarTask", 1, 0, 1, 1);
    when(mockEnvironment.getTaskConfiguration()).thenReturn(taskConfiguration);
    when(mockEnvironment.getExecutionConfig()).thenReturn(executionConfig);
    when(mockEnvironment.getMetricGroup()).thenReturn(metricGroup);
    when(mockEnvironment.getTaskManagerInfo()).thenReturn(taskManagerRuntimeInfo);
    when(mockEnvironment.getTaskInfo()).thenReturn(taskInfo);
    when(mockEnvironment.getUserClassLoader()).thenReturn(AsyncWaitOperatorTest.class.getClassLoader());
    final OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<>(operator, IntSerializer.INSTANCE, mockEnvironment);
    final long initialTime = 0L;
    final ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    testHarness.open();
    testHarness.setProcessingTime(initialTime);
    synchronized (testHarness.getCheckpointLock()) {
        testHarness.processElement(new StreamRecord<>(1, initialTime));
        testHarness.setProcessingTime(initialTime + 5L);
        testHarness.processElement(new StreamRecord<>(2, initialTime + 5L));
    }
    // trigger the timeout of the first stream record
    testHarness.setProcessingTime(initialTime + timeout + 1L);
    // allow the second async stream record to be processed
    LazyAsyncFunction.countDown();
    // wait until all async collectors in the buffer have been emitted out.
    synchronized (testHarness.getCheckpointLock()) {
        testHarness.close();
    }
    expectedOutput.add(new StreamRecord<>(2, initialTime + 5L));
    TestHarnessUtil.assertOutputEquals("Output with watermark was not correct.", expectedOutput, testHarness.getOutput());
    ArgumentCaptor<Throwable> argumentCaptor = ArgumentCaptor.forClass(Throwable.class);
    verify(mockEnvironment).failExternally(argumentCaptor.capture());
    Throwable failureCause = argumentCaptor.getValue();
    Assert.assertNotNull(failureCause.getCause());
    Assert.assertTrue(failureCause.getCause() instanceof ExecutionException);
    Assert.assertNotNull(failureCause.getCause().getCause());
    Assert.assertTrue(failureCause.getCause().getCause() instanceof TimeoutException);
}

Example 2 with TaskManagerRuntimeInfo

use of org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo in project flink by apache.

the class AccumulatingAlignedProcessingTimeWindowOperatorTest method createMockTask.

// ------------------------------------------------------------------------
private static StreamTask<?, ?> createMockTask() {
    Configuration configuration = new Configuration();
    configuration.setString(CoreOptions.STATE_BACKEND, "jobmanager");
    StreamTask<?, ?> task = mock(StreamTask.class);
    when(task.getAccumulatorMap()).thenReturn(new HashMap<String, Accumulator<?, ?>>());
    when(task.getName()).thenReturn("Test task name");
    when(task.getExecutionConfig()).thenReturn(new ExecutionConfig());
    final TaskManagerRuntimeInfo mockTaskManagerRuntimeInfo = mock(TaskManagerRuntimeInfo.class);
    when(mockTaskManagerRuntimeInfo.getConfiguration()).thenReturn(configuration);
    final Environment env = mock(Environment.class);
    when(env.getTaskInfo()).thenReturn(new TaskInfo("Test task name", 1, 0, 1, 0));
    when(env.getUserClassLoader()).thenReturn(AggregatingAlignedProcessingTimeWindowOperatorTest.class.getClassLoader());
    when(env.getMetricGroup()).thenReturn(new UnregisteredTaskMetricsGroup());
    when(env.getTaskManagerInfo()).thenReturn(new TestingTaskManagerRuntimeInfo());
    when(task.getEnvironment()).thenReturn(env);
    return task;
}

Example 3 with TaskManagerRuntimeInfo

use of org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo in project flink by apache.

the class StreamTaskTerminationTest method testConcurrentAsyncCheckpointCannotFailFinishedStreamTask.

/**
 * FLINK-6833
 *
 * <p>Tests that a finished stream task cannot be failed by an asynchronous checkpointing
 * operation after the stream task has stopped running.
 */
@Test
public void testConcurrentAsyncCheckpointCannotFailFinishedStreamTask() throws Exception {
    final Configuration taskConfiguration = new Configuration();
    final StreamConfig streamConfig = new StreamConfig(taskConfiguration);
    final NoOpStreamOperator<Long> noOpStreamOperator = new NoOpStreamOperator<>();
    final StateBackend blockingStateBackend = new BlockingStateBackend();
    streamConfig.setStreamOperator(noOpStreamOperator);
    streamConfig.setOperatorID(new OperatorID());
    streamConfig.setStateBackend(blockingStateBackend);
    final long checkpointId = 0L;
    final long checkpointTimestamp = 0L;
    final JobInformation jobInformation = new JobInformation(new JobID(), "Test Job", new SerializedValue<>(new ExecutionConfig()), new Configuration(), Collections.emptyList(), Collections.emptyList());
    final TaskInformation taskInformation = new TaskInformation(new JobVertexID(), "Test Task", 1, 1, BlockingStreamTask.class.getName(), taskConfiguration);
    final TaskManagerRuntimeInfo taskManagerRuntimeInfo = new TestingTaskManagerRuntimeInfo();
    final ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();
    final Task task = new Task(jobInformation, taskInformation, new ExecutionAttemptID(), new AllocationID(), 0, 0, Collections.<ResultPartitionDeploymentDescriptor>emptyList(), Collections.<InputGateDeploymentDescriptor>emptyList(), MemoryManagerBuilder.newBuilder().setMemorySize(32L * 1024L).build(), new IOManagerAsync(), shuffleEnvironment, new KvStateService(new KvStateRegistry(), null, null), mock(BroadcastVariableManager.class), new TaskEventDispatcher(), ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES, new TestTaskStateManager(), mock(TaskManagerActions.class), mock(InputSplitProvider.class), mock(CheckpointResponder.class), new NoOpTaskOperatorEventGateway(), new TestGlobalAggregateManager(), TestingClassLoaderLease.newBuilder().build(), mock(FileCache.class), taskManagerRuntimeInfo, UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(), new NoOpResultPartitionConsumableNotifier(), mock(PartitionProducerStateChecker.class), Executors.directExecutor());
    CompletableFuture<Void> taskRun = CompletableFuture.runAsync(() -> task.run(), TestingUtils.defaultExecutor());
    // wait until the stream task started running
    RUN_LATCH.await();
    // trigger a checkpoint
    task.triggerCheckpointBarrier(checkpointId, checkpointTimestamp, CheckpointOptions.forCheckpointWithDefaultLocation());
    // wait until the task has completed execution
    taskRun.get();
    // check that no failure occurred
    if (task.getFailureCause() != null) {
        throw new Exception("Task failed", task.getFailureCause());
    }
    // check that we have entered the finished state
    assertEquals(ExecutionState.FINISHED, task.getExecutionState());
}

Example 4 with TaskManagerRuntimeInfo

use of org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo in project flink by apache.

the class RocksDBStateBackendConfigTest method getMockEnvironment.

static Environment getMockEnvironment(File[] tempDirs) {
    final String[] tempDirStrings = new String[tempDirs.length];
    for (int i = 0; i < tempDirs.length; i++) {
        tempDirStrings[i] = tempDirs[i].getAbsolutePath();
    }
    IOManager ioMan = mock(IOManager.class);
    when(ioMan.getSpillingDirectories()).thenReturn(tempDirs);
    Environment env = mock(Environment.class);
    when(env.getJobID()).thenReturn(new JobID());
    when(env.getUserClassLoader()).thenReturn(RocksDBStateBackendConfigTest.class.getClassLoader());
    when(env.getIOManager()).thenReturn(ioMan);
    when(env.getTaskKvStateRegistry()).thenReturn(new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()));
    TaskInfo taskInfo = mock(TaskInfo.class);
    when(env.getTaskInfo()).thenReturn(taskInfo);
    when(taskInfo.getIndexOfThisSubtask()).thenReturn(0);
    TaskManagerRuntimeInfo tmInfo = new TestingTaskManagerRuntimeInfo(new Configuration(), tempDirStrings);
    when(env.getTaskManagerInfo()).thenReturn(tmInfo);
    return env;
}

Example 5 with TaskManagerRuntimeInfo

use of org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo in project flink by apache.

the class StreamTaskSystemExitTest method createSystemExitTask.

private Task createSystemExitTask(final String invokableClassName, StreamOperator<?> operator) throws Exception {
    final Configuration taskConfiguration = new Configuration();
    final StreamConfig streamConfig = new StreamConfig(taskConfiguration);
    streamConfig.setOperatorID(new OperatorID());
    streamConfig.setStreamOperator(operator);
    // for source run
    streamConfig.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);
    final JobInformation jobInformation = new JobInformation(new JobID(), "Test Job", new SerializedValue<>(new ExecutionConfig()), new Configuration(), Collections.emptyList(), Collections.emptyList());
    final TaskInformation taskInformation = new TaskInformation(new JobVertexID(), "Test Task", 1, 1, invokableClassName, taskConfiguration);
    final TaskManagerRuntimeInfo taskManagerRuntimeInfo = new TestingTaskManagerRuntimeInfo();
    final ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();
    return new Task(jobInformation, taskInformation, new ExecutionAttemptID(), new AllocationID(), 0, 0, Collections.<ResultPartitionDeploymentDescriptor>emptyList(), Collections.<InputGateDeploymentDescriptor>emptyList(), MemoryManagerBuilder.newBuilder().setMemorySize(32L * 1024L).build(), new IOManagerAsync(), shuffleEnvironment, new KvStateService(new KvStateRegistry(), null, null), mock(BroadcastVariableManager.class), new TaskEventDispatcher(), ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES, new TestTaskStateManager(), mock(TaskManagerActions.class), mock(InputSplitProvider.class), mock(CheckpointResponder.class), new NoOpTaskOperatorEventGateway(), new TestGlobalAggregateManager(), TestingClassLoaderLease.newBuilder().build(), mock(FileCache.class), taskManagerRuntimeInfo, UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(), new NoOpResultPartitionConsumableNotifier(), mock(PartitionProducerStateChecker.class), Executors.directExecutor());
}