Examples with TaskMetricGroup org.apache.flink.runtime.metrics.groups.TaskMetricGroup used on opensource projects

Example 6 with TaskMetricGroup

use of org.apache.flink.runtime.metrics.groups.TaskMetricGroup in project flink by apache.

the class OneInputStreamTaskTest method testCheckpointBarrierMetrics.

/**
 * Tests the checkpoint related metrics are registered into {@link TaskIOMetricGroup} correctly
 * while generating the {@link OneInputStreamTask}.
 */
@Test
public void testCheckpointBarrierMetrics() throws Exception {
    final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(OneInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    streamConfig.setStreamOperator(new TestOperator());
    final Map<String, Metric> metrics = new ConcurrentHashMap<>();
    final TaskMetricGroup taskMetricGroup = StreamTaskTestHarness.createTaskMetricGroup(metrics);
    final StreamMockEnvironment environment = testHarness.createEnvironment();
    environment.setTaskMetricGroup(taskMetricGroup);
    testHarness.invoke(environment);
    testHarness.waitForTaskRunning();
    assertThat(metrics, IsMapContaining.hasKey(MetricNames.CHECKPOINT_ALIGNMENT_TIME));
    assertThat(metrics, IsMapContaining.hasKey(MetricNames.CHECKPOINT_START_DELAY_TIME));
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
}

Example 7 with TaskMetricGroup

use of org.apache.flink.runtime.metrics.groups.TaskMetricGroup in project flink by apache.

the class TaskAsyncCallTest method createTask.

private Task createTask(Class<? extends AbstractInvokable> invokableClass) throws Exception {
    final TestingClassLoaderLease classLoaderHandle = TestingClassLoaderLease.newBuilder().setGetOrResolveClassLoaderFunction((permanentBlobKeys, urls) -> TestingUserCodeClassLoader.newBuilder().setClassLoader(new TestUserCodeClassLoader()).build()).build();
    ResultPartitionConsumableNotifier consumableNotifier = new NoOpResultPartitionConsumableNotifier();
    PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
    Executor executor = mock(Executor.class);
    TaskMetricGroup taskMetricGroup = UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();
    JobInformation jobInformation = new JobInformation(new JobID(), "Job Name", new SerializedValue<>(new ExecutionConfig()), new Configuration(), Collections.emptyList(), Collections.emptyList());
    TaskInformation taskInformation = new TaskInformation(new JobVertexID(), "Test Task", 1, 1, invokableClass.getName(), new Configuration());
    return new Task(jobInformation, taskInformation, new ExecutionAttemptID(), new AllocationID(), 0, 0, Collections.<ResultPartitionDeploymentDescriptor>emptyList(), Collections.<InputGateDeploymentDescriptor>emptyList(), mock(MemoryManager.class), mock(IOManager.class), shuffleEnvironment, new KvStateService(new KvStateRegistry(), null, null), mock(BroadcastVariableManager.class), new TaskEventDispatcher(), ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES, new TestTaskStateManager(), mock(TaskManagerActions.class), mock(InputSplitProvider.class), mock(CheckpointResponder.class), new NoOpTaskOperatorEventGateway(), new TestGlobalAggregateManager(), classLoaderHandle, mock(FileCache.class), new TestingTaskManagerRuntimeInfo(), taskMetricGroup, consumableNotifier, partitionProducerStateChecker, executor);
}

Example 8 with TaskMetricGroup

use of org.apache.flink.runtime.metrics.groups.TaskMetricGroup in project flink by apache.

the class TwoInputStreamTaskTest method testCheckpointBarrierMetrics.

/**
 * Tests the checkpoint related metrics are registered into {@link TaskIOMetricGroup} correctly
 * while generating the {@link TwoInputStreamTask}.
 */
@Test
public void testCheckpointBarrierMetrics() throws Exception {
    final TwoInputStreamTaskTestHarness<String, Integer, String> testHarness = new TwoInputStreamTaskTestHarness<>(TwoInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    final StreamConfig streamConfig = testHarness.getStreamConfig();
    final CoStreamMap<String, Integer, String> coMapOperator = new CoStreamMap<>(new IdentityMap());
    testHarness.setupOutputForSingletonOperatorChain();
    streamConfig.setStreamOperator(coMapOperator);
    final Map<String, Metric> metrics = new ConcurrentHashMap<>();
    final TaskMetricGroup taskMetricGroup = StreamTaskTestHarness.createTaskMetricGroup(metrics);
    final StreamMockEnvironment environment = testHarness.createEnvironment();
    environment.setTaskMetricGroup(taskMetricGroup);
    testHarness.invoke(environment);
    testHarness.waitForTaskRunning();
    assertThat(metrics, IsMapContaining.hasKey(MetricNames.CHECKPOINT_ALIGNMENT_TIME));
    assertThat(metrics, IsMapContaining.hasKey(MetricNames.CHECKPOINT_START_DELAY_TIME));
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
}

Example 9 with TaskMetricGroup

use of org.apache.flink.runtime.metrics.groups.TaskMetricGroup in project flink by apache.

the class TaskExecutor method submitTask.

// ======================================================================
//  RPC methods
// ======================================================================
// ----------------------------------------------------------------------
// Task lifecycle RPCs
// ----------------------------------------------------------------------
@RpcMethod
public Acknowledge submitTask(TaskDeploymentDescriptor tdd, UUID jobManagerLeaderId) throws TaskSubmissionException {
    // first, deserialize the pre-serialized information
    final JobInformation jobInformation;
    final TaskInformation taskInformation;
    try {
        jobInformation = tdd.getSerializedJobInformation().deserializeValue(getClass().getClassLoader());
        taskInformation = tdd.getSerializedTaskInformation().deserializeValue(getClass().getClassLoader());
    } catch (IOException | ClassNotFoundException e) {
        throw new TaskSubmissionException("Could not deserialize the job or task information.", e);
    }
    final JobID jobId = jobInformation.getJobId();
    final JobManagerConnection jobManagerConnection = jobManagerTable.get(jobId);
    if (jobManagerConnection == null) {
        final String message = "Could not submit task because there is no JobManager " + "associated for the job " + jobId + '.';
        log.debug(message);
        throw new TaskSubmissionException(message);
    }
    if (!jobManagerConnection.getLeaderId().equals(jobManagerLeaderId)) {
        final String message = "Rejecting the task submission because the job manager leader id " + jobManagerLeaderId + " does not match the expected job manager leader id " + jobManagerConnection.getLeaderId() + '.';
        log.debug(message);
        throw new TaskSubmissionException(message);
    }
    if (!taskSlotTable.existsActiveSlot(jobId, tdd.getAllocationId())) {
        final String message = "No task slot allocated for job ID " + jobId + " and allocation ID " + tdd.getAllocationId() + '.';
        log.debug(message);
        throw new TaskSubmissionException(message);
    }
    TaskMetricGroup taskMetricGroup = taskManagerMetricGroup.addTaskForJob(jobInformation.getJobId(), jobInformation.getJobName(), taskInformation.getJobVertexId(), tdd.getExecutionAttemptId(), taskInformation.getTaskName(), tdd.getSubtaskIndex(), tdd.getAttemptNumber());
    InputSplitProvider inputSplitProvider = new RpcInputSplitProvider(jobManagerConnection.getLeaderId(), jobManagerConnection.getJobManagerGateway(), jobInformation.getJobId(), taskInformation.getJobVertexId(), tdd.getExecutionAttemptId(), taskManagerConfiguration.getTimeout());
    TaskManagerActions taskManagerActions = jobManagerConnection.getTaskManagerActions();
    CheckpointResponder checkpointResponder = jobManagerConnection.getCheckpointResponder();
    LibraryCacheManager libraryCache = jobManagerConnection.getLibraryCacheManager();
    ResultPartitionConsumableNotifier resultPartitionConsumableNotifier = jobManagerConnection.getResultPartitionConsumableNotifier();
    PartitionProducerStateChecker partitionStateChecker = jobManagerConnection.getPartitionStateChecker();
    Task task = new Task(jobInformation, taskInformation, tdd.getExecutionAttemptId(), tdd.getAllocationId(), tdd.getSubtaskIndex(), tdd.getAttemptNumber(), tdd.getProducedPartitions(), tdd.getInputGates(), tdd.getTargetSlotNumber(), tdd.getTaskStateHandles(), memoryManager, ioManager, networkEnvironment, broadcastVariableManager, taskManagerActions, inputSplitProvider, checkpointResponder, libraryCache, fileCache, taskManagerConfiguration, taskMetricGroup, resultPartitionConsumableNotifier, partitionStateChecker, getRpcService().getExecutor());
    log.info("Received task {}.", task.getTaskInfo().getTaskNameWithSubtasks());
    boolean taskAdded;
    try {
        taskAdded = taskSlotTable.addTask(task);
    } catch (SlotNotFoundException | SlotNotActiveException e) {
        throw new TaskSubmissionException("Could not submit task.", e);
    }
    if (taskAdded) {
        task.startTaskThread();
        return Acknowledge.get();
    } else {
        final String message = "TaskManager already contains a task for id " + task.getExecutionId() + '.';
        log.debug(message);
        throw new TaskSubmissionException(message);
    }
}

Example 10 with TaskMetricGroup

use of org.apache.flink.runtime.metrics.groups.TaskMetricGroup in project flink by apache.

the class SynchronousCheckpointITCase method createTask.

// --------------------------		Boilerplate tools copied from the TaskAsyncCallTest
// --------------------------
private Task createTask(Class<? extends TaskInvokable> invokableClass) throws Exception {
    ResultPartitionConsumableNotifier consumableNotifier = new NoOpResultPartitionConsumableNotifier();
    PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
    Executor executor = mock(Executor.class);
    ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();
    TaskMetricGroup taskMetricGroup = UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();
    JobInformation jobInformation = new JobInformation(new JobID(), "Job Name", new SerializedValue<>(new ExecutionConfig()), new Configuration(), Collections.emptyList(), Collections.emptyList());
    TaskInformation taskInformation = new TaskInformation(new JobVertexID(), "Test Task", 1, 1, invokableClass.getName(), new Configuration());
    return new Task(jobInformation, taskInformation, new ExecutionAttemptID(), new AllocationID(), 0, 0, Collections.<ResultPartitionDeploymentDescriptor>emptyList(), Collections.<InputGateDeploymentDescriptor>emptyList(), mock(MemoryManager.class), mock(IOManager.class), shuffleEnvironment, new KvStateService(new KvStateRegistry(), null, null), mock(BroadcastVariableManager.class), new TaskEventDispatcher(), ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES, new TestTaskStateManager(), mock(TaskManagerActions.class), mock(InputSplitProvider.class), mock(CheckpointResponder.class), new NoOpTaskOperatorEventGateway(), new TestGlobalAggregateManager(), TestingClassLoaderLease.newBuilder().build(), mock(FileCache.class), new TestingTaskManagerRuntimeInfo(), taskMetricGroup, consumableNotifier, partitionProducerStateChecker, executor);
}