Examples with TaskMetricGroup org.apache.flink.runtime.metrics.groups.TaskMetricGroup used on opensource projects

Example 11 with TaskMetricGroup

use of org.apache.flink.runtime.metrics.groups.TaskMetricGroup in project flink by apache.

the class SourceStreamTaskTestBase method testMetrics.

public void testMetrics(FunctionWithException<Environment, ? extends StreamTask<Integer, ?>, Exception> taskFactory, StreamOperatorFactory<?> operatorFactory, Matcher<Double> busyTimeMatcher) throws Exception {
    long sleepTime = 42;
    StreamTaskMailboxTestHarnessBuilder<Integer> builder = new StreamTaskMailboxTestHarnessBuilder<>(taskFactory, INT_TYPE_INFO);
    final Map<String, Metric> metrics = new ConcurrentHashMap<>();
    final TaskMetricGroup taskMetricGroup = StreamTaskTestHarness.createTaskMetricGroup(metrics);
    try (StreamTaskMailboxTestHarness<Integer> harness = builder.setupOutputForSingletonOperatorChain(operatorFactory).setTaskMetricGroup(taskMetricGroup).build()) {
        Future<Boolean> triggerFuture = harness.streamTask.triggerCheckpointAsync(new CheckpointMetaData(1L, System.currentTimeMillis()), CheckpointOptions.forCheckpointWithDefaultLocation());
        OneShotLatch checkpointAcknowledgeLatch = new OneShotLatch();
        harness.getCheckpointResponder().setAcknowledgeLatch(checkpointAcknowledgeLatch);
        assertFalse(triggerFuture.isDone());
        Thread.sleep(sleepTime);
        while (!triggerFuture.isDone()) {
            harness.streamTask.runMailboxStep();
        }
        Gauge<Long> checkpointStartDelayGauge = (Gauge<Long>) metrics.get(MetricNames.CHECKPOINT_START_DELAY_TIME);
        assertThat(checkpointStartDelayGauge.getValue(), greaterThanOrEqualTo(sleepTime * 1_000_000));
        Gauge<Double> busyTimeGauge = (Gauge<Double>) metrics.get(MetricNames.TASK_BUSY_TIME);
        assertThat(busyTimeGauge.getValue(), busyTimeMatcher);
        checkpointAcknowledgeLatch.await();
        TestCheckpointResponder.AcknowledgeReport acknowledgeReport = Iterables.getOnlyElement(harness.getCheckpointResponder().getAcknowledgeReports());
        assertThat(acknowledgeReport.getCheckpointMetrics().getCheckpointStartDelayNanos(), greaterThanOrEqualTo(sleepTime * 1_000_000));
    }
}

Example 12 with TaskMetricGroup

use of org.apache.flink.runtime.metrics.groups.TaskMetricGroup in project flink by apache.

the class TwoInputStreamTaskTest method testOperatorMetricReuse.

@Test
public void testOperatorMetricReuse() throws Exception {
    final TwoInputStreamTaskTestHarness<String, String, String> testHarness = new TwoInputStreamTaskTestHarness<>(TwoInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOperatorChain(new OperatorID(), new DuplicatingOperator()).chain(new OperatorID(), new OneInputStreamTaskTest.DuplicatingOperator(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig())).chain(new OperatorID(), new OneInputStreamTaskTest.DuplicatingOperator(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig())).finish();
    final TaskMetricGroup taskMetricGroup = TaskManagerMetricGroup.createTaskManagerMetricGroup(NoOpMetricRegistry.INSTANCE, "host", ResourceID.generate()).addJob(new JobID(), "jobname").addTask(new JobVertexID(), new ExecutionAttemptID(), "task", 0, 0);
    final StreamMockEnvironment env = new StreamMockEnvironment(testHarness.jobConfig, testHarness.taskConfig, testHarness.memorySize, new MockInputSplitProvider(), testHarness.bufferSize, new TestTaskStateManager()) {

        @Override
        public TaskMetricGroup getMetricGroup() {
            return taskMetricGroup;
        }
    };
    final Counter numRecordsInCounter = taskMetricGroup.getIOMetricGroup().getNumRecordsInCounter();
    final Counter numRecordsOutCounter = taskMetricGroup.getIOMetricGroup().getNumRecordsOutCounter();
    testHarness.invoke(env);
    testHarness.waitForTaskRunning();
    final int numRecords1 = 5;
    final int numRecords2 = 3;
    for (int x = 0; x < numRecords1; x++) {
        testHarness.processElement(new StreamRecord<>("hello"), 0, 0);
    }
    for (int x = 0; x < numRecords2; x++) {
        testHarness.processElement(new StreamRecord<>("hello"), 1, 0);
    }
    testHarness.waitForInputProcessing();
    assertEquals(numRecords1 + numRecords2, numRecordsInCounter.getCount());
    assertEquals((numRecords1 + numRecords2) * 2 * 2 * 2, numRecordsOutCounter.getCount());
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
}

Example 13 with TaskMetricGroup

use of org.apache.flink.runtime.metrics.groups.TaskMetricGroup in project flink by apache.

the class SingleInputGateTest method createSingleInputGate.

static SingleInputGate createSingleInputGate(IntermediateResultPartitionID[] partitionIds, ResultPartitionType resultPartitionType, SubpartitionIndexRange subpartitionIndexRange, NettyShuffleEnvironment netEnv, ResourceID localLocation, ConnectionManager connectionManager, ResultPartitionManager resultPartitionManager) throws IOException {
    ShuffleDescriptor[] channelDescs = new ShuffleDescriptor[] { // Local
    createRemoteWithIdAndLocation(partitionIds[0], localLocation), // Remote
    createRemoteWithIdAndLocation(partitionIds[1], ResourceID.generate()), // Unknown
    new UnknownShuffleDescriptor(new ResultPartitionID(partitionIds[2], new ExecutionAttemptID())) };
    InputGateDeploymentDescriptor gateDesc = new InputGateDeploymentDescriptor(new IntermediateDataSetID(), resultPartitionType, subpartitionIndexRange, new TaskDeploymentDescriptor.NonOffloaded<>(CompressedSerializedValue.fromObject(channelDescs)));
    final TaskMetricGroup taskMetricGroup = UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();
    return new SingleInputGateFactory(localLocation, netEnv.getConfiguration(), connectionManager != null ? connectionManager : netEnv.getConnectionManager(), resultPartitionManager != null ? resultPartitionManager : netEnv.getResultPartitionManager(), new TaskEventDispatcher(), netEnv.getNetworkBufferPool()).create(netEnv.createShuffleIOOwnerContext("TestTask", taskMetricGroup.executionId(), taskMetricGroup), 0, gateDesc, SingleInputGateBuilder.NO_OP_PRODUCER_CHECKER);
}

Example 14 with TaskMetricGroup

use of org.apache.flink.runtime.metrics.groups.TaskMetricGroup in project flink by apache.

the class TestTaskBuilder method build.

public Task build() throws Exception {
    final JobVertexID jobVertexId = new JobVertexID();
    final SerializedValue<ExecutionConfig> serializedExecutionConfig = new SerializedValue<>(executionConfig);
    final JobInformation jobInformation = new JobInformation(jobId, "Test Job", serializedExecutionConfig, new Configuration(), requiredJarFileBlobKeys, Collections.emptyList());
    final TaskInformation taskInformation = new TaskInformation(jobVertexId, "Test Task", 1, 1, invokable.getName(), taskConfig);
    final TaskMetricGroup taskMetricGroup = UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();
    return new Task(jobInformation, taskInformation, executionAttemptId, allocationID, 0, 0, resultPartitions, inputGates, MemoryManagerBuilder.newBuilder().setMemorySize(1024 * 1024).build(), mock(IOManager.class), shuffleEnvironment, kvStateService, new BroadcastVariableManager(), new TaskEventDispatcher(), externalResourceInfoProvider, new TestTaskStateManager(), taskManagerActions, new MockInputSplitProvider(), testCheckpointResponder, new NoOpTaskOperatorEventGateway(), new TestGlobalAggregateManager(), classLoaderHandle, mock(FileCache.class), new TestingTaskManagerRuntimeInfo(taskManagerConfig), taskMetricGroup, consumableNotifier, partitionProducerStateChecker, executor);
}

Example 15 with TaskMetricGroup

use of org.apache.flink.runtime.metrics.groups.TaskMetricGroup in project flink by apache.

the class StreamNetworkBenchmarkEnvironment method createInputGateWithMetrics.

private IndexedInputGate createInputGateWithMetrics(SingleInputGateFactory gateFactory, InputGateDeploymentDescriptor gateDescriptor, int gateIndex) {
    final TaskMetricGroup taskMetricGroup = UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();
    final SingleInputGate singleGate = gateFactory.create(receiverEnv.createShuffleIOOwnerContext("receiving task[" + gateIndex + "]", taskMetricGroup.executionId(), taskMetricGroup), gateIndex, gateDescriptor, SingleInputGateBuilder.NO_OP_PRODUCER_CHECKER);
    return new InputGateWithMetrics(singleGate, new SimpleCounter());
}