Examples with OperatorIOMetricGroup org.apache.flink.metrics.groups.OperatorIOMetricGroup used on opensource projects

Example 1 with OperatorIOMetricGroup

use of org.apache.flink.metrics.groups.OperatorIOMetricGroup in project flink by apache.

the class FileWriterTest method testNumberRecordsOutCounter.

@Test
public void testNumberRecordsOutCounter() throws IOException, InterruptedException {
    final OperatorIOMetricGroup operatorIOMetricGroup = UnregisteredMetricGroups.createUnregisteredOperatorMetricGroup().getIOMetricGroup();
    File outDir = TEMP_FOLDER.newFolder();
    Path path = new Path(outDir.toURI());
    Counter recordsCounter = operatorIOMetricGroup.getNumRecordsOutCounter();
    SinkWriter.Context context = new ContextImpl();
    FileWriter<String> fileWriter = createWriter(path, DefaultRollingPolicy.builder().build(), new OutputFileConfig("part-", ""), operatorIOMetricGroup);
    assertEquals(0, recordsCounter.getCount());
    fileWriter.write("1", context);
    assertEquals(1, recordsCounter.getCount());
    fileWriter.write("2", context);
    fileWriter.write("3", context);
    assertEquals(3, recordsCounter.getCount());
}

Example 2 with OperatorIOMetricGroup

use of org.apache.flink.metrics.groups.OperatorIOMetricGroup in project flink by apache.

the class KafkaWriterITCase method testIncreasingRecordBasedCounters.

@Test
public void testIncreasingRecordBasedCounters() throws Exception {
    final OperatorIOMetricGroup operatorIOMetricGroup = UnregisteredMetricGroups.createUnregisteredOperatorMetricGroup().getIOMetricGroup();
    final InternalSinkWriterMetricGroup metricGroup = InternalSinkWriterMetricGroup.mock(metricListener.getMetricGroup(), operatorIOMetricGroup);
    try (final KafkaWriter<Integer> writer = createWriterWithConfiguration(getKafkaClientConfiguration(), DeliveryGuarantee.NONE, metricGroup)) {
        final Counter numBytesOut = operatorIOMetricGroup.getNumBytesOutCounter();
        final Counter numRecordsOut = operatorIOMetricGroup.getNumRecordsOutCounter();
        assertEquals(numBytesOut.getCount(), 0L);
        writer.write(1, SINK_WRITER_CONTEXT);
        timeService.trigger();
        assertEquals(numRecordsOut.getCount(), 1);
        assertThat(numBytesOut.getCount(), greaterThan(0L));
    }
}

Example 3 with OperatorIOMetricGroup

use of org.apache.flink.metrics.groups.OperatorIOMetricGroup in project flink by apache.

the class ChainedOperatorsMetricTest method testOperatorIOMetricReuse.

@Test
public void testOperatorIOMetricReuse() throws Exception {
    // environment
    initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE);
    this.mockEnv = new MockEnvironmentBuilder().setTaskName(HEAD_OPERATOR_NAME).setManagedMemorySize(MEMORY_MANAGER_SIZE).setInputSplitProvider(this.inputSplitProvider).setBufferSize(NETWORK_BUFFER_SIZE).setMetricGroup(TaskManagerMetricGroup.createTaskManagerMetricGroup(NoOpMetricRegistry.INSTANCE, "host", ResourceID.generate()).addJob(new JobID(), "jobName").addTask(new JobVertexID(), new ExecutionAttemptID(), "task", 0, 0)).build();
    final int keyCnt = 100;
    final int valCnt = 20;
    final int numRecords = keyCnt * valCnt;
    addInput(new UniformRecordGenerator(keyCnt, valCnt, false), 0);
    addOutput(this.outList);
    // the chained operator
    addChainedOperator();
    // creates the head operator and assembles the chain
    registerTask(FlatMapDriver.class, DuplicatingFlatMapFunction.class);
    final BatchTask<FlatMapFunction<Record, Record>, Record> testTask = new BatchTask<>(this.mockEnv);
    testTask.invoke();
    Assert.assertEquals(numRecords * 2 * 2, this.outList.size());
    final TaskMetricGroup taskMetricGroup = mockEnv.getMetricGroup();
    // verify task-level metrics
    {
        final TaskIOMetricGroup ioMetricGroup = taskMetricGroup.getIOMetricGroup();
        final Counter numRecordsInCounter = ioMetricGroup.getNumRecordsInCounter();
        final Counter numRecordsOutCounter = ioMetricGroup.getNumRecordsOutCounter();
        Assert.assertEquals(numRecords, numRecordsInCounter.getCount());
        Assert.assertEquals(numRecords * 2 * 2, numRecordsOutCounter.getCount());
    }
    // verify head operator metrics
    {
        // this only returns the existing group and doesn't create a new one
        final OperatorMetricGroup operatorMetricGroup1 = taskMetricGroup.getOrAddOperator(HEAD_OPERATOR_NAME);
        final OperatorIOMetricGroup ioMetricGroup = operatorMetricGroup1.getIOMetricGroup();
        final Counter numRecordsInCounter = ioMetricGroup.getNumRecordsInCounter();
        final Counter numRecordsOutCounter = ioMetricGroup.getNumRecordsOutCounter();
        Assert.assertEquals(numRecords, numRecordsInCounter.getCount());
        Assert.assertEquals(numRecords * 2, numRecordsOutCounter.getCount());
    }
    // verify chained operator metrics
    {
        // this only returns the existing group and doesn't create a new one
        final InternalOperatorMetricGroup operatorMetricGroup1 = taskMetricGroup.getOrAddOperator(CHAINED_OPERATOR_NAME);
        final InternalOperatorIOMetricGroup ioMetricGroup = operatorMetricGroup1.getIOMetricGroup();
        final Counter numRecordsInCounter = ioMetricGroup.getNumRecordsInCounter();
        final Counter numRecordsOutCounter = ioMetricGroup.getNumRecordsOutCounter();
        Assert.assertEquals(numRecords * 2, numRecordsInCounter.getCount());
        Assert.assertEquals(numRecords * 2 * 2, numRecordsOutCounter.getCount());
    }
}

Example 4 with OperatorIOMetricGroup

use of org.apache.flink.metrics.groups.OperatorIOMetricGroup in project flink by apache.

the class ElasticsearchWriterITCase method testIncrementByteOutMetric.

@Test
void testIncrementByteOutMetric() throws Exception {
    final String index = "test-inc-byte-out";
    final OperatorIOMetricGroup operatorIOMetricGroup = UnregisteredMetricGroups.createUnregisteredOperatorMetricGroup().getIOMetricGroup();
    final InternalSinkWriterMetricGroup metricGroup = InternalSinkWriterMetricGroup.mock(metricListener.getMetricGroup(), operatorIOMetricGroup);
    final int flushAfterNActions = 2;
    final BulkProcessorConfig bulkProcessorConfig = new BulkProcessorConfig(flushAfterNActions, -1, -1, FlushBackoffType.NONE, 0, 0);
    try (final ElasticsearchWriter<Tuple2<Integer, String>> writer = createWriter(index, false, bulkProcessorConfig, metricGroup)) {
        final Counter numBytesOut = operatorIOMetricGroup.getNumBytesOutCounter();
        assertEquals(numBytesOut.getCount(), 0);
        writer.write(Tuple2.of(1, buildMessage(1)), null);
        writer.write(Tuple2.of(2, buildMessage(2)), null);
        writer.blockingFlushAllActions();
        long first = numBytesOut.getCount();
        assertTrue(first > 0);
        writer.write(Tuple2.of(1, buildMessage(1)), null);
        writer.write(Tuple2.of(2, buildMessage(2)), null);
        writer.blockingFlushAllActions();
        assertTrue(numBytesOut.getCount() > first);
    }
}