Examples with LongCount io.trino.plugin.base.metrics.LongCount used on opensource projects

Example 1 with LongCount

use of io.trino.plugin.base.metrics.LongCount in project trino by trinodb.

the class TestOperatorStats method testAdd.

@Test
public void testAdd() {
    OperatorStats actual = EXPECTED.add(ImmutableList.of(EXPECTED, EXPECTED));
    assertEquals(actual.getStageId(), 0);
    assertEquals(actual.getOperatorId(), 41);
    assertEquals(actual.getOperatorType(), "test");
    assertEquals(actual.getTotalDrivers(), 3 * 1);
    assertEquals(actual.getAddInputCalls(), 3 * 2);
    assertEquals(actual.getAddInputWall(), new Duration(3 * 3, NANOSECONDS));
    assertEquals(actual.getAddInputCpu(), new Duration(3 * 4, NANOSECONDS));
    assertEquals(actual.getPhysicalInputDataSize(), DataSize.ofBytes(3 * 51));
    assertEquals(actual.getPhysicalInputPositions(), 3 * 511);
    assertEquals(actual.getInternalNetworkInputDataSize(), DataSize.ofBytes(3 * 52));
    assertEquals(actual.getInternalNetworkInputPositions(), 3 * 522);
    assertEquals(actual.getRawInputDataSize(), DataSize.ofBytes(3 * 5));
    assertEquals(actual.getInputDataSize(), DataSize.ofBytes(3 * 6));
    assertEquals(actual.getInputPositions(), 3 * 7);
    assertEquals(actual.getSumSquaredInputPositions(), 3 * 8.0);
    assertEquals(actual.getGetOutputCalls(), 3 * 9);
    assertEquals(actual.getGetOutputWall(), new Duration(3 * 10, NANOSECONDS));
    assertEquals(actual.getGetOutputCpu(), new Duration(3 * 11, NANOSECONDS));
    assertEquals(actual.getOutputDataSize(), DataSize.ofBytes(3 * 12));
    assertEquals(actual.getOutputPositions(), 3 * 13);
    assertEquals(actual.getDynamicFilterSplitsProcessed(), 3 * 533);
    assertEquals(actual.getMetrics().getMetrics(), ImmutableMap.of("metrics", new LongCount(3 * 42)));
    assertEquals(actual.getConnectorMetrics().getMetrics(), ImmutableMap.of("connectorMetrics", new LongCount(3 * 43)));
    assertEquals(actual.getPhysicalWrittenDataSize(), DataSize.ofBytes(3 * 14));
    assertEquals(actual.getBlockedWall(), new Duration(3 * 15, NANOSECONDS));
    assertEquals(actual.getFinishCalls(), 3 * 16);
    assertEquals(actual.getFinishWall(), new Duration(3 * 17, NANOSECONDS));
    assertEquals(actual.getFinishCpu(), new Duration(3 * 18, NANOSECONDS));
    assertEquals(actual.getUserMemoryReservation(), DataSize.ofBytes(3 * 19));
    assertEquals(actual.getRevocableMemoryReservation(), DataSize.ofBytes(3 * 20));
    assertEquals(actual.getPeakUserMemoryReservation(), DataSize.ofBytes(22));
    assertEquals(actual.getPeakRevocableMemoryReservation(), DataSize.ofBytes(24));
    assertEquals(actual.getPeakTotalMemoryReservation(), DataSize.ofBytes(25));
    assertEquals(actual.getSpilledDataSize(), DataSize.ofBytes(3 * 26));
    assertNull(actual.getInfo());
}

Example 2 with LongCount

use of io.trino.plugin.base.metrics.LongCount in project trino by trinodb.

the class TestWorkProcessorSourceOperatorAdapter method testMetrics.

@Test
public void testMetrics() {
    DriverContext driverContext = TestingTaskContext.builder(MoreExecutors.directExecutor(), scheduledExecutor, TEST_SESSION).build().addPipelineContext(0, true, true, false).addDriverContext();
    TestWorkProcessorOperatorFactory workProcessorFactory = new TestWorkProcessorOperatorFactory();
    OperatorContext context = driverContext.addOperatorContext(workProcessorFactory.getOperatorId(), workProcessorFactory.getPlanNodeId(), workProcessorFactory.getOperatorType());
    Operator operator = new WorkProcessorSourceOperatorAdapter(context, workProcessorFactory);
    operator.getOutput();
    assertThat(operator.isFinished()).isFalse();
    assertThat(getOnlyElement(context.getNestedOperatorStats()).getMetrics().getMetrics()).hasSize(2).containsEntry("testOperatorMetric", new LongCount(1));
    assertThat(getOnlyElement(context.getNestedOperatorStats()).getConnectorMetrics().getMetrics()).isEqualTo(ImmutableMap.of("testConnectorMetric", new LongCount(2), "Physical input read time", new DurationTiming(new Duration(7, NANOSECONDS))));
    operator.getOutput();
    assertThat(operator.isFinished()).isTrue();
    assertThat(getOnlyElement(context.getNestedOperatorStats()).getMetrics().getMetrics()).hasSize(2).containsEntry("testOperatorMetric", new LongCount(2));
    assertThat(getOnlyElement(context.getNestedOperatorStats()).getConnectorMetrics().getMetrics()).isEqualTo(ImmutableMap.of("testConnectorMetric", new LongCount(3), "Physical input read time", new DurationTiming(new Duration(7, NANOSECONDS))));
}

Example 3 with LongCount

use of io.trino.plugin.base.metrics.LongCount in project trino by trinodb.

the class TestMemoryConnectorTest method testCustomMetricsScanFilter.

@Test
// TODO (https://github.com/trinodb/trino/issues/8691) fix the test
@Flaky(issue = "https://github.com/trinodb/trino/issues/8691", match = "ComparisonFailure: expected:<LongCount\\{total=\\[\\d+]}> but was:<(LongCount\\{total=\\[\\d+]}|null)>")
public void testCustomMetricsScanFilter() {
    Metrics metrics = collectCustomMetrics("SELECT partkey FROM part WHERE partkey % 1000 > 0");
    assertThat(metrics.getMetrics().get("rows")).isEqualTo(new LongCount(PART_COUNT));
    assertThat(metrics.getMetrics().get("started")).isEqualTo(metrics.getMetrics().get("finished"));
    assertThat(((Count<?>) metrics.getMetrics().get("finished")).getTotal()).isGreaterThan(0);
}

Example 4 with LongCount

use of io.trino.plugin.base.metrics.LongCount in project trino by trinodb.

the class TestWorkProcessorOperatorAdapter method testMetrics.

@Test
public void testMetrics() {
    DriverContext driverContext = TestingTaskContext.builder(MoreExecutors.directExecutor(), scheduledExecutor, TEST_SESSION).build().addPipelineContext(0, true, true, false).addDriverContext();
    OperatorFactory factory = createAdapterOperatorFactory(new TestWorkProcessorOperatorFactory());
    Operator operator = factory.createOperator(driverContext);
    OperatorContext context = operator.getOperatorContext();
    operator.getOutput();
    assertThat(operator.isFinished()).isFalse();
    assertThat(getOnlyElement(context.getNestedOperatorStats()).getMetrics().getMetrics()).hasSize(2).containsEntry("testOperatorMetric", new LongCount(1));
    operator.getOutput();
    assertThat(operator.isFinished()).isTrue();
    assertThat(getOnlyElement(context.getNestedOperatorStats()).getMetrics().getMetrics()).hasSize(2).containsEntry("testOperatorMetric", new LongCount(2));
}

Example 5 with LongCount

use of io.trino.plugin.base.metrics.LongCount in project trino by trinodb.

the class TestWorkProcessorPipelineSourceOperator method testWorkProcessorPipelineSourceOperator.

@Test(timeOut = 10_000)
public void testWorkProcessorPipelineSourceOperator() throws InterruptedException {
    Split split = createSplit();
    Page page1 = createPage(1);
    Page page2 = createPage(2);
    Page page3 = createPage(3);
    Page page4 = createPage(4);
    Page page5 = createPage(5);
    Transformation<Split, Page> sourceOperatorPages = transformationFrom(ImmutableList.of(Transform.of(Optional.of(split), TransformationState.ofResult(page1, false)), Transform.of(Optional.of(split), TransformationState.ofResult(page2, true))));
    SettableFuture<Void> firstBlockedFuture = SettableFuture.create();
    Transformation<Page, Page> firstOperatorPages = transformationFrom(ImmutableList.of(Transform.of(Optional.of(page1), TransformationState.blocked(firstBlockedFuture)), Transform.of(Optional.of(page1), TransformationState.ofResult(page3, true)), Transform.of(Optional.of(page2), TransformationState.ofResult(page4, false)), Transform.of(Optional.of(page2), TransformationState.finished())), (left, right) -> left.getPositionCount() == right.getPositionCount());
    SettableFuture<Void> secondBlockedFuture = SettableFuture.create();
    Transformation<Page, Page> secondOperatorPages = transformationFrom(ImmutableList.of(Transform.of(Optional.of(page3), TransformationState.ofResult(page5, true)), Transform.of(Optional.of(page4), TransformationState.needsMoreData()), Transform.of(Optional.empty(), TransformationState.blocked(secondBlockedFuture))), (left, right) -> left.getPositionCount() == right.getPositionCount());
    TestWorkProcessorSourceOperatorFactory sourceOperatorFactory = new TestWorkProcessorSourceOperatorFactory(1, new PlanNodeId("1"), sourceOperatorPages);
    TestWorkProcessorOperatorFactory firstOperatorFactory = new TestWorkProcessorOperatorFactory(2, firstOperatorPages);
    TestWorkProcessorOperatorFactory secondOperatorFactory = new TestWorkProcessorOperatorFactory(3, secondOperatorPages);
    SourceOperatorFactory pipelineOperatorFactory = (SourceOperatorFactory) getOnlyElement(WorkProcessorPipelineSourceOperator.convertOperators(ImmutableList.of(new OperatorFactoryWithTypes(sourceOperatorFactory, ImmutableList.of(BIGINT)), new OperatorFactoryWithTypes(firstOperatorFactory, ImmutableList.of(BIGINT)), new OperatorFactoryWithTypes(secondOperatorFactory, ImmutableList.of(BIGINT))), DataSize.ofBytes(0), 0));
    DriverContext driverContext = TestingTaskContext.builder(MoreExecutors.directExecutor(), scheduledExecutor, TEST_SESSION).build().addPipelineContext(0, true, true, false).addDriverContext();
    SourceOperator pipelineOperator = pipelineOperatorFactory.createOperator(driverContext);
    OperatorContext pipelineOperatorContext = pipelineOperator.getOperatorContext();
    // make sure WorkProcessorOperator memory is accounted for
    sourceOperatorFactory.sourceOperator.memoryTrackingContext.localUserMemoryContext().setBytes(123);
    assertEquals(driverContext.getMemoryUsage(), 123);
    assertNull(pipelineOperator.getOutput());
    assertFalse(pipelineOperator.isBlocked().isDone());
    // blocking on splits should not account for blocked time for any WorkProcessorOperator
    pipelineOperatorContext.getNestedOperatorStats().forEach(operatorStats -> assertEquals(operatorStats.getBlockedWall().toMillis(), 0));
    pipelineOperator.addSplit(split);
    assertTrue(pipelineOperator.isBlocked().isDone());
    assertNull(pipelineOperator.getOutput());
    assertFalse(pipelineOperator.isBlocked().isDone());
    Thread.sleep(100);
    firstBlockedFuture.set(null);
    assertTrue(pipelineOperator.isBlocked().isDone());
    // blocking of first WorkProcessorOperator should be accounted for in stats
    List<OperatorStats> operatorStats = pipelineOperatorContext.getNestedOperatorStats();
    assertEquals(operatorStats.get(0).getBlockedWall().toMillis(), 0);
    assertTrue(operatorStats.get(1).getBlockedWall().toMillis() > 0);
    assertEquals(operatorStats.get(2).getBlockedWall().toMillis(), 0);
    assertEquals(getTestingOperatorInfo(operatorStats.get(1)).count, 2);
    assertEquals(getTestingOperatorInfo(operatorStats.get(2)).count, 2);
    assertEquals(pipelineOperator.getOutput().getPositionCount(), page5.getPositionCount());
    // sourceOperator should yield
    driverContext.getYieldSignal().forceYieldForTesting();
    assertNull(pipelineOperator.getOutput());
    driverContext.getYieldSignal().resetYieldForTesting();
    // firstOperatorPages should finish. This should cause sourceOperator and firstOperatorPages to close.
    // secondOperatorPages should block
    assertNull(pipelineOperator.getOutput());
    assertFalse(pipelineOperator.isBlocked().isDone());
    assertTrue(sourceOperatorFactory.sourceOperator.closed);
    assertTrue(firstOperatorFactory.operator.closed);
    assertFalse(secondOperatorFactory.operator.closed);
    // first operator should return final operator info
    assertEquals(getTestingOperatorInfo(operatorStats.get(1)).count, 3);
    assertEquals(getTestingOperatorInfo(operatorStats.get(2)).count, 2);
    operatorStats = pipelineOperatorContext.getNestedOperatorStats();
    assertEquals(getTestingOperatorInfo(operatorStats.get(1)).count, 3);
    assertEquals(getTestingOperatorInfo(operatorStats.get(2)).count, 3);
    // cause early operator finish
    pipelineOperator.finish();
    // operator is still blocked on secondBlockedFuture
    assertFalse(pipelineOperator.isFinished());
    assertTrue(secondOperatorFactory.operator.closed);
    secondBlockedFuture.set(null);
    assertTrue(pipelineOperator.isBlocked().isDone());
    assertNull(pipelineOperator.getOutput());
    assertTrue(pipelineOperator.isFinished());
    // assert operator stats are correct
    operatorStats = pipelineOperatorContext.getNestedOperatorStats();
    assertEquals(operatorStats.get(0).getOutputPositions(), 3);
    assertEquals(operatorStats.get(1).getInputPositions(), 3);
    assertEquals(operatorStats.get(0).getOutputDataSize().toBytes(), 27);
    assertEquals(operatorStats.get(1).getInputDataSize().toBytes(), 27);
    assertEquals(operatorStats.get(1).getOutputPositions(), 7);
    assertEquals(operatorStats.get(2).getInputPositions(), 7);
    assertEquals(operatorStats.get(1).getOutputDataSize().toBytes(), 63);
    assertEquals(operatorStats.get(2).getInputDataSize().toBytes(), 63);
    assertEquals(operatorStats.get(2).getOutputPositions(), 5);
    assertEquals(operatorStats.get(2).getOutputDataSize().toBytes(), 45);
    assertThat(operatorStats.get(1).getMetrics().getMetrics()).hasSize(2).containsEntry("testOperatorMetric", new LongCount(1));
    // assert source operator stats are correct
    OperatorStats sourceOperatorStats = operatorStats.get(0);
    assertThat(sourceOperatorStats.getMetrics().getMetrics()).hasSize(3).containsEntry("testSourceMetric", new LongCount(1)).containsEntry("testSourceClosed", new LongCount(1));
    assertEquals(sourceOperatorStats.getConnectorMetrics().getMetrics(), ImmutableMap.of("testSourceConnectorMetric", new LongCount(2), "testSourceConnectorClosed", new LongCount(1), "Physical input read time", new DurationTiming(new Duration(7, NANOSECONDS))));
    assertEquals(sourceOperatorStats.getDynamicFilterSplitsProcessed(), 42L);
    assertEquals(sourceOperatorStats.getPhysicalInputDataSize(), DataSize.ofBytes(1));
    assertEquals(sourceOperatorStats.getPhysicalInputPositions(), 2);
    assertEquals(sourceOperatorStats.getInternalNetworkInputDataSize(), DataSize.ofBytes(3));
    assertEquals(sourceOperatorStats.getInternalNetworkInputPositions(), 4);
    assertEquals(sourceOperatorStats.getInputDataSize(), DataSize.ofBytes(5));
    assertEquals(sourceOperatorStats.getInputPositions(), 6);
    assertEquals(sourceOperatorStats.getAddInputWall(), new Duration(0, NANOSECONDS));
    // pipeline input stats should match source WorkProcessorOperator stats
    PipelineStats pipelineStats = pipelineOperator.getOperatorContext().getDriverContext().getPipelineContext().getPipelineStats();
    assertEquals(sourceOperatorStats.getPhysicalInputDataSize(), pipelineStats.getPhysicalInputDataSize());
    assertEquals(sourceOperatorStats.getPhysicalInputPositions(), pipelineStats.getPhysicalInputPositions());
    assertEquals(sourceOperatorStats.getInternalNetworkInputDataSize(), pipelineStats.getInternalNetworkInputDataSize());
    assertEquals(sourceOperatorStats.getInternalNetworkInputPositions(), pipelineStats.getInternalNetworkInputPositions());
    assertEquals(sourceOperatorStats.getInputDataSize(), pipelineStats.getProcessedInputDataSize());
    assertEquals(sourceOperatorStats.getInputPositions(), pipelineStats.getProcessedInputPositions());
    assertEquals(sourceOperatorStats.getAddInputWall(), pipelineStats.getPhysicalInputReadTime());
    // assert pipeline metrics
    List<OperatorStats> operatorSummaries = pipelineStats.getOperatorSummaries();
    assertThat(operatorSummaries.get(0).getMetrics().getMetrics()).hasSize(3).containsEntry("testSourceMetric", new LongCount(1)).containsEntry("testSourceClosed", new LongCount(1));
    assertEquals(operatorSummaries.get(0).getConnectorMetrics().getMetrics(), ImmutableMap.of("testSourceConnectorMetric", new LongCount(2), "testSourceConnectorClosed", new LongCount(1), "Physical input read time", new DurationTiming(new Duration(7, NANOSECONDS))));
    assertThat(operatorSummaries.get(1).getMetrics().getMetrics()).hasSize(2).containsEntry("testOperatorMetric", new LongCount(1));
}