Examples with ColumnStatisticMetadata com.facebook.presto.spi.statistics.ColumnStatisticMetadata used on opensource projects

Example 1 with ColumnStatisticMetadata

use of com.facebook.presto.spi.statistics.ColumnStatisticMetadata in project presto by prestodb.

the class TestStatisticAggregationsDescriptor method testColumnStatisticMetadataKeySerializationRoundTrip.

@Test
public void testColumnStatisticMetadataKeySerializationRoundTrip() {
    for (String column : COLUMNS) {
        for (ColumnStatisticType type : ColumnStatisticType.values()) {
            ColumnStatisticMetadata expected = new ColumnStatisticMetadata(column, type);
            assertEquals(deserialize(serialize(expected)), expected);
        }
    }
}

Example 2 with ColumnStatisticMetadata

use of com.facebook.presto.spi.statistics.ColumnStatisticMetadata in project presto by prestodb.

the class TestTableFinishOperator method testTableWriteCommit.

@Test
public void testTableWriteCommit() throws Exception {
    TestingTableFinisher tableFinisher = new TestingTableFinisher();
    TestingPageSinkCommitter pageSinkCommitter = new TestingPageSinkCommitter();
    ColumnStatisticMetadata statisticMetadata = new ColumnStatisticMetadata("column", MAX_VALUE);
    StatisticAggregationsDescriptor<Integer> descriptor = new StatisticAggregationsDescriptor<>(ImmutableMap.of(), ImmutableMap.of(), ImmutableMap.of(statisticMetadata, 0));
    Session session = testSessionBuilder().setSystemProperty("statistics_cpu_timer_enabled", "true").build();
    TableFinishOperatorFactory operatorFactory = new TableFinishOperatorFactory(0, new PlanNodeId("node"), tableFinisher, pageSinkCommitter, new AggregationOperator.AggregationOperatorFactory(1, new PlanNodeId("test"), AggregationNode.Step.SINGLE, ImmutableList.of(LONG_MAX.bind(ImmutableList.of(STATS_START_CHANNEL), Optional.empty())), true), descriptor, session, TABLE_COMMIT_CONTEXT_CODEC, false);
    DriverContext driverContext = createTaskContext(scheduledExecutor, scheduledExecutor, session).addPipelineContext(0, true, true, false).addDriverContext();
    TableFinishOperator operator = (TableFinishOperator) operatorFactory.createOperator(driverContext);
    List<Type> inputTypes = ImmutableList.of(BIGINT, VARBINARY, VARBINARY, BIGINT);
    // pages for non-grouped execution
    // expect lifespan committer not to be called and stats
    operator.addInput(rowPagesBuilder(inputTypes).row(null, null, getTableCommitContextBytes(Lifespan.taskWide(), 0, 0, NO_COMMIT, false), 1).build().get(0));
    operator.addInput(rowPagesBuilder(inputTypes).row(3, new byte[] { 2 }, getTableCommitContextBytes(Lifespan.taskWide(), 0, 0, NO_COMMIT, true), null).build().get(0));
    assertTrue(pageSinkCommitter.getCommittedFragments().isEmpty());
    // pages for unrecoverable grouped execution
    // expect lifespan committer not to be called
    operator.addInput(rowPagesBuilder(inputTypes).row(null, null, getTableCommitContextBytes(Lifespan.driverGroup(1), 1, 1, NO_COMMIT, false), 4).build().get(0));
    operator.addInput(rowPagesBuilder(inputTypes).row(6, new byte[] { 5 }, getTableCommitContextBytes(Lifespan.driverGroup(1), 1, 1, NO_COMMIT, true), null).build().get(0));
    assertTrue(pageSinkCommitter.getCommittedFragments().isEmpty());
    // pages for failed recoverable grouped execution
    // expect lifespan committer not to be called and page ignored
    operator.addInput(rowPagesBuilder(inputTypes).row(null, null, getTableCommitContextBytes(Lifespan.driverGroup(2), 2, 2, LIFESPAN_COMMIT, false), 100).build().get(0));
    assertTrue(pageSinkCommitter.getCommittedFragments().isEmpty());
    // pages for successful recoverable grouped execution
    // expect lifespan committer to be called and pages published
    operator.addInput(rowPagesBuilder(inputTypes).row(null, null, getTableCommitContextBytes(Lifespan.driverGroup(2), 2, 3, LIFESPAN_COMMIT, false), 9).build().get(0));
    operator.addInput(rowPagesBuilder(inputTypes).row(11, new byte[] { 10 }, getTableCommitContextBytes(Lifespan.driverGroup(2), 2, 3, LIFESPAN_COMMIT, true), null).build().get(0));
    assertEquals(getOnlyElement(pageSinkCommitter.getCommittedFragments()), ImmutableList.of(Slices.wrappedBuffer(new byte[] { 10 })));
    assertThat(driverContext.getSystemMemoryUsage()).isGreaterThan(0);
    assertEquals(driverContext.getMemoryUsage(), 0);
    assertTrue(operator.isBlocked().isDone());
    assertTrue(operator.needsInput());
    operator.finish();
    assertFalse(operator.isFinished());
    assertNull(operator.getOutput());
    List<Type> outputTypes = ImmutableList.of(BIGINT);
    assertPageEquals(outputTypes, operator.getOutput(), rowPagesBuilder(outputTypes).row(20).build().get(0));
    assertTrue(operator.isBlocked().isDone());
    assertFalse(operator.needsInput());
    assertTrue(operator.isFinished());
    operator.close();
    assertEquals(tableFinisher.getFragments(), ImmutableList.of(Slices.wrappedBuffer(new byte[] { 2 }), Slices.wrappedBuffer(new byte[] { 5 }), Slices.wrappedBuffer(new byte[] { 10 })));
    assertEquals(tableFinisher.getComputedStatistics().size(), 1);
    assertEquals(getOnlyElement(tableFinisher.getComputedStatistics()).getColumnStatistics().size(), 1);
    Block expectedStatisticsBlock = new LongArrayBlockBuilder(null, 1).writeLong(9).closeEntry().build();
    assertBlockEquals(BIGINT, getOnlyElement(tableFinisher.getComputedStatistics()).getColumnStatistics().get(statisticMetadata), expectedStatisticsBlock);
    TableFinishInfo tableFinishInfo = operator.getInfo();
    assertThat(tableFinishInfo.getStatisticsWallTime().getValue(NANOSECONDS)).isGreaterThan(0);
    assertThat(tableFinishInfo.getStatisticsCpuTime().getValue(NANOSECONDS)).isGreaterThan(0);
    assertEquals(driverContext.getSystemMemoryUsage(), 0);
    assertEquals(driverContext.getMemoryUsage(), 0);
}

Example 3 with ColumnStatisticMetadata

use of com.facebook.presto.spi.statistics.ColumnStatisticMetadata in project presto by prestodb.

the class TestTableFinishOperator method testStatisticsAggregation.

@Test
public void testStatisticsAggregation() throws Exception {
    TestingTableFinisher tableFinisher = new TestingTableFinisher();
    TestingPageSinkCommitter pageSinkCommitter = new TestingPageSinkCommitter();
    ColumnStatisticMetadata statisticMetadata = new ColumnStatisticMetadata("column", MAX_VALUE);
    StatisticAggregationsDescriptor<Integer> descriptor = new StatisticAggregationsDescriptor<>(ImmutableMap.of(), ImmutableMap.of(), ImmutableMap.of(statisticMetadata, 0));
    Session session = testSessionBuilder().setSystemProperty("statistics_cpu_timer_enabled", "true").build();
    TableFinishOperatorFactory operatorFactory = new TableFinishOperatorFactory(0, new PlanNodeId("node"), tableFinisher, pageSinkCommitter, new AggregationOperator.AggregationOperatorFactory(1, new PlanNodeId("test"), AggregationNode.Step.SINGLE, ImmutableList.of(LONG_MAX.bind(ImmutableList.of(STATS_START_CHANNEL), Optional.empty())), true), descriptor, session, TABLE_COMMIT_CONTEXT_CODEC, false);
    DriverContext driverContext = createTaskContext(scheduledExecutor, scheduledExecutor, session).addPipelineContext(0, true, true, false).addDriverContext();
    TableFinishOperator operator = (TableFinishOperator) operatorFactory.createOperator(driverContext);
    List<Type> inputTypes = ImmutableList.of(BIGINT, VARBINARY, VARBINARY, BIGINT);
    byte[] tableCommitContextForStatsPage = getTableCommitContextBytes(Lifespan.taskWide(), 0, 0, NO_COMMIT, false);
    operator.addInput(rowPagesBuilder(inputTypes).row(null, null, tableCommitContextForStatsPage, 6).build().get(0));
    operator.addInput(rowPagesBuilder(inputTypes).row(null, null, tableCommitContextForStatsPage, 7).build().get(0));
    byte[] tableCommitContextForFragmentsPage = getTableCommitContextBytes(Lifespan.taskWide(), 0, 0, NO_COMMIT, true);
    operator.addInput(rowPagesBuilder(inputTypes).row(4, new byte[] { 1 }, tableCommitContextForFragmentsPage, null).build().get(0));
    operator.addInput(rowPagesBuilder(inputTypes).row(5, new byte[] { 2 }, tableCommitContextForFragmentsPage, null).build().get(0));
    assertThat(driverContext.getSystemMemoryUsage()).isGreaterThan(0);
    assertEquals(driverContext.getMemoryUsage(), 0);
    assertTrue(operator.isBlocked().isDone());
    assertTrue(operator.needsInput());
    operator.finish();
    assertFalse(operator.isFinished());
    assertNull(operator.getOutput());
    List<Type> outputTypes = ImmutableList.of(BIGINT);
    assertPageEquals(outputTypes, operator.getOutput(), rowPagesBuilder(outputTypes).row(9).build().get(0));
    assertTrue(operator.isBlocked().isDone());
    assertFalse(operator.needsInput());
    assertTrue(operator.isFinished());
    operator.close();
    assertEquals(tableFinisher.getFragments(), ImmutableList.of(Slices.wrappedBuffer(new byte[] { 1 }), Slices.wrappedBuffer(new byte[] { 2 })));
    assertEquals(tableFinisher.getComputedStatistics().size(), 1);
    assertEquals(getOnlyElement(tableFinisher.getComputedStatistics()).getColumnStatistics().size(), 1);
    Block expectedStatisticsBlock = new LongArrayBlockBuilder(null, 1).writeLong(7).closeEntry().build();
    assertBlockEquals(BIGINT, getOnlyElement(tableFinisher.getComputedStatistics()).getColumnStatistics().get(statisticMetadata), expectedStatisticsBlock);
    TableFinishInfo tableFinishInfo = operator.getInfo();
    assertThat(tableFinishInfo.getStatisticsWallTime().getValue(NANOSECONDS)).isGreaterThan(0);
    assertThat(tableFinishInfo.getStatisticsCpuTime().getValue(NANOSECONDS)).isGreaterThan(0);
    assertTrue(pageSinkCommitter.getCommittedFragments().isEmpty());
    assertEquals(driverContext.getSystemMemoryUsage(), 0);
    assertEquals(driverContext.getMemoryUsage(), 0);
}

Example 4 with ColumnStatisticMetadata

use of com.facebook.presto.spi.statistics.ColumnStatisticMetadata in project presto by prestodb.

the class HiveMetadata method createPartitionStatistics.

private PartitionStatistics createPartitionStatistics(ConnectorSession session, Map<String, Type> columnTypes, ComputedStatistics computedStatistics) {
    Map<ColumnStatisticMetadata, Block> computedColumnStatistics = computedStatistics.getColumnStatistics();
    Block rowCountBlock = Optional.ofNullable(computedStatistics.getTableStatistics().get(ROW_COUNT)).orElseThrow(() -> new VerifyException("rowCount not present"));
    verify(!rowCountBlock.isNull(0), "rowCount must never be null");
    long rowCount = BIGINT.getLong(rowCountBlock, 0);
    HiveBasicStatistics rowCountOnlyBasicStatistics = new HiveBasicStatistics(OptionalLong.empty(), OptionalLong.of(rowCount), OptionalLong.empty(), OptionalLong.empty());
    return createPartitionStatistics(session, rowCountOnlyBasicStatistics, columnTypes, computedColumnStatistics);
}

Example 5 with ColumnStatisticMetadata

use of com.facebook.presto.spi.statistics.ColumnStatisticMetadata in project presto by prestodb.

the class StatisticsAggregationPlanner method createStatisticsAggregation.

public TableStatisticAggregation createStatisticsAggregation(TableStatisticsMetadata statisticsMetadata, Map<String, VariableReferenceExpression> columnToVariableMap, boolean useOriginalExpression) {
    StatisticAggregationsDescriptor.Builder<VariableReferenceExpression> descriptor = StatisticAggregationsDescriptor.builder();
    List<String> groupingColumns = statisticsMetadata.getGroupingColumns();
    List<VariableReferenceExpression> groupingVariables = groupingColumns.stream().map(columnToVariableMap::get).collect(toImmutableList());
    for (int i = 0; i < groupingVariables.size(); i++) {
        descriptor.addGrouping(groupingColumns.get(i), groupingVariables.get(i));
    }
    ImmutableMap.Builder<VariableReferenceExpression, AggregationNode.Aggregation> aggregations = ImmutableMap.builder();
    StandardFunctionResolution functionResolution = new FunctionResolution(metadata.getFunctionAndTypeManager());
    for (TableStatisticType type : statisticsMetadata.getTableStatistics()) {
        if (type != ROW_COUNT) {
            throw new PrestoException(NOT_SUPPORTED, "Table-wide statistic type not supported: " + type);
        }
        AggregationNode.Aggregation aggregation = new AggregationNode.Aggregation(new CallExpression("count", functionResolution.countFunction(), BIGINT, ImmutableList.of()), Optional.empty(), Optional.empty(), false, Optional.empty());
        VariableReferenceExpression variable = variableAllocator.newVariable("rowCount", BIGINT);
        aggregations.put(variable, aggregation);
        descriptor.addTableStatistic(ROW_COUNT, variable);
    }
    for (ColumnStatisticMetadata columnStatisticMetadata : statisticsMetadata.getColumnStatistics()) {
        String columnName = columnStatisticMetadata.getColumnName();
        ColumnStatisticType statisticType = columnStatisticMetadata.getStatisticType();
        VariableReferenceExpression inputVariable = columnToVariableMap.get(columnName);
        verify(inputVariable != null, "inputVariable is null");
        ColumnStatisticsAggregation aggregation = createColumnAggregation(statisticType, inputVariable, useOriginalExpression);
        VariableReferenceExpression variable = variableAllocator.newVariable(statisticType + ":" + columnName, aggregation.getOutputType());
        aggregations.put(variable, aggregation.getAggregation());
        descriptor.addColumnStatistic(columnStatisticMetadata, variable);
    }
    StatisticAggregations aggregation = new StatisticAggregations(aggregations.build(), groupingVariables);
    return new TableStatisticAggregation(aggregation, descriptor.build());
}