Examples with ColumnStatistics com.facebook.presto.spi.statistics.ColumnStatistics used on opensource projects

Example 1 with ColumnStatistics

use of com.facebook.presto.spi.statistics.ColumnStatistics in project presto by prestodb.

the class TestTpchMetadata method testColumnStats.

private void testColumnStats(String schema, TpchTable<?> table, TpchColumn<?> column, Constraint<ColumnHandle> constraint, ColumnStatistics expected) {
    TpchTableHandle tableHandle = tpchMetadata.getTableHandle(session, new SchemaTableName(schema, table.getTableName()));
    List<ColumnHandle> columnHandles = ImmutableList.copyOf(tpchMetadata.getColumnHandles(session, tableHandle).values());
    TableStatistics tableStatistics = tpchMetadata.getTableStatistics(session, tableHandle, Optional.empty(), columnHandles, constraint);
    ColumnHandle columnHandle = tpchMetadata.getColumnHandles(session, tableHandle).get(column.getSimplifiedColumnName());
    ColumnStatistics actual = tableStatistics.getColumnStatistics().get(columnHandle);
    EstimateAssertion estimateAssertion = new EstimateAssertion(TOLERANCE);
    estimateAssertion.assertClose(actual.getDistinctValuesCount(), expected.getDistinctValuesCount(), "distinctValuesCount");
    estimateAssertion.assertClose(actual.getDataSize(), expected.getDataSize(), "dataSize");
    estimateAssertion.assertClose(actual.getNullsFraction(), expected.getNullsFraction(), "nullsFraction");
    estimateAssertion.assertClose(actual.getRange(), expected.getRange(), "range");
}

Example 2 with ColumnStatistics

use of com.facebook.presto.spi.statistics.ColumnStatistics in project presto by prestodb.

the class MetastoreHiveStatisticsProvider method createZeroStatistics.

private TableStatistics createZeroStatistics(Map<String, ColumnHandle> columns, Map<String, Type> columnTypes) {
    TableStatistics.Builder result = TableStatistics.builder();
    result.setRowCount(Estimate.of(0));
    result.setTotalSize(Estimate.of(0));
    columns.forEach((columnName, columnHandle) -> {
        Type columnType = columnTypes.get(columnName);
        verify(columnType != null, "columnType is missing for column: %s", columnName);
        ColumnStatistics.Builder columnStatistics = ColumnStatistics.builder();
        columnStatistics.setNullsFraction(Estimate.of(0));
        columnStatistics.setDistinctValuesCount(Estimate.of(0));
        if (hasDataSize(columnType)) {
            columnStatistics.setDataSize(Estimate.of(0));
        }
        result.setColumnStatistics(columnHandle, columnStatistics.build());
    });
    return result.build();
}

Example 3 with ColumnStatistics

use of com.facebook.presto.spi.statistics.ColumnStatistics in project presto by prestodb.

the class MetastoreHiveStatisticsProvider method getTableStatistics.

private static TableStatistics getTableStatistics(Map<String, ColumnHandle> columns, Map<String, Type> columnTypes, List<HivePartition> partitions, Map<String, PartitionStatistics> statistics) {
    if (statistics.isEmpty()) {
        return TableStatistics.empty();
    }
    checkArgument(!partitions.isEmpty(), "partitions is empty");
    OptionalDouble optionalAverageRowsPerPartition = calculateAverageRowsPerPartition(statistics.values());
    if (!optionalAverageRowsPerPartition.isPresent()) {
        return TableStatistics.empty();
    }
    double averageRowsPerPartition = optionalAverageRowsPerPartition.getAsDouble();
    verify(averageRowsPerPartition >= 0, "averageRowsPerPartition must be greater than or equal to zero");
    int queriedPartitionsCount = partitions.size();
    double rowCount = averageRowsPerPartition * queriedPartitionsCount;
    TableStatistics.Builder result = TableStatistics.builder();
    result.setRowCount(Estimate.of(rowCount));
    OptionalDouble optionalAverageSizePerPartition = calculateAverageSizePerPartition(statistics.values());
    if (optionalAverageSizePerPartition.isPresent()) {
        double averageSizePerPartition = optionalAverageSizePerPartition.getAsDouble();
        verify(averageSizePerPartition >= 0, "averageSizePerPartition must be greater than or equal to zero: %s", averageSizePerPartition);
        double totalSize = averageSizePerPartition * queriedPartitionsCount;
        result.setTotalSize(Estimate.of(totalSize));
    }
    for (Map.Entry<String, ColumnHandle> column : columns.entrySet()) {
        String columnName = column.getKey();
        HiveColumnHandle columnHandle = (HiveColumnHandle) column.getValue();
        Type columnType = columnTypes.get(columnName);
        ColumnStatistics columnStatistics;
        if (columnHandle.isPartitionKey()) {
            columnStatistics = createPartitionColumnStatistics(columnHandle, columnType, partitions, statistics, averageRowsPerPartition, rowCount);
        } else {
            columnStatistics = createDataColumnStatistics(columnName, columnType, rowCount, statistics.values());
        }
        result.setColumnStatistics(columnHandle, columnStatistics);
    }
    return result.build();
}

Example 4 with ColumnStatistics

use of com.facebook.presto.spi.statistics.ColumnStatistics in project presto by prestodb.

the class TableScanStatsRule method doCalculate.

@Override
protected Optional<PlanNodeStatsEstimate> doCalculate(TableScanNode node, StatsProvider sourceStats, Lookup lookup, Session session, TypeProvider types) {
    // TODO Construct predicate like AddExchanges's LayoutConstraintEvaluator
    Constraint<ColumnHandle> constraint = new Constraint<>(node.getCurrentConstraint());
    TableStatistics tableStatistics = metadata.getTableStatistics(session, node.getTable(), ImmutableList.copyOf(node.getAssignments().values()), constraint);
    Map<VariableReferenceExpression, VariableStatsEstimate> outputVariableStats = new HashMap<>();
    for (Map.Entry<VariableReferenceExpression, ColumnHandle> entry : node.getAssignments().entrySet()) {
        Optional<ColumnStatistics> columnStatistics = Optional.ofNullable(tableStatistics.getColumnStatistics().get(entry.getValue()));
        outputVariableStats.put(entry.getKey(), columnStatistics.map(statistics -> StatsUtil.toVariableStatsEstimate(tableStatistics, statistics)).orElse(VariableStatsEstimate.unknown()));
    }
    return Optional.of(PlanNodeStatsEstimate.builder().setOutputRowCount(tableStatistics.getRowCount().getValue()).setTotalSize(tableStatistics.getTotalSize().getValue()).setConfident(true).addVariableStatistics(outputVariableStats).build());
}

Example 5 with ColumnStatistics

use of com.facebook.presto.spi.statistics.ColumnStatistics in project presto by prestodb.

the class ConnectorFilterStatsCalculatorService method toPlanNodeStats.

private static PlanNodeStatsEstimate toPlanNodeStats(TableStatistics tableStatistics, Map<ColumnHandle, String> columnNames, Map<String, Type> columnTypes) {
    PlanNodeStatsEstimate.Builder builder = PlanNodeStatsEstimate.builder().setOutputRowCount(tableStatistics.getRowCount().getValue());
    for (Map.Entry<ColumnHandle, ColumnStatistics> entry : tableStatistics.getColumnStatistics().entrySet()) {
        String columnName = columnNames.get(entry.getKey());
        VariableReferenceExpression variable = new VariableReferenceExpression(Optional.empty(), columnName, columnTypes.get(columnName));
        builder.addVariableStatistics(variable, toVariableStatsEstimate(tableStatistics, entry.getValue()));
    }
    return builder.build();
}