Examples with ColumnStatisticType io.prestosql.spi.statistics.ColumnStatisticType used on opensource projects

Example 6 with ColumnStatisticType

use of io.prestosql.spi.statistics.ColumnStatisticType in project boostkit-bigdata by kunpengcompute.

the class HiveMetadata method finishStatisticsCollection.

@Override
public void finishStatisticsCollection(ConnectorSession session, ConnectorTableHandle tableHandle, Collection<ComputedStatistics> computedStatistics) {
    HiveIdentity identity = new HiveIdentity(session);
    HiveTableHandle handle = (HiveTableHandle) tableHandle;
    SchemaTableName tableName = handle.getSchemaTableName();
    Table table = metastore.getTable(identity, tableName.getSchemaName(), tableName.getTableName()).orElseThrow(() -> new TableNotFoundException(handle.getSchemaTableName()));
    List<Column> partitionColumns = table.getPartitionColumns();
    List<String> partitionColumnNames = partitionColumns.stream().map(Column::getName).collect(toImmutableList());
    List<HiveColumnHandle> hiveColumnHandles = hiveColumnHandles(table);
    Map<String, Type> columnTypes = hiveColumnHandles.stream().filter(columnHandle -> !columnHandle.isHidden()).collect(toImmutableMap(HiveColumnHandle::getName, column -> column.getHiveType().getType(typeManager)));
    Map<List<String>, ComputedStatistics> computedStatisticsMap = Statistics.createComputedStatisticsToPartitionMap(computedStatistics, partitionColumnNames, columnTypes);
    if (partitionColumns.isEmpty()) {
        // commit analyze to unpartitioned table
        metastore.setTableStatistics(identity, table, createPartitionStatistics(session, columnTypes, computedStatisticsMap.get(ImmutableList.<String>of())));
    } else {
        List<List<String>> partitionValuesList;
        if (handle.getAnalyzePartitionValues().isPresent()) {
            partitionValuesList = handle.getAnalyzePartitionValues().get();
        } else {
            partitionValuesList = metastore.getPartitionNames(identity, handle.getSchemaName(), handle.getTableName()).orElseThrow(() -> new TableNotFoundException(((HiveTableHandle) tableHandle).getSchemaTableName())).stream().map(HiveUtil::toPartitionValues).collect(toImmutableList());
        }
        ImmutableMap.Builder<List<String>, PartitionStatistics> partitionStatistics = ImmutableMap.builder();
        Map<String, Set<ColumnStatisticType>> columnStatisticTypes = hiveColumnHandles.stream().filter(columnHandle -> !partitionColumnNames.contains(columnHandle.getName())).filter(column -> !column.isHidden()).collect(toImmutableMap(HiveColumnHandle::getName, column -> ImmutableSet.copyOf(metastore.getSupportedColumnStatistics(typeManager.getType(column.getTypeSignature())))));
        Supplier<PartitionStatistics> emptyPartitionStatistics = Suppliers.memoize(() -> Statistics.createEmptyPartitionStatistics(columnTypes, columnStatisticTypes));
        int usedComputedStatistics = 0;
        for (List<String> partitionValues : partitionValuesList) {
            ComputedStatistics collectedStatistics = computedStatisticsMap.get(partitionValues);
            if (collectedStatistics == null) {
                partitionStatistics.put(partitionValues, emptyPartitionStatistics.get());
            } else {
                usedComputedStatistics++;
                partitionStatistics.put(partitionValues, createPartitionStatistics(session, columnTypes, collectedStatistics));
            }
        }
        verify(usedComputedStatistics == computedStatistics.size(), "All computed statistics must be used");
        metastore.setPartitionStatistics(identity, table, partitionStatistics.build());
    }
}

Example 7 with ColumnStatisticType

use of io.prestosql.spi.statistics.ColumnStatisticType in project hetu-core by openlookeng.

the class TestStatisticAggregationsDescriptor method createTestDescriptor.

private static StatisticAggregationsDescriptor<Symbol> createTestDescriptor() {
    StatisticAggregationsDescriptor.Builder<Symbol> builder = StatisticAggregationsDescriptor.builder();
    PlanSymbolAllocator planSymbolAllocator = new PlanSymbolAllocator();
    for (String column : COLUMNS) {
        for (ColumnStatisticType type : ColumnStatisticType.values()) {
            builder.addColumnStatistic(new ColumnStatisticMetadata(column, type), testSymbol(planSymbolAllocator));
        }
        builder.addGrouping(column, testSymbol(planSymbolAllocator));
    }
    builder.addTableStatistic(ROW_COUNT, testSymbol(planSymbolAllocator));
    return builder.build();
}

Example 8 with ColumnStatisticType

use of io.prestosql.spi.statistics.ColumnStatisticType in project hetu-core by openlookeng.

the class StatisticsUtils method fromComputedStatistics.

public static TableStatisticsData fromComputedStatistics(ConnectorSession session, Collection<ComputedStatistics> computedStatistics, long rowCount, Map<String, ColumnHandle> columnHandles, TypeManager typeManager) {
    TableStatisticsData.Builder tableStatBuilder = TableStatisticsData.builder();
    tableStatBuilder.setRowCount(rowCount);
    // organize the column stats into a per-column view
    Map<String, Map<ColumnStatisticType, Block>> perColumnStats = new HashMap<>();
    for (ComputedStatistics stat : computedStatistics) {
        for (Map.Entry<ColumnStatisticMetadata, Block> entry : stat.getColumnStatistics().entrySet()) {
            perColumnStats.putIfAbsent(entry.getKey().getColumnName(), new HashMap<>());
            perColumnStats.get(entry.getKey().getColumnName()).put(entry.getKey().getStatisticType(), entry.getValue());
        }
    }
    // build the per-column statistics
    for (Map.Entry<String, ColumnHandle> entry : columnHandles.entrySet()) {
        Map<ColumnStatisticType, Block> columnStat = perColumnStats.get(entry.getKey());
        if (columnStat == null) {
            continue;
        }
        MemoryColumnHandle handle = (MemoryColumnHandle) entry.getValue();
        Type columnType = handle.getType(typeManager);
        tableStatBuilder.setColumnStatistics(handle.getColumnName(), fromComputedStatistics(session, columnStat, rowCount, columnType));
    }
    return tableStatBuilder.build();
}