Examples with PartitionStatistics com.facebook.presto.hive.metastore.PartitionStatistics used on opensource projects

Example 26 with PartitionStatistics

use of com.facebook.presto.hive.metastore.PartitionStatistics in project presto by prestodb.

the class AlluxioHiveMetastore method getTableStatistics.

@Override
public PartitionStatistics getTableStatistics(MetastoreContext metastoreContext, String databaseName, String tableName) {
    try {
        Table table = getTable(metastoreContext, databaseName, tableName).orElseThrow(() -> new PrestoException(HIVE_METASTORE_ERROR, String.format("Could not retrieve table %s.%s", databaseName, tableName)));
        HiveBasicStatistics basicStatistics = getHiveBasicStatistics(table.getParameters());
        List<Column> columns = table.getPartitionColumns();
        List<String> columnNames = columns.stream().map(Column::getName).collect(toImmutableList());
        List<ColumnStatisticsInfo> columnStatistics = client.getTableColumnStatistics(table.getDatabaseName(), table.getTableName(), columnNames);
        return new PartitionStatistics(basicStatistics, groupStatisticsByColumn(metastoreContext, columnStatistics, basicStatistics.getRowCount()));
    } catch (Exception e) {
        throw new PrestoException(HIVE_METASTORE_ERROR, e);
    }
}

Example 27 with PartitionStatistics

use of com.facebook.presto.hive.metastore.PartitionStatistics in project presto by prestodb.

the class FileHiveMetastore method updateTableStatistics.

@Override
public synchronized void updateTableStatistics(MetastoreContext metastoreContext, String databaseName, String tableName, Function<PartitionStatistics, PartitionStatistics> update) {
    PartitionStatistics originalStatistics = getTableStatistics(metastoreContext, databaseName, tableName);
    PartitionStatistics updatedStatistics = update.apply(originalStatistics);
    Path tableMetadataDirectory = getTableMetadataDirectory(databaseName, tableName);
    TableMetadata tableMetadata = readSchemaFile("table", tableMetadataDirectory, tableCodec).orElseThrow(() -> new TableNotFoundException(new SchemaTableName(databaseName, tableName)));
    TableMetadata updatedMetadata = tableMetadata.withParameters(updateStatisticsParameters(tableMetadata.getParameters(), updatedStatistics.getBasicStatistics())).withColumnStatistics(updatedStatistics.getColumnStatistics());
    writeSchemaFile("table", tableMetadataDirectory, tableCodec, updatedMetadata, true);
}

Example 28 with PartitionStatistics

use of com.facebook.presto.hive.metastore.PartitionStatistics in project presto by prestodb.

the class FileHiveMetastore method getTableStatistics.

@Override
public synchronized PartitionStatistics getTableStatistics(MetastoreContext metastoreContext, String databaseName, String tableName) {
    Path tableMetadataDirectory = getTableMetadataDirectory(databaseName, tableName);
    TableMetadata tableMetadata = readSchemaFile("table", tableMetadataDirectory, tableCodec).orElseThrow(() -> new TableNotFoundException(new SchemaTableName(databaseName, tableName)));
    HiveBasicStatistics basicStatistics = getHiveBasicStatistics(tableMetadata.getParameters());
    Map<String, HiveColumnStatistics> columnStatistics = tableMetadata.getColumnStatistics();
    return new PartitionStatistics(basicStatistics, columnStatistics);
}

Example 29 with PartitionStatistics

use of com.facebook.presto.hive.metastore.PartitionStatistics in project presto by prestodb.

the class HiveMetadata method finishStatisticsCollection.

@Override
public void finishStatisticsCollection(ConnectorSession session, ConnectorTableHandle tableHandle, Collection<ComputedStatistics> computedStatistics) {
    HiveTableHandle handle = (HiveTableHandle) tableHandle;
    SchemaTableName tableName = handle.getSchemaTableName();
    MetastoreContext metastoreContext = getMetastoreContext(session);
    Table table = metastore.getTable(metastoreContext, tableName.getSchemaName(), tableName.getTableName()).orElseThrow(() -> new TableNotFoundException(handle.getSchemaTableName()));
    List<Column> partitionColumns = table.getPartitionColumns();
    List<String> partitionColumnNames = partitionColumns.stream().map(Column::getName).collect(toImmutableList());
    List<HiveColumnHandle> hiveColumnHandles = hiveColumnHandles(table);
    Map<String, Type> columnTypes = hiveColumnHandles.stream().filter(columnHandle -> !columnHandle.isHidden()).collect(toImmutableMap(HiveColumnHandle::getName, column -> column.getHiveType().getType(typeManager)));
    Map<List<String>, ComputedStatistics> computedStatisticsMap = createComputedStatisticsToPartitionMap(computedStatistics, partitionColumnNames, columnTypes);
    if (partitionColumns.isEmpty()) {
        // commit analyze to unpartitioned table
        metastore.setTableStatistics(metastoreContext, table, createPartitionStatistics(session, columnTypes, computedStatisticsMap.get(ImmutableList.<String>of())));
    } else {
        List<List<String>> partitionValuesList;
        if (handle.getAnalyzePartitionValues().isPresent()) {
            partitionValuesList = handle.getAnalyzePartitionValues().get();
        } else {
            partitionValuesList = metastore.getPartitionNames(metastoreContext, handle.getSchemaName(), handle.getTableName()).orElseThrow(() -> new TableNotFoundException(((HiveTableHandle) tableHandle).getSchemaTableName())).stream().map(MetastoreUtil::toPartitionValues).collect(toImmutableList());
        }
        ImmutableMap.Builder<List<String>, PartitionStatistics> partitionStatistics = ImmutableMap.builder();
        Map<String, Set<ColumnStatisticType>> columnStatisticTypes = hiveColumnHandles.stream().filter(columnHandle -> !partitionColumnNames.contains(columnHandle.getName())).filter(column -> !column.isHidden()).collect(toImmutableMap(HiveColumnHandle::getName, column -> ImmutableSet.copyOf(metastore.getSupportedColumnStatistics(metastoreContext, typeManager.getType(column.getTypeSignature())))));
        Supplier<PartitionStatistics> emptyPartitionStatistics = Suppliers.memoize(() -> createEmptyPartitionStatistics(columnTypes, columnStatisticTypes));
        int usedComputedStatistics = 0;
        for (List<String> partitionValues : partitionValuesList) {
            ComputedStatistics collectedStatistics = computedStatisticsMap.get(partitionValues);
            if (collectedStatistics == null) {
                partitionStatistics.put(partitionValues, emptyPartitionStatistics.get());
            } else {
                usedComputedStatistics++;
                partitionStatistics.put(partitionValues, createPartitionStatistics(session, columnTypes, collectedStatistics));
            }
        }
        verify(usedComputedStatistics == computedStatistics.size(), "All computed statistics must be used");
        metastore.setPartitionStatistics(metastoreContext, table, partitionStatistics.build());
    }
}

Example 30 with PartitionStatistics

use of com.facebook.presto.hive.metastore.PartitionStatistics in project presto by prestodb.

the class HiveMetadata method createTable.

@Override
public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, boolean ignoreExisting) {
    PrestoTableType tableType = isExternalTable(tableMetadata.getProperties()) ? EXTERNAL_TABLE : MANAGED_TABLE;
    Table table = prepareTable(session, tableMetadata, tableType);
    PrincipalPrivileges principalPrivileges = buildInitialPrivilegeSet(table.getOwner());
    HiveBasicStatistics basicStatistics = table.getPartitionColumns().isEmpty() ? createZeroStatistics() : createEmptyStatistics();
    metastore.createTable(session, table, principalPrivileges, Optional.empty(), ignoreExisting, new PartitionStatistics(basicStatistics, ImmutableMap.of()));
}