Examples with PartitionStatistics com.facebook.presto.hive.metastore.PartitionStatistics used on opensource projects

Example 21 with PartitionStatistics

use of com.facebook.presto.hive.metastore.PartitionStatistics in project presto by prestodb.

the class ThriftHiveMetastore method getPartitionStatistics.

@Override
public Map<String, PartitionStatistics> getPartitionStatistics(MetastoreContext metastoreContext, String databaseName, String tableName, Set<String> partitionNames) {
    Table table = getTable(metastoreContext, databaseName, tableName).orElseThrow(() -> new TableNotFoundException(new SchemaTableName(databaseName, tableName)));
    List<String> dataColumns = table.getSd().getCols().stream().map(FieldSchema::getName).collect(toImmutableList());
    List<String> partitionColumns = table.getPartitionKeys().stream().map(FieldSchema::getName).collect(toImmutableList());
    Map<String, HiveBasicStatistics> partitionBasicStatistics = getPartitionsByNames(metastoreContext, databaseName, tableName, ImmutableList.copyOf(partitionNames)).stream().collect(toImmutableMap(partition -> makePartName(partitionColumns, partition.getValues()), partition -> getHiveBasicStatistics(partition.getParameters())));
    Map<String, OptionalLong> partitionRowCounts = partitionBasicStatistics.entrySet().stream().collect(toImmutableMap(Map.Entry::getKey, entry -> entry.getValue().getRowCount()));
    Map<String, Map<String, HiveColumnStatistics>> partitionColumnStatistics = getPartitionColumnStatistics(metastoreContext, databaseName, tableName, partitionNames, dataColumns, partitionRowCounts);
    ImmutableMap.Builder<String, PartitionStatistics> result = ImmutableMap.builder();
    for (String partitionName : partitionNames) {
        HiveBasicStatistics basicStatistics = partitionBasicStatistics.getOrDefault(partitionName, createEmptyStatistics());
        Map<String, HiveColumnStatistics> columnStatistics = partitionColumnStatistics.getOrDefault(partitionName, ImmutableMap.of());
        result.put(partitionName, new PartitionStatistics(basicStatistics, columnStatistics));
    }
    return result.build();
}

Example 22 with PartitionStatistics

use of com.facebook.presto.hive.metastore.PartitionStatistics in project presto by prestodb.

the class ThriftHiveMetastore method updatePartitionStatistics.

@Override
public synchronized void updatePartitionStatistics(MetastoreContext metastoreContext, String databaseName, String tableName, String partitionName, Function<PartitionStatistics, PartitionStatistics> update) {
    PartitionStatistics currentStatistics = requireNonNull(getPartitionStatistics(metastoreContext, databaseName, tableName, ImmutableSet.of(partitionName)).get(partitionName), "getPartitionStatistics() returned null");
    PartitionStatistics updatedStatistics = update.apply(currentStatistics);
    List<Partition> partitions = getPartitionsByNames(metastoreContext, databaseName, tableName, ImmutableList.of(partitionName));
    if (partitions.size() != 1) {
        throw new PrestoException(HIVE_METASTORE_ERROR, "Metastore returned multiple partitions for name: " + partitionName);
    }
    Partition originalPartition = getOnlyElement(partitions);
    Partition modifiedPartition = originalPartition.deepCopy();
    HiveBasicStatistics basicStatistics = updatedStatistics.getBasicStatistics();
    modifiedPartition.setParameters(updateStatisticsParameters(modifiedPartition.getParameters(), basicStatistics));
    alterPartitionWithoutStatistics(metastoreContext, databaseName, tableName, modifiedPartition);
    Map<String, HiveType> columns = modifiedPartition.getSd().getCols().stream().collect(toImmutableMap(FieldSchema::getName, schema -> metastoreContext.getColumnConverter().toColumn(schema).getType()));
    setPartitionColumnStatistics(metastoreContext, databaseName, tableName, partitionName, columns, updatedStatistics.getColumnStatistics(), basicStatistics.getRowCount());
    Set<String> removedStatistics = difference(currentStatistics.getColumnStatistics().keySet(), updatedStatistics.getColumnStatistics().keySet());
    removedStatistics.forEach(column -> deletePartitionColumnStatistics(metastoreContext, databaseName, tableName, partitionName, column));
}

Example 23 with PartitionStatistics

use of com.facebook.presto.hive.metastore.PartitionStatistics in project presto by prestodb.

the class GlueHiveMetastore method updateTableStatistics.

@Override
public void updateTableStatistics(MetastoreContext metastoreContext, String databaseName, String tableName, Function<PartitionStatistics, PartitionStatistics> update) {
    PartitionStatistics currentStatistics = getTableStatistics(metastoreContext, databaseName, tableName);
    PartitionStatistics updatedStatistics = update.apply(currentStatistics);
    if (!updatedStatistics.getColumnStatistics().isEmpty()) {
        throw new PrestoException(NOT_SUPPORTED, "Glue metastore does not support column level statistics");
    }
    Table table = getTableOrElseThrow(metastoreContext, databaseName, tableName);
    try {
        TableInput tableInput = GlueInputConverter.convertTable(table);
        tableInput.setParameters(updateStatisticsParameters(table.getParameters(), updatedStatistics.getBasicStatistics()));
        UpdateTableRequest request = new UpdateTableRequest().withCatalogId(catalogId).withDatabaseName(databaseName).withTableInput(tableInput);
        stats.getUpdateTable().record(() -> glueClient.updateTable(request));
    } catch (EntityNotFoundException e) {
        throw new TableNotFoundException(new SchemaTableName(databaseName, tableName));
    } catch (AmazonServiceException e) {
        throw new PrestoException(HIVE_METASTORE_ERROR, e);
    }
}

Example 24 with PartitionStatistics

use of com.facebook.presto.hive.metastore.PartitionStatistics in project presto by prestodb.

the class GlueHiveMetastore method getPartitionStatistics.

@Override
public Map<String, PartitionStatistics> getPartitionStatistics(MetastoreContext metastoreContext, String databaseName, String tableName, Set<String> partitionNames) {
    ImmutableMap.Builder<String, PartitionStatistics> result = ImmutableMap.builder();
    getPartitionsByNames(metastoreContext, databaseName, tableName, ImmutableList.copyOf(partitionNames)).forEach((partitionName, optionalPartition) -> {
        Partition partition = optionalPartition.orElseThrow(() -> new PartitionNotFoundException(new SchemaTableName(databaseName, tableName), toPartitionValues(partitionName)));
        PartitionStatistics partitionStatistics = new PartitionStatistics(getHiveBasicStatistics(partition.getParameters()), ImmutableMap.of());
        result.put(partitionName, partitionStatistics);
    });
    return result.build();
}

Example 25 with PartitionStatistics

use of com.facebook.presto.hive.metastore.PartitionStatistics in project presto by prestodb.

the class InMemoryHiveMetastore method getTableStatistics.

@Override
public synchronized PartitionStatistics getTableStatistics(MetastoreContext metastoreContext, String databaseName, String tableName) {
    SchemaTableName schemaTableName = new SchemaTableName(databaseName, tableName);
    PartitionStatistics statistics = columnStatistics.get(schemaTableName);
    if (statistics == null) {
        statistics = new PartitionStatistics(createEmptyStatistics(), ImmutableMap.of());
    }
    return statistics;
}