Search in sources :

Example 1 with HiveColumnStatistics

use of com.facebook.presto.hive.metastore.HiveColumnStatistics in project presto by prestodb.

the class AlluxioHiveMetastore method getPartitionStatistics.

@Override
public Map<String, PartitionStatistics> getPartitionStatistics(MetastoreContext metastoreContext, String databaseName, String tableName, Set<String> partitionNames) {
    Table table = getTable(metastoreContext, databaseName, tableName).orElseThrow(() -> new TableNotFoundException(new SchemaTableName(databaseName, tableName)));
    Map<String, HiveBasicStatistics> partitionBasicStatistics = getPartitionsByNames(metastoreContext, databaseName, tableName, ImmutableList.copyOf(partitionNames)).entrySet().stream().filter(entry -> entry.getValue().isPresent()).collect(toImmutableMap(entry -> MetastoreUtil.makePartName(table.getPartitionColumns(), entry.getValue().get().getValues()), entry -> getHiveBasicStatistics(entry.getValue().get().getParameters())));
    Map<String, OptionalLong> partitionRowCounts = partitionBasicStatistics.entrySet().stream().collect(toImmutableMap(Map.Entry::getKey, entry -> entry.getValue().getRowCount()));
    List<String> dataColumns = table.getDataColumns().stream().map(Column::getName).collect(toImmutableList());
    Map<String, List<ColumnStatisticsInfo>> columnStatisticss;
    try {
        columnStatisticss = client.getPartitionColumnStatistics(table.getDatabaseName(), table.getTableName(), partitionBasicStatistics.keySet().stream().collect(toImmutableList()), dataColumns);
    } catch (AlluxioStatusException e) {
        throw new PrestoException(HIVE_METASTORE_ERROR, e);
    }
    Map<String, Map<String, HiveColumnStatistics>> partitionColumnStatistics = columnStatisticss.entrySet().stream().filter(entry -> !entry.getValue().isEmpty()).collect(toImmutableMap(Map.Entry::getKey, entry -> groupStatisticsByColumn(metastoreContext, entry.getValue(), partitionRowCounts.getOrDefault(entry.getKey(), OptionalLong.empty()))));
    ImmutableMap.Builder<String, PartitionStatistics> result = ImmutableMap.builder();
    for (String partitionName : partitionBasicStatistics.keySet()) {
        HiveBasicStatistics basicStatistics = partitionBasicStatistics.get(partitionName);
        Map<String, HiveColumnStatistics> columnStatistics = partitionColumnStatistics.getOrDefault(partitionName, ImmutableMap.of());
        result.put(partitionName, new PartitionStatistics(basicStatistics, columnStatistics));
    }
    return result.build();
}
Also used : ColumnStatisticsInfo(alluxio.grpc.table.ColumnStatisticsInfo) Table(com.facebook.presto.hive.metastore.Table) NotFoundException(com.facebook.presto.spi.NotFoundException) Column(com.facebook.presto.hive.metastore.Column) Database(com.facebook.presto.hive.metastore.Database) PartitionWithStatistics(com.facebook.presto.hive.metastore.PartitionWithStatistics) HiveMetastore(com.facebook.presto.hive.metastore.thrift.HiveMetastore) PrestoPrincipal(com.facebook.presto.spi.security.PrestoPrincipal) Inject(com.google.inject.Inject) HiveType(com.facebook.presto.hive.HiveType) MetastoreContext(com.facebook.presto.hive.metastore.MetastoreContext) HiveColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics) PrestoException(com.facebook.presto.spi.PrestoException) MetastoreUtil.convertPredicateToParts(com.facebook.presto.hive.metastore.MetastoreUtil.convertPredicateToParts) Function(java.util.function.Function) Partition(com.facebook.presto.hive.metastore.Partition) Duration(io.airlift.units.Duration) OptionalLong(java.util.OptionalLong) MetastoreUtil(com.facebook.presto.hive.metastore.MetastoreUtil) MetastoreUtil.getHiveBasicStatistics(com.facebook.presto.hive.metastore.MetastoreUtil.getHiveBasicStatistics) SchemaTableName(com.facebook.presto.spi.SchemaTableName) HIVE_METASTORE_ERROR(com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR) ImmutableList(com.google.common.collect.ImmutableList) ExtendedHiveMetastore(com.facebook.presto.hive.metastore.ExtendedHiveMetastore) TableMasterClient(alluxio.client.table.TableMasterClient) Map(java.util.Map) Objects.requireNonNull(java.util.Objects.requireNonNull) Constraint(alluxio.grpc.table.Constraint) HiveBasicStatistics(com.facebook.presto.hive.HiveBasicStatistics) AlluxioStatusException(alluxio.exception.status.AlluxioStatusException) Type(com.facebook.presto.common.type.Type) PartitionNameWithVersion(com.facebook.presto.hive.metastore.PartitionNameWithVersion) ImmutableMap(com.google.common.collect.ImmutableMap) PrincipalPrivileges(com.facebook.presto.hive.metastore.PrincipalPrivileges) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) Domain(com.facebook.presto.common.predicate.Domain) PartitionInfo(alluxio.grpc.table.layout.hive.PartitionInfo) List(java.util.List) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) RoleGrant(com.facebook.presto.spi.security.RoleGrant) ColumnStatisticType(com.facebook.presto.spi.statistics.ColumnStatisticType) PartitionStatistics(com.facebook.presto.hive.metastore.PartitionStatistics) Optional(java.util.Optional) HivePrivilegeInfo(com.facebook.presto.hive.metastore.HivePrivilegeInfo) Collections(java.util.Collections) Table(com.facebook.presto.hive.metastore.Table) AlluxioStatusException(alluxio.exception.status.AlluxioStatusException) PrestoException(com.facebook.presto.spi.PrestoException) HiveColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics) MetastoreUtil.getHiveBasicStatistics(com.facebook.presto.hive.metastore.MetastoreUtil.getHiveBasicStatistics) HiveBasicStatistics(com.facebook.presto.hive.HiveBasicStatistics) SchemaTableName(com.facebook.presto.spi.SchemaTableName) ImmutableMap(com.google.common.collect.ImmutableMap) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) PartitionStatistics(com.facebook.presto.hive.metastore.PartitionStatistics) OptionalLong(java.util.OptionalLong) ImmutableList(com.google.common.collect.ImmutableList) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) List(java.util.List) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap)

Example 2 with HiveColumnStatistics

use of com.facebook.presto.hive.metastore.HiveColumnStatistics in project presto by prestodb.

the class ThriftHiveMetastore method getTableStatistics.

@Override
public PartitionStatistics getTableStatistics(MetastoreContext metastoreContext, String databaseName, String tableName) {
    Table table = getTable(metastoreContext, databaseName, tableName).orElseThrow(() -> new TableNotFoundException(new SchemaTableName(databaseName, tableName)));
    List<String> dataColumns = table.getSd().getCols().stream().map(FieldSchema::getName).collect(toImmutableList());
    HiveBasicStatistics basicStatistics = getHiveBasicStatistics(table.getParameters());
    Map<String, HiveColumnStatistics> columnStatistics = getTableColumnStatistics(metastoreContext, databaseName, tableName, dataColumns, basicStatistics.getRowCount());
    return new PartitionStatistics(basicStatistics, columnStatistics);
}
Also used : TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) Table(org.apache.hadoop.hive.metastore.api.Table) ThriftMetastoreUtil.fromMetastoreApiTable(com.facebook.presto.hive.metastore.thrift.ThriftMetastoreUtil.fromMetastoreApiTable) PartitionStatistics(com.facebook.presto.hive.metastore.PartitionStatistics) HiveColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics) MetastoreUtil.getHiveBasicStatistics(com.facebook.presto.hive.metastore.MetastoreUtil.getHiveBasicStatistics) HiveBasicStatistics(com.facebook.presto.hive.HiveBasicStatistics) SchemaTableName(com.facebook.presto.spi.SchemaTableName)

Example 3 with HiveColumnStatistics

use of com.facebook.presto.hive.metastore.HiveColumnStatistics in project presto by prestodb.

the class ThriftHiveMetastore method storePartitionColumnStatistics.

private void storePartitionColumnStatistics(MetastoreContext metastoreContext, String databaseName, String tableName, String partitionName, PartitionWithStatistics partitionWithStatistics) {
    PartitionStatistics statistics = partitionWithStatistics.getStatistics();
    Map<String, HiveColumnStatistics> columnStatistics = statistics.getColumnStatistics();
    if (columnStatistics.isEmpty()) {
        return;
    }
    Map<String, HiveType> columnTypes = partitionWithStatistics.getPartition().getColumns().stream().collect(toImmutableMap(Column::getName, Column::getType));
    setPartitionColumnStatistics(metastoreContext, databaseName, tableName, partitionName, columnTypes, columnStatistics, statistics.getBasicStatistics().getRowCount());
}
Also used : PartitionStatistics(com.facebook.presto.hive.metastore.PartitionStatistics) HiveColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics) HiveType(com.facebook.presto.hive.HiveType)

Example 4 with HiveColumnStatistics

use of com.facebook.presto.hive.metastore.HiveColumnStatistics in project presto by prestodb.

the class TestThriftHiveMetastoreUtil method testEmptyDecimalStatsToColumnStatistics.

@Test
public void testEmptyDecimalStatsToColumnStatistics() {
    DecimalColumnStatsData emptyDecimalColumnStatsData = new DecimalColumnStatsData();
    ColumnStatisticsObj columnStatisticsObj = new ColumnStatisticsObj("my_col", DECIMAL_TYPE_NAME, decimalStats(emptyDecimalColumnStatsData));
    HiveColumnStatistics actual = fromMetastoreApiColumnStatistics(columnStatisticsObj, OptionalLong.empty());
    assertEquals(actual.getIntegerStatistics(), Optional.empty());
    assertEquals(actual.getDoubleStatistics(), Optional.empty());
    assertEquals(actual.getDecimalStatistics(), Optional.of(new DecimalStatistics(Optional.empty(), Optional.empty())));
    assertEquals(actual.getDateStatistics(), Optional.empty());
    assertEquals(actual.getBooleanStatistics(), Optional.empty());
    assertEquals(actual.getMaxValueSizeInBytes(), OptionalLong.empty());
    assertEquals(actual.getTotalSizeInBytes(), OptionalLong.empty());
    assertEquals(actual.getNullsCount(), OptionalLong.empty());
    assertEquals(actual.getDistinctValuesCount(), OptionalLong.empty());
}
Also used : DecimalColumnStatsData(org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData) ColumnStatisticsObj(org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj) DecimalStatistics(com.facebook.presto.hive.metastore.DecimalStatistics) HiveColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics) Test(org.testng.annotations.Test)

Example 5 with HiveColumnStatistics

use of com.facebook.presto.hive.metastore.HiveColumnStatistics in project presto by prestodb.

the class TestThriftHiveMetastoreUtil method testEmptyBinaryStatsToColumnStatistics.

@Test
public void testEmptyBinaryStatsToColumnStatistics() {
    BinaryColumnStatsData emptyBinaryColumnStatsData = new BinaryColumnStatsData();
    ColumnStatisticsObj columnStatisticsObj = new ColumnStatisticsObj("my_col", BINARY_TYPE_NAME, binaryStats(emptyBinaryColumnStatsData));
    HiveColumnStatistics actual = fromMetastoreApiColumnStatistics(columnStatisticsObj, OptionalLong.empty());
    assertEquals(actual.getIntegerStatistics(), Optional.empty());
    assertEquals(actual.getDoubleStatistics(), Optional.empty());
    assertEquals(actual.getDecimalStatistics(), Optional.empty());
    assertEquals(actual.getDateStatistics(), Optional.empty());
    assertEquals(actual.getBooleanStatistics(), Optional.empty());
    assertEquals(actual.getMaxValueSizeInBytes(), OptionalLong.empty());
    assertEquals(actual.getTotalSizeInBytes(), OptionalLong.empty());
    assertEquals(actual.getNullsCount(), OptionalLong.empty());
    assertEquals(actual.getDistinctValuesCount(), OptionalLong.empty());
}
Also used : ColumnStatisticsObj(org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj) HiveColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics) BinaryColumnStatsData(org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData) Test(org.testng.annotations.Test)

Aggregations

HiveColumnStatistics (com.facebook.presto.hive.metastore.HiveColumnStatistics)23 ColumnStatisticsObj (org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj)16 Test (org.testng.annotations.Test)15 PartitionStatistics (com.facebook.presto.hive.metastore.PartitionStatistics)8 HiveBasicStatistics (com.facebook.presto.hive.HiveBasicStatistics)6 SchemaTableName (com.facebook.presto.spi.SchemaTableName)6 MetastoreContext (com.facebook.presto.hive.metastore.MetastoreContext)5 PrestoException (com.facebook.presto.spi.PrestoException)5 ImmutableMap (com.google.common.collect.ImmutableMap)5 Map (java.util.Map)5 Optional (java.util.Optional)5 Type (com.facebook.presto.common.type.Type)4 DateStatistics (com.facebook.presto.hive.metastore.DateStatistics)4 DoubleStatistics (com.facebook.presto.hive.metastore.DoubleStatistics)4 Chars.isCharType (com.facebook.presto.common.type.Chars.isCharType)3 DATE (com.facebook.presto.common.type.DateType.DATE)3 Varchars.isVarcharType (com.facebook.presto.common.type.Varchars.isVarcharType)3 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)3 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)3 String.format (java.lang.String.format)3