Search in sources :

Example 1 with IntegerStatistics

use of io.trino.plugin.hive.metastore.IntegerStatistics in project trino by trinodb.

the class TestStatistics method testMergeIntegerColumnStatistics.

@Test
public void testMergeIntegerColumnStatistics() {
    assertMergeHiveColumnStatistics(HiveColumnStatistics.builder().setIntegerStatistics(new IntegerStatistics(OptionalLong.empty(), OptionalLong.empty())).build(), HiveColumnStatistics.builder().setIntegerStatistics(new IntegerStatistics(OptionalLong.empty(), OptionalLong.empty())).build(), HiveColumnStatistics.builder().setIntegerStatistics(new IntegerStatistics(OptionalLong.empty(), OptionalLong.empty())).build());
    assertMergeHiveColumnStatistics(HiveColumnStatistics.builder().setIntegerStatistics(new IntegerStatistics(OptionalLong.of(1), OptionalLong.of(2))).build(), HiveColumnStatistics.builder().setIntegerStatistics(new IntegerStatistics(OptionalLong.empty(), OptionalLong.empty())).build(), HiveColumnStatistics.builder().setIntegerStatistics(new IntegerStatistics(OptionalLong.of(1), OptionalLong.of(2))).build());
    assertMergeHiveColumnStatistics(HiveColumnStatistics.builder().setIntegerStatistics(new IntegerStatistics(OptionalLong.of(1), OptionalLong.of(2))).build(), HiveColumnStatistics.builder().setIntegerStatistics(new IntegerStatistics(OptionalLong.of(0), OptionalLong.of(3))).build(), HiveColumnStatistics.builder().setIntegerStatistics(new IntegerStatistics(OptionalLong.of(0), OptionalLong.of(3))).build());
}
Also used : IntegerStatistics(io.trino.plugin.hive.metastore.IntegerStatistics) Test(org.testng.annotations.Test)

Example 2 with IntegerStatistics

use of io.trino.plugin.hive.metastore.IntegerStatistics in project trino by trinodb.

the class TestThriftMetastoreUtil method testLongStatsToColumnStatistics.

@Test
public void testLongStatsToColumnStatistics() {
    LongColumnStatsData longColumnStatsData = new LongColumnStatsData();
    longColumnStatsData.setLowValue(0);
    longColumnStatsData.setHighValue(100);
    longColumnStatsData.setNumNulls(1);
    longColumnStatsData.setNumDVs(20);
    ColumnStatisticsObj columnStatisticsObj = new ColumnStatisticsObj("my_col", BIGINT_TYPE_NAME, longStats(longColumnStatsData));
    HiveColumnStatistics actual = fromMetastoreApiColumnStatistics(columnStatisticsObj, OptionalLong.of(1000));
    assertEquals(actual.getIntegerStatistics(), Optional.of(new IntegerStatistics(OptionalLong.of(0), OptionalLong.of(100))));
    assertEquals(actual.getDoubleStatistics(), Optional.empty());
    assertEquals(actual.getDecimalStatistics(), Optional.empty());
    assertEquals(actual.getDateStatistics(), Optional.empty());
    assertEquals(actual.getBooleanStatistics(), Optional.empty());
    assertEquals(actual.getMaxValueSizeInBytes(), OptionalLong.empty());
    assertEquals(actual.getTotalSizeInBytes(), OptionalLong.empty());
    assertEquals(actual.getNullsCount(), OptionalLong.of(1));
    assertEquals(actual.getDistinctValuesCount(), OptionalLong.of(19));
}
Also used : ColumnStatisticsObj(org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj) HiveColumnStatistics(io.trino.plugin.hive.metastore.HiveColumnStatistics) LongColumnStatsData(org.apache.hadoop.hive.metastore.api.LongColumnStatsData) IntegerStatistics(io.trino.plugin.hive.metastore.IntegerStatistics) Test(org.testng.annotations.Test)

Example 3 with IntegerStatistics

use of io.trino.plugin.hive.metastore.IntegerStatistics in project trino by trinodb.

the class TestStatistics method testFromComputedStatistics.

@Test
public void testFromComputedStatistics() {
    Function<Integer, Block> singleIntegerValueBlock = value -> BigintType.BIGINT.createBlockBuilder(null, 1).writeLong(value).build();
    ComputedStatistics statistics = ComputedStatistics.builder(ImmutableList.of(), ImmutableList.of()).addTableStatistic(TableStatisticType.ROW_COUNT, singleIntegerValueBlock.apply(5)).addColumnStatistic(new ColumnStatisticMetadata("a_column", MIN_VALUE), singleIntegerValueBlock.apply(1)).addColumnStatistic(new ColumnStatisticMetadata("a_column", MAX_VALUE), singleIntegerValueBlock.apply(5)).addColumnStatistic(new ColumnStatisticMetadata("a_column", NUMBER_OF_DISTINCT_VALUES), singleIntegerValueBlock.apply(5)).addColumnStatistic(new ColumnStatisticMetadata("a_column", NUMBER_OF_NON_NULL_VALUES), singleIntegerValueBlock.apply(5)).addColumnStatistic(new ColumnStatisticMetadata("b_column", NUMBER_OF_NON_NULL_VALUES), singleIntegerValueBlock.apply(4)).build();
    Map<String, Type> columnTypes = ImmutableMap.of("a_column", INTEGER, "b_column", VARCHAR);
    Map<String, HiveColumnStatistics> columnStatistics = Statistics.fromComputedStatistics(statistics.getColumnStatistics(), columnTypes, 5);
    assertThat(columnStatistics).hasSize(2);
    assertThat(columnStatistics.keySet()).contains("a_column", "b_column");
    assertThat(columnStatistics.get("a_column")).isEqualTo(HiveColumnStatistics.builder().setIntegerStatistics(new IntegerStatistics(OptionalLong.of(1), OptionalLong.of(5))).setNullsCount(0).setDistinctValuesCount(5).build());
    assertThat(columnStatistics.get("b_column")).isEqualTo(HiveColumnStatistics.builder().setNullsCount(1).build());
}
Also used : DateStatistics(io.trino.plugin.hive.metastore.DateStatistics) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) Type(io.trino.spi.type.Type) OptionalDouble(java.util.OptionalDouble) HiveColumnStatistics.createBinaryColumnStatistics(io.trino.plugin.hive.metastore.HiveColumnStatistics.createBinaryColumnStatistics) Test(org.testng.annotations.Test) HiveBasicStatistics.createEmptyStatistics(io.trino.plugin.hive.HiveBasicStatistics.createEmptyStatistics) HiveBasicStatistics.createZeroStatistics(io.trino.plugin.hive.HiveBasicStatistics.createZeroStatistics) NUMBER_OF_DISTINCT_VALUES(io.trino.spi.statistics.ColumnStatisticType.NUMBER_OF_DISTINCT_VALUES) ColumnStatisticMetadata(io.trino.spi.statistics.ColumnStatisticMetadata) Function(java.util.function.Function) Float.floatToIntBits(java.lang.Float.floatToIntBits) ADD(io.trino.plugin.hive.util.Statistics.ReduceOperator.ADD) Utils.nativeValueToBlock(io.trino.spi.predicate.Utils.nativeValueToBlock) ColumnStatisticType(io.trino.spi.statistics.ColumnStatisticType) BigintType(io.trino.spi.type.BigintType) BigDecimal(java.math.BigDecimal) OptionalLong(java.util.OptionalLong) Statistics.merge(io.trino.plugin.hive.util.Statistics.merge) HiveColumnStatistics(io.trino.plugin.hive.metastore.HiveColumnStatistics) VARCHAR(io.trino.spi.type.VarcharType.VARCHAR) ImmutableList(com.google.common.collect.ImmutableList) Block(io.trino.spi.block.Block) Map(java.util.Map) Statistics.createHiveColumnStatistics(io.trino.plugin.hive.util.Statistics.createHiveColumnStatistics) INTEGER(io.trino.spi.type.IntegerType.INTEGER) BooleanStatistics(io.trino.plugin.hive.metastore.BooleanStatistics) Statistics.reduce(io.trino.plugin.hive.util.Statistics.reduce) NUMBER_OF_NON_NULL_VALUES(io.trino.spi.statistics.ColumnStatisticType.NUMBER_OF_NON_NULL_VALUES) IntegerStatistics(io.trino.plugin.hive.metastore.IntegerStatistics) HiveBasicStatistics(io.trino.plugin.hive.HiveBasicStatistics) ImmutableMap(com.google.common.collect.ImmutableMap) HiveColumnStatistics.createIntegerColumnStatistics(io.trino.plugin.hive.metastore.HiveColumnStatistics.createIntegerColumnStatistics) ComputedStatistics(io.trino.spi.statistics.ComputedStatistics) DecimalStatistics(io.trino.plugin.hive.metastore.DecimalStatistics) DoubleStatistics(io.trino.plugin.hive.metastore.DoubleStatistics) DOUBLE(io.trino.spi.type.DoubleType.DOUBLE) HiveColumnStatistics.createBooleanColumnStatistics(io.trino.plugin.hive.metastore.HiveColumnStatistics.createBooleanColumnStatistics) MAX_VALUE(io.trino.spi.statistics.ColumnStatisticType.MAX_VALUE) LocalDate(java.time.LocalDate) SUBTRACT(io.trino.plugin.hive.util.Statistics.ReduceOperator.SUBTRACT) Optional(java.util.Optional) TableStatisticType(io.trino.spi.statistics.TableStatisticType) MIN_VALUE(io.trino.spi.statistics.ColumnStatisticType.MIN_VALUE) REAL(io.trino.spi.type.RealType.REAL) ColumnStatisticMetadata(io.trino.spi.statistics.ColumnStatisticMetadata) Type(io.trino.spi.type.Type) ColumnStatisticType(io.trino.spi.statistics.ColumnStatisticType) BigintType(io.trino.spi.type.BigintType) TableStatisticType(io.trino.spi.statistics.TableStatisticType) ComputedStatistics(io.trino.spi.statistics.ComputedStatistics) Utils.nativeValueToBlock(io.trino.spi.predicate.Utils.nativeValueToBlock) Block(io.trino.spi.block.Block) HiveColumnStatistics(io.trino.plugin.hive.metastore.HiveColumnStatistics) Statistics.createHiveColumnStatistics(io.trino.plugin.hive.util.Statistics.createHiveColumnStatistics) IntegerStatistics(io.trino.plugin.hive.metastore.IntegerStatistics) Test(org.testng.annotations.Test)

Example 4 with IntegerStatistics

use of io.trino.plugin.hive.metastore.IntegerStatistics in project trino by trinodb.

the class TestThriftMetastoreUtil method testEmptyLongStatsToColumnStatistics.

@Test
public void testEmptyLongStatsToColumnStatistics() {
    LongColumnStatsData emptyLongColumnStatsData = new LongColumnStatsData();
    ColumnStatisticsObj columnStatisticsObj = new ColumnStatisticsObj("my_col", BIGINT_TYPE_NAME, longStats(emptyLongColumnStatsData));
    HiveColumnStatistics actual = fromMetastoreApiColumnStatistics(columnStatisticsObj, OptionalLong.empty());
    assertEquals(actual.getIntegerStatistics(), Optional.of(new IntegerStatistics(OptionalLong.empty(), OptionalLong.empty())));
    assertEquals(actual.getDoubleStatistics(), Optional.empty());
    assertEquals(actual.getDecimalStatistics(), Optional.empty());
    assertEquals(actual.getDateStatistics(), Optional.empty());
    assertEquals(actual.getBooleanStatistics(), Optional.empty());
    assertEquals(actual.getMaxValueSizeInBytes(), OptionalLong.empty());
    assertEquals(actual.getTotalSizeInBytes(), OptionalLong.empty());
    assertEquals(actual.getNullsCount(), OptionalLong.empty());
    assertEquals(actual.getDistinctValuesCount(), OptionalLong.empty());
}
Also used : ColumnStatisticsObj(org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj) HiveColumnStatistics(io.trino.plugin.hive.metastore.HiveColumnStatistics) LongColumnStatsData(org.apache.hadoop.hive.metastore.api.LongColumnStatsData) IntegerStatistics(io.trino.plugin.hive.metastore.IntegerStatistics) Test(org.testng.annotations.Test)

Aggregations

IntegerStatistics (io.trino.plugin.hive.metastore.IntegerStatistics)4 Test (org.testng.annotations.Test)4 HiveColumnStatistics (io.trino.plugin.hive.metastore.HiveColumnStatistics)3 ImmutableList (com.google.common.collect.ImmutableList)1 ImmutableMap (com.google.common.collect.ImmutableMap)1 HiveBasicStatistics (io.trino.plugin.hive.HiveBasicStatistics)1 HiveBasicStatistics.createEmptyStatistics (io.trino.plugin.hive.HiveBasicStatistics.createEmptyStatistics)1 HiveBasicStatistics.createZeroStatistics (io.trino.plugin.hive.HiveBasicStatistics.createZeroStatistics)1 BooleanStatistics (io.trino.plugin.hive.metastore.BooleanStatistics)1 DateStatistics (io.trino.plugin.hive.metastore.DateStatistics)1 DecimalStatistics (io.trino.plugin.hive.metastore.DecimalStatistics)1 DoubleStatistics (io.trino.plugin.hive.metastore.DoubleStatistics)1 HiveColumnStatistics.createBinaryColumnStatistics (io.trino.plugin.hive.metastore.HiveColumnStatistics.createBinaryColumnStatistics)1 HiveColumnStatistics.createBooleanColumnStatistics (io.trino.plugin.hive.metastore.HiveColumnStatistics.createBooleanColumnStatistics)1 HiveColumnStatistics.createIntegerColumnStatistics (io.trino.plugin.hive.metastore.HiveColumnStatistics.createIntegerColumnStatistics)1 ADD (io.trino.plugin.hive.util.Statistics.ReduceOperator.ADD)1 SUBTRACT (io.trino.plugin.hive.util.Statistics.ReduceOperator.SUBTRACT)1 Statistics.createHiveColumnStatistics (io.trino.plugin.hive.util.Statistics.createHiveColumnStatistics)1 Statistics.merge (io.trino.plugin.hive.util.Statistics.merge)1 Statistics.reduce (io.trino.plugin.hive.util.Statistics.reduce)1