Examples with HivePartition com.facebook.presto.hive.HivePartition used on opensource projects

Example 1 with HivePartition

use of com.facebook.presto.hive.HivePartition in project presto by prestodb.

the class TestMetastoreHiveStatisticsProvider method testGetTableStatisticsUnpartitioned.

@Test
public void testGetTableStatisticsUnpartitioned() {
    PartitionStatistics statistics = PartitionStatistics.builder().setBasicStatistics(new HiveBasicStatistics(OptionalLong.empty(), OptionalLong.of(1000), OptionalLong.of(5000), OptionalLong.empty())).setColumnStatistics(ImmutableMap.of(COLUMN, createIntegerColumnStatistics(OptionalLong.of(-100), OptionalLong.of(100), OptionalLong.of(500), OptionalLong.of(300)))).build();
    MetastoreHiveStatisticsProvider statisticsProvider = new MetastoreHiveStatisticsProvider((session, table, hivePartitions) -> ImmutableMap.of(UNPARTITIONED_ID, statistics));
    TestingConnectorSession session = new TestingConnectorSession(new HiveSessionProperties(new HiveClientConfig(), new OrcFileWriterConfig(), new ParquetFileWriterConfig(), new CacheConfig()).getSessionProperties());
    HiveColumnHandle columnHandle = new HiveColumnHandle(COLUMN, HIVE_LONG, BIGINT.getTypeSignature(), 2, REGULAR, Optional.empty(), Optional.empty());
    TableStatistics expected = TableStatistics.builder().setRowCount(Estimate.of(1000)).setTotalSize(Estimate.of(5000)).setColumnStatistics(columnHandle, ColumnStatistics.builder().setRange(new DoubleRange(-100, 100)).setNullsFraction(Estimate.of(0.5)).setDistinctValuesCount(Estimate.of(300)).build()).build();
    assertEquals(statisticsProvider.getTableStatistics(session, TABLE, ImmutableMap.of(COLUMN, columnHandle), ImmutableMap.of(COLUMN, BIGINT), ImmutableList.of(new HivePartition(TABLE))), expected);
}

Example 2 with HivePartition

use of com.facebook.presto.hive.HivePartition in project presto by prestodb.

the class MetastoreHiveStatisticsProvider method calculateRangeForPartitioningKey.

@VisibleForTesting
static Optional<DoubleRange> calculateRangeForPartitioningKey(HiveColumnHandle column, Type type, List<HivePartition> partitions) {
    if (!isRangeSupported(type)) {
        return Optional.empty();
    }
    List<Double> values = partitions.stream().map(HivePartition::getKeys).map(keys -> keys.get(column)).filter(value -> !value.isNull()).map(NullableValue::getValue).map(value -> convertPartitionValueToDouble(type, value)).collect(toImmutableList());
    if (values.isEmpty()) {
        return Optional.empty();
    }
    double min = values.get(0);
    double max = values.get(0);
    for (Double value : values) {
        if (value > max) {
            max = value;
        }
        if (value < min) {
            min = value;
        }
    }
    return Optional.of(new DoubleRange(min, max));
}

Example 3 with HivePartition

use of com.facebook.presto.hive.HivePartition in project presto by prestodb.

the class MetastoreHiveStatisticsProvider method calculateDataSizeForPartitioningKey.

@VisibleForTesting
static Estimate calculateDataSizeForPartitioningKey(HiveColumnHandle column, Type type, List<HivePartition> partitions, Map<String, PartitionStatistics> statistics, double averageRowsPerPartition) {
    if (!hasDataSize(type)) {
        return Estimate.unknown();
    }
    double dataSize = 0;
    for (HivePartition partition : partitions) {
        int length = getSize(partition.getKeys().get(column));
        double rowCount = getPartitionRowCount(partition.getPartitionId(), statistics).orElse(averageRowsPerPartition);
        dataSize += length * rowCount;
    }
    return Estimate.of(dataSize);
}

Example 4 with HivePartition

use of com.facebook.presto.hive.HivePartition in project presto by prestodb.

the class TestMetastoreHiveStatisticsProvider method testNullablePartitionValue.

@Test
public void testNullablePartitionValue() {
    HivePartition partitionWithNull = partition("p1=__HIVE_DEFAULT_PARTITION__/p2=1");
    assertTrue(partitionWithNull.getKeys().containsValue(new NullableValue(VARCHAR, null)));
    HivePartition partitionNameWithSlashN = partition("p1=\\N/p2=2");
    assertTrue(partitionNameWithSlashN.getKeys().containsValue(new NullableValue(VARCHAR, Slices.utf8Slice("\\N"))));
    assertFalse(partitionNameWithSlashN.getKeys().containsValue(new NullableValue(VARCHAR, null)));
}

Example 5 with HivePartition

use of com.facebook.presto.hive.HivePartition in project presto by prestodb.

the class TestMetastoreHiveStatisticsProvider method testGetPartitionsSample.

@Test
public void testGetPartitionsSample() {
    HivePartition p1 = partition("p1=string1/p2=1234");
    HivePartition p2 = partition("p1=string2/p2=2345");
    HivePartition p3 = partition("p1=string3/p2=3456");
    HivePartition p4 = partition("p1=string4/p2=4567");
    HivePartition p5 = partition("p1=string5/p2=5678");
    assertEquals(getPartitionsSample(ImmutableList.of(p1), 1), ImmutableList.of(p1));
    assertEquals(getPartitionsSample(ImmutableList.of(p1), 2), ImmutableList.of(p1));
    assertEquals(getPartitionsSample(ImmutableList.of(p1, p2), 2), ImmutableList.of(p1, p2));
    assertEquals(getPartitionsSample(ImmutableList.of(p1, p2, p3), 2), ImmutableList.of(p1, p3));
    assertEquals(getPartitionsSample(ImmutableList.of(p1, p2, p3, p4), 1), getPartitionsSample(ImmutableList.of(p1, p2, p3, p4), 1));
    assertEquals(getPartitionsSample(ImmutableList.of(p1, p2, p3, p4), 3), getPartitionsSample(ImmutableList.of(p1, p2, p3, p4), 3));
    assertEquals(getPartitionsSample(ImmutableList.of(p1, p2, p3, p4, p5), 3), ImmutableList.of(p1, p5, p4));
}