Search in sources :

Example 46 with ColumnStatistics

use of com.facebook.presto.orc.metadata.statistics.ColumnStatistics in project presto by prestodb.

the class TupleDomainOrcPredicate method getDomain.

@VisibleForTesting
public static Domain getDomain(Type type, long rowCount, ColumnStatistics columnStatistics) {
    if (rowCount == 0) {
        return Domain.none(type);
    }
    if (columnStatistics == null) {
        return Domain.all(type);
    }
    if (columnStatistics.hasNumberOfValues() && columnStatistics.getNumberOfValues() == 0) {
        return Domain.onlyNull(type);
    }
    boolean hasNullValue = columnStatistics.getNumberOfValues() != rowCount;
    if (type.getJavaType() == boolean.class && columnStatistics.getBooleanStatistics() != null) {
        BooleanStatistics booleanStatistics = columnStatistics.getBooleanStatistics();
        boolean hasTrueValues = (booleanStatistics.getTrueValueCount() != 0);
        boolean hasFalseValues = (columnStatistics.getNumberOfValues() != booleanStatistics.getTrueValueCount());
        if (hasTrueValues && hasFalseValues) {
            return Domain.all(BOOLEAN);
        }
        if (hasTrueValues) {
            return Domain.create(ValueSet.of(BOOLEAN, true), hasNullValue);
        }
        if (hasFalseValues) {
            return Domain.create(ValueSet.of(BOOLEAN, false), hasNullValue);
        }
    } else if (isShortDecimal(type) && columnStatistics.getDecimalStatistics() != null) {
        return createDomain(type, hasNullValue, columnStatistics.getDecimalStatistics(), value -> rescale(value, (DecimalType) type).unscaledValue().longValue());
    } else if (isLongDecimal(type) && columnStatistics.getDecimalStatistics() != null) {
        return createDomain(type, hasNullValue, columnStatistics.getDecimalStatistics(), value -> encodeUnscaledValue(rescale(value, (DecimalType) type).unscaledValue()));
    } else if (isCharType(type) && columnStatistics.getStringStatistics() != null) {
        return createDomain(type, hasNullValue, columnStatistics.getStringStatistics(), value -> truncateToLengthAndTrimSpaces(value, type));
    } else if (isVarcharType(type) && columnStatistics.getStringStatistics() != null) {
        return createDomain(type, hasNullValue, columnStatistics.getStringStatistics());
    } else if (type.getTypeSignature().getBase().equals(StandardTypes.DATE) && columnStatistics.getDateStatistics() != null) {
        return createDomain(type, hasNullValue, columnStatistics.getDateStatistics(), value -> (long) value);
    } else if (type.getJavaType() == long.class && columnStatistics.getIntegerStatistics() != null) {
        return createDomain(type, hasNullValue, columnStatistics.getIntegerStatistics());
    } else if (type.getJavaType() == double.class && columnStatistics.getDoubleStatistics() != null) {
        return createDomain(type, hasNullValue, columnStatistics.getDoubleStatistics());
    } else if (REAL.equals(type) && columnStatistics.getDoubleStatistics() != null) {
        return createDomain(type, hasNullValue, columnStatistics.getDoubleStatistics(), value -> (long) floatToRawIntBits(value.floatValue()));
    }
    return Domain.create(ValueSet.all(type), hasNullValue);
}
Also used : StandardTypes(com.facebook.presto.common.type.StandardTypes) DecimalType(com.facebook.presto.common.type.DecimalType) Slice(io.airlift.slice.Slice) Chars.isCharType(com.facebook.presto.common.type.Chars.isCharType) Decimals.encodeUnscaledValue(com.facebook.presto.common.type.Decimals.encodeUnscaledValue) TINYINT(com.facebook.presto.common.type.TinyintType.TINYINT) Function(java.util.function.Function) REAL(com.facebook.presto.common.type.RealType.REAL) Float.floatToRawIntBits(java.lang.Float.floatToRawIntBits) Chars.truncateToLengthAndTrimSpaces(com.facebook.presto.common.type.Chars.truncateToLengthAndTrimSpaces) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) ImmutableList(com.google.common.collect.ImmutableList) HiveBloomFilter(com.facebook.presto.orc.metadata.statistics.HiveBloomFilter) Map(java.util.Map) Objects.requireNonNull(java.util.Objects.requireNonNull) BOOLEAN(com.facebook.presto.common.type.BooleanType.BOOLEAN) Varchars.isVarcharType(com.facebook.presto.common.type.Varchars.isVarcharType) Type(com.facebook.presto.common.type.Type) BIGINT(com.facebook.presto.common.type.BigintType.BIGINT) DOUBLE(com.facebook.presto.common.type.DoubleType.DOUBLE) Collection(java.util.Collection) VarcharType(com.facebook.presto.common.type.VarcharType) Domain(com.facebook.presto.common.predicate.Domain) TupleDomain(com.facebook.presto.common.predicate.TupleDomain) Decimals.isLongDecimal(com.facebook.presto.common.type.Decimals.isLongDecimal) Range(com.facebook.presto.common.predicate.Range) ColumnStatistics(com.facebook.presto.orc.metadata.statistics.ColumnStatistics) List(java.util.List) Decimals.isShortDecimal(com.facebook.presto.common.type.Decimals.isShortDecimal) BooleanStatistics(com.facebook.presto.orc.metadata.statistics.BooleanStatistics) SMALLINT(com.facebook.presto.common.type.SmallintType.SMALLINT) Decimals.rescale(com.facebook.presto.common.type.Decimals.rescale) INTEGER(com.facebook.presto.common.type.IntegerType.INTEGER) VarbinaryType(com.facebook.presto.common.type.VarbinaryType) Optional(java.util.Optional) BloomFilter(com.facebook.presto.orc.metadata.statistics.BloomFilter) VisibleForTesting(com.google.common.annotations.VisibleForTesting) ValueSet(com.facebook.presto.common.predicate.ValueSet) RangeStatistics(com.facebook.presto.orc.metadata.statistics.RangeStatistics) MoreObjects.toStringHelper(com.google.common.base.MoreObjects.toStringHelper) BooleanStatistics(com.facebook.presto.orc.metadata.statistics.BooleanStatistics) DecimalType(com.facebook.presto.common.type.DecimalType) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Aggregations

ColumnStatistics (com.facebook.presto.orc.metadata.statistics.ColumnStatistics)46 ImmutableList (com.google.common.collect.ImmutableList)22 Slice (io.airlift.slice.Slice)22 List (java.util.List)22 ArrayList (java.util.ArrayList)19 Stream (com.facebook.presto.orc.metadata.Stream)18 StreamDataOutput (com.facebook.presto.orc.stream.StreamDataOutput)15 RowGroupIndex (com.facebook.presto.orc.metadata.RowGroupIndex)14 BooleanStreamCheckpoint (com.facebook.presto.orc.checkpoint.BooleanStreamCheckpoint)12 PresentOutputStream (com.facebook.presto.orc.stream.PresentOutputStream)12 ImmutableMap (com.google.common.collect.ImmutableMap)11 LongStreamCheckpoint (com.facebook.presto.orc.checkpoint.LongStreamCheckpoint)7 OrcType (com.facebook.presto.orc.metadata.OrcType)7 LongOutputStream (com.facebook.presto.orc.stream.LongOutputStream)7 Map (java.util.Map)7 Type (com.facebook.presto.common.type.Type)6 IOException (java.io.IOException)6 HashMap (java.util.HashMap)6 ColumnEncoding (com.facebook.presto.orc.metadata.ColumnEncoding)5 StripeEncryptionGroup (com.facebook.presto.orc.metadata.StripeEncryptionGroup)5