Search in sources :

Example 16 with TupleDomainFilter

use of com.facebook.presto.common.predicate.TupleDomainFilter in project presto by prestodb.

the class TestListFilter method buildListFilter.

private static ListFilter buildListFilter(Map<RowAndColumn, TupleDomainFilter> filters, Integer[][][] data) {
    Map<Subfield, TupleDomainFilter> subfieldFilters = filters.entrySet().stream().collect(toImmutableMap(entry -> toSubfield(entry.getKey()), Map.Entry::getValue));
    ListFilter filter = new ListFilter(makeStreamDescriptor(2), subfieldFilters);
    int[] lengths = Arrays.stream(data).mapToInt(v -> v.length).toArray();
    filter.populateElementFilters(data.length, null, lengths, Arrays.stream(lengths).sum());
    int[] nestedLenghts = Arrays.stream(data).flatMap(Arrays::stream).mapToInt(v -> v.length).toArray();
    ((ListFilter) filter.getChild()).populateElementFilters(Arrays.stream(lengths).sum(), null, nestedLenghts, Arrays.stream(nestedLenghts).sum());
    return filter;
}
Also used : LIST(com.facebook.presto.orc.metadata.OrcType.OrcTypeKind.LIST) Arrays(java.util.Arrays) ListFilter(com.facebook.presto.orc.reader.ListFilter) ImmutableMap(com.google.common.collect.ImmutableMap) BigintRange(com.facebook.presto.common.predicate.TupleDomainFilter.BigintRange) Assert.assertEquals(org.testng.Assert.assertEquals) Test(org.testng.annotations.Test) Random(java.util.Random) String.format(java.lang.String.format) TupleDomainFilter(com.facebook.presto.common.predicate.TupleDomainFilter) Objects(java.util.Objects) INT(com.facebook.presto.orc.metadata.OrcType.OrcTypeKind.INT) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) Subfield(com.facebook.presto.common.Subfield) ImmutableList(com.google.common.collect.ImmutableList) Map(java.util.Map) Optional(java.util.Optional) OrcType(com.facebook.presto.orc.metadata.OrcType) PositionalFilter(com.facebook.presto.common.predicate.TupleDomainFilter.PositionalFilter) ListFilter(com.facebook.presto.orc.reader.ListFilter) Arrays(java.util.Arrays) ImmutableMap(com.google.common.collect.ImmutableMap) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) Map(java.util.Map) Subfield(com.facebook.presto.common.Subfield) TupleDomainFilter(com.facebook.presto.common.predicate.TupleDomainFilter)

Example 17 with TupleDomainFilter

use of com.facebook.presto.common.predicate.TupleDomainFilter in project presto by prestodb.

the class TestCoercingFilters method testIntegerToVarchar.

@Test
public void testIntegerToVarchar() {
    TupleDomainFilter filter = BytesRange.of("10".getBytes(), false, "10".getBytes(), false, false);
    HiveCoercer coercer = new IntegerNumberToVarcharCoercer(INTEGER, VARCHAR);
    TupleDomainFilter coercingFilter = coercer.toCoercingFilter(filter, new Subfield("c"));
    assertTrue(coercingFilter.testLong(10));
    assertFalse(coercingFilter.testLong(25));
    assertFalse(coercingFilter.testNull());
}
Also used : IntegerNumberToVarcharCoercer(com.facebook.presto.hive.HiveCoercer.IntegerNumberToVarcharCoercer) Subfield(com.facebook.presto.common.Subfield) TupleDomainFilter(com.facebook.presto.common.predicate.TupleDomainFilter) Test(org.testng.annotations.Test)

Example 18 with TupleDomainFilter

use of com.facebook.presto.common.predicate.TupleDomainFilter in project presto by prestodb.

the class TestCoercingFilters method testVarcharToInteger.

@Test
public void testVarcharToInteger() {
    TupleDomainFilter filter = BigintRange.of(100, Integer.MAX_VALUE, false);
    HiveCoercer coercer = new VarcharToIntegerNumberCoercer(VARCHAR, INTEGER);
    TupleDomainFilter coercingFilter = coercer.toCoercingFilter(filter, new Subfield("c"));
    assertTrue(coercingFilter.testLength(1));
    assertTrue(coercingFilter.testLength(2));
    assertTrue(coercingFilter.testLength(3));
    assertTrue(coercingFilter.testBytes("100".getBytes(), 0, 3));
    assertTrue(coercingFilter.testBytes("145".getBytes(), 0, 3));
    assertTrue(coercingFilter.testBytes("2147483647".getBytes(), 0, 10));
    assertFalse(coercingFilter.testBytes("50".getBytes(), 0, 2));
    assertFalse(coercingFilter.testBytes("-50".getBytes(), 0, 3));
    // parsing error
    assertFalse(coercingFilter.testBytes("abc".getBytes(), 0, 3));
    // out of range
    assertFalse(coercingFilter.testBytes("2147483648".getBytes(), 0, 10));
    assertFalse(coercingFilter.testNull());
}
Also used : VarcharToIntegerNumberCoercer(com.facebook.presto.hive.HiveCoercer.VarcharToIntegerNumberCoercer) Subfield(com.facebook.presto.common.Subfield) TupleDomainFilter(com.facebook.presto.common.predicate.TupleDomainFilter) Test(org.testng.annotations.Test)

Example 19 with TupleDomainFilter

use of com.facebook.presto.common.predicate.TupleDomainFilter in project presto by prestodb.

the class SelectiveStreamReaders method createStreamReader.

public static SelectiveStreamReader createStreamReader(StreamDescriptor streamDescriptor, Map<Subfield, TupleDomainFilter> filters, Optional<Type> outputType, List<Subfield> requiredSubfields, DateTimeZone hiveStorageTimeZone, OrcRecordReaderOptions options, boolean legacyMapSubscript, OrcAggregatedMemoryContext systemMemoryContext) {
    OrcTypeKind type = streamDescriptor.getOrcTypeKind();
    switch(type) {
        case BOOLEAN:
            {
                checkArgument(requiredSubfields.isEmpty(), "Boolean stream reader doesn't support subfields");
                verifyStreamType(streamDescriptor, outputType, BooleanType.class::isInstance);
                return new BooleanSelectiveStreamReader(streamDescriptor, getOptionalOnlyFilter(type, filters), outputType.isPresent(), systemMemoryContext.newOrcLocalMemoryContext(SelectiveStreamReaders.class.getSimpleName()));
            }
        case BYTE:
            {
                checkArgument(requiredSubfields.isEmpty(), "Byte stream reader doesn't support subfields");
                verifyStreamType(streamDescriptor, outputType, TinyintType.class::isInstance);
                return new ByteSelectiveStreamReader(streamDescriptor, getOptionalOnlyFilter(type, filters), outputType.isPresent(), systemMemoryContext.newOrcLocalMemoryContext(SelectiveStreamReaders.class.getSimpleName()));
            }
        case SHORT:
        case INT:
        case LONG:
        case DATE:
            {
                checkArgument(requiredSubfields.isEmpty(), "Primitive type stream reader doesn't support subfields");
                verifyStreamType(streamDescriptor, outputType, t -> t instanceof BigintType || t instanceof IntegerType || t instanceof SmallintType || t instanceof DateType);
                return new LongSelectiveStreamReader(streamDescriptor, getOptionalOnlyFilter(type, filters), outputType, systemMemoryContext);
            }
        case FLOAT:
            {
                checkArgument(requiredSubfields.isEmpty(), "Float type stream reader doesn't support subfields");
                verifyStreamType(streamDescriptor, outputType, RealType.class::isInstance);
                return new FloatSelectiveStreamReader(streamDescriptor, getOptionalOnlyFilter(type, filters), outputType.isPresent(), systemMemoryContext.newOrcLocalMemoryContext(SelectiveStreamReaders.class.getSimpleName()));
            }
        case DOUBLE:
            checkArgument(requiredSubfields.isEmpty(), "Double stream reader doesn't support subfields");
            verifyStreamType(streamDescriptor, outputType, DoubleType.class::isInstance);
            return new DoubleSelectiveStreamReader(streamDescriptor, getOptionalOnlyFilter(type, filters), outputType.isPresent(), systemMemoryContext.newOrcLocalMemoryContext(SelectiveStreamReaders.class.getSimpleName()));
        case BINARY:
        case STRING:
        case VARCHAR:
        case CHAR:
            checkArgument(requiredSubfields.isEmpty(), "Primitive stream reader doesn't support subfields");
            verifyStreamType(streamDescriptor, outputType, t -> t instanceof VarcharType || t instanceof CharType || t instanceof VarbinaryType);
            return new SliceSelectiveStreamReader(streamDescriptor, getOptionalOnlyFilter(type, filters), outputType, systemMemoryContext);
        case TIMESTAMP:
            {
                checkArgument(requiredSubfields.isEmpty(), "Timestamp stream reader doesn't support subfields");
                verifyStreamType(streamDescriptor, outputType, TimestampType.class::isInstance);
                return new TimestampSelectiveStreamReader(streamDescriptor, getOptionalOnlyFilter(type, filters), hiveStorageTimeZone, outputType.isPresent(), systemMemoryContext.newOrcLocalMemoryContext(SelectiveStreamReaders.class.getSimpleName()), options);
            }
        case LIST:
            verifyStreamType(streamDescriptor, outputType, ArrayType.class::isInstance);
            return new ListSelectiveStreamReader(streamDescriptor, filters, requiredSubfields, null, 0, outputType, hiveStorageTimeZone, options, legacyMapSubscript, systemMemoryContext);
        case STRUCT:
            verifyStreamType(streamDescriptor, outputType, RowType.class::isInstance);
            return new StructSelectiveStreamReader(streamDescriptor, filters, requiredSubfields, outputType, hiveStorageTimeZone, options, legacyMapSubscript, systemMemoryContext);
        case MAP:
            verifyStreamType(streamDescriptor, outputType, MapType.class::isInstance);
            return new MapSelectiveStreamReader(streamDescriptor, filters, requiredSubfields, outputType, hiveStorageTimeZone, options, legacyMapSubscript, systemMemoryContext);
        case DECIMAL:
            {
                verifyStreamType(streamDescriptor, outputType, DecimalType.class::isInstance);
                if (streamDescriptor.getOrcType().getPrecision().get() <= MAX_SHORT_PRECISION) {
                    return new ShortDecimalSelectiveStreamReader(streamDescriptor, getOptionalOnlyFilter(type, filters), outputType, systemMemoryContext.newOrcLocalMemoryContext(SelectiveStreamReaders.class.getSimpleName()));
                } else {
                    return new LongDecimalSelectiveStreamReader(streamDescriptor, getOptionalOnlyFilter(type, filters), outputType, systemMemoryContext.newOrcLocalMemoryContext(SelectiveStreamReaders.class.getSimpleName()));
                }
            }
        case UNION:
        default:
            throw new IllegalArgumentException("Unsupported type: " + type);
    }
}
Also used : StreamDescriptor(com.facebook.presto.orc.StreamDescriptor) Iterables(com.google.common.collect.Iterables) DateTimeZone(org.joda.time.DateTimeZone) MapType(com.facebook.presto.common.type.MapType) DecimalType(com.facebook.presto.common.type.DecimalType) BooleanType(com.facebook.presto.common.type.BooleanType) OrcTypeKind(com.facebook.presto.orc.metadata.OrcType.OrcTypeKind) MAX_SHORT_PRECISION(com.facebook.presto.common.type.Decimals.MAX_SHORT_PRECISION) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) IntegerType(com.facebook.presto.common.type.IntegerType) Subfield(com.facebook.presto.common.Subfield) Map(java.util.Map) ArrayType(com.facebook.presto.common.type.ArrayType) CharType(com.facebook.presto.common.type.CharType) Type(com.facebook.presto.common.type.Type) TinyintType(com.facebook.presto.common.type.TinyintType) OrcAggregatedMemoryContext(com.facebook.presto.orc.OrcAggregatedMemoryContext) ImmutableMap(com.google.common.collect.ImmutableMap) Predicate(java.util.function.Predicate) BigintType(com.facebook.presto.common.type.BigintType) OrcRecordReaderOptions(com.facebook.presto.orc.OrcRecordReaderOptions) VarcharType(com.facebook.presto.common.type.VarcharType) RealType(com.facebook.presto.common.type.RealType) String.format(java.lang.String.format) TupleDomainFilter(com.facebook.presto.common.predicate.TupleDomainFilter) SmallintType(com.facebook.presto.common.type.SmallintType) List(java.util.List) VarbinaryType(com.facebook.presto.common.type.VarbinaryType) Optional(java.util.Optional) Arrays.ensureCapacity(com.facebook.presto.common.array.Arrays.ensureCapacity) DateType(com.facebook.presto.common.type.DateType) DoubleType(com.facebook.presto.common.type.DoubleType) RowType(com.facebook.presto.common.type.RowType) TimestampType(com.facebook.presto.common.type.TimestampType) VarcharType(com.facebook.presto.common.type.VarcharType) RowType(com.facebook.presto.common.type.RowType) OrcTypeKind(com.facebook.presto.orc.metadata.OrcType.OrcTypeKind) MapType(com.facebook.presto.common.type.MapType) ArrayType(com.facebook.presto.common.type.ArrayType) VarbinaryType(com.facebook.presto.common.type.VarbinaryType) SmallintType(com.facebook.presto.common.type.SmallintType) DateType(com.facebook.presto.common.type.DateType) BigintType(com.facebook.presto.common.type.BigintType) IntegerType(com.facebook.presto.common.type.IntegerType) DoubleType(com.facebook.presto.common.type.DoubleType) CharType(com.facebook.presto.common.type.CharType)

Example 20 with TupleDomainFilter

use of com.facebook.presto.common.predicate.TupleDomainFilter in project presto by prestodb.

the class ListSelectiveStreamReader method getTopLevelFilter.

private static Optional<TupleDomainFilter> getTopLevelFilter(Map<Subfield, TupleDomainFilter> filters) {
    Map<Subfield, TupleDomainFilter> topLevelFilters = Maps.filterEntries(filters, entry -> entry.getKey().getPath().isEmpty());
    if (topLevelFilters.isEmpty()) {
        return Optional.empty();
    }
    checkArgument(topLevelFilters.size() == 1, "ARRAY column may have at most one top-level range filter");
    TupleDomainFilter filter = Iterables.getOnlyElement(topLevelFilters.values());
    checkArgument(filter == IS_NULL || filter == IS_NOT_NULL, "Top-level range filter on ARRAY column must be IS NULL or IS NOT NULL");
    return Optional.of(filter);
}
Also used : Subfield(com.facebook.presto.common.Subfield) TupleDomainFilter(com.facebook.presto.common.predicate.TupleDomainFilter)

Aggregations

TupleDomainFilter (com.facebook.presto.common.predicate.TupleDomainFilter)23 Subfield (com.facebook.presto.common.Subfield)20 ImmutableMap (com.google.common.collect.ImmutableMap)12 Map (java.util.Map)12 ImmutableList (com.google.common.collect.ImmutableList)11 Test (org.testng.annotations.Test)11 Type (com.facebook.presto.common.type.Type)9 ImmutableMap.toImmutableMap (com.google.common.collect.ImmutableMap.toImmutableMap)9 Optional (java.util.Optional)9 List (java.util.List)8 Block (com.facebook.presto.common.block.Block)7 HashMap (java.util.HashMap)7 Objects (java.util.Objects)7 Page (com.facebook.presto.common.Page)6 BigintRange (com.facebook.presto.common.predicate.TupleDomainFilter.BigintRange)6 CharType (com.facebook.presto.common.type.CharType)6 DecimalType (com.facebook.presto.common.type.DecimalType)6 ArrayList (java.util.ArrayList)6 Arrays (java.util.Arrays)6 PositionalFilter (com.facebook.presto.common.predicate.TupleDomainFilter.PositionalFilter)5