Examples with TupleDomainFilter com.facebook.presto.common.predicate.TupleDomainFilter used on opensource projects

Example 1 with TupleDomainFilter

use of com.facebook.presto.common.predicate.TupleDomainFilter in project presto by prestodb.

the class StructSelectiveStreamReader method getTopLevelFilter.

private static Optional<TupleDomainFilter> getTopLevelFilter(Map<Subfield, TupleDomainFilter> filters) {
    Map<Subfield, TupleDomainFilter> topLevelFilters = Maps.filterEntries(filters, entry -> entry.getKey().getPath().isEmpty());
    if (topLevelFilters.isEmpty()) {
        return Optional.empty();
    }
    checkArgument(topLevelFilters.size() == 1, "ROW column may have at most one top-level range filter");
    TupleDomainFilter filter = Iterables.getOnlyElement(topLevelFilters.values());
    checkArgument(filter == IS_NULL || filter == IS_NOT_NULL, "Top-level range filter on ROW column must be IS NULL or IS NOT NULL");
    return Optional.of(filter);
}

Example 2 with TupleDomainFilter

use of com.facebook.presto.common.predicate.TupleDomainFilter in project presto by prestodb.

the class StructSelectiveStreamReader method checkMissingFieldFilters.

private boolean checkMissingFieldFilters(Collection<StreamDescriptor> nestedStreams, Map<Subfield, TupleDomainFilter> filters) {
    if (filters.isEmpty()) {
        return true;
    }
    Set<String> presentFieldNames = nestedStreams.stream().map(StreamDescriptor::getFieldName).map(name -> name.toLowerCase(Locale.ENGLISH)).collect(toImmutableSet());
    for (Map.Entry<Subfield, TupleDomainFilter> entry : filters.entrySet()) {
        Subfield subfield = entry.getKey();
        if (subfield.getPath().isEmpty()) {
            continue;
        }
        String fieldName = ((Subfield.NestedField) subfield.getPath().get(0)).getName();
        if (presentFieldNames.contains(fieldName)) {
            continue;
        }
        // Check out the filter. If filter allows nulls, then all rows pass, otherwise, no row passes.
        TupleDomainFilter filter = entry.getValue();
        checkArgument(filter.isDeterministic(), "Non-deterministic range filters are not supported yet");
        if (!filter.testNull()) {
            return false;
        }
    }
    return true;
}

Example 3 with TupleDomainFilter

use of com.facebook.presto.common.predicate.TupleDomainFilter in project presto by prestodb.

the class OrcTester method testRow.

private static boolean testRow(List<Type> types, List<List<?>> values, int row, Map<Integer, Map<Subfield, TupleDomainFilter>> columnFilters) {
    for (int column = 0; column < types.size(); column++) {
        Map<Subfield, TupleDomainFilter> filters = columnFilters.get(column);
        if (filters == null) {
            continue;
        }
        Type type = types.get(column);
        Object value = values.get(column).get(row);
        for (Map.Entry<Subfield, TupleDomainFilter> entry : filters.entrySet()) {
            if (!testSubfieldValue(type, value, entry.getKey(), entry.getValue())) {
                return false;
            }
        }
    }
    return true;
}

Example 4 with TupleDomainFilter

use of com.facebook.presto.common.predicate.TupleDomainFilter in project presto by prestodb.

the class OrcTester method assertFileContentsPresto.

private static void assertFileContentsPresto(List<Type> types, TempFile tempFile, List<List<?>> expectedValues, boolean skipFirstBatch, boolean skipStripe, OrcEncoding orcEncoding, Format format, boolean isHiveWriter, boolean useSelectiveOrcReader, List<OrcReaderSettings> settings, Map<Integer, Slice> intermediateEncryptionKeys) throws IOException {
    OrcPredicate orcPredicate = createOrcPredicate(types, expectedValues, format, isHiveWriter);
    Map<Integer, Type> includedColumns = IntStream.range(0, types.size()).boxed().collect(toImmutableMap(Function.identity(), types::get));
    List<Integer> outputColumns = IntStream.range(0, types.size()).boxed().collect(toImmutableList());
    if (useSelectiveOrcReader) {
        assertFileContentsPresto(types, tempFile.getFile(), expectedValues, orcEncoding, orcPredicate, Optional.empty(), ImmutableList.of(), ImmutableMap.of(), ImmutableMap.of(), intermediateEncryptionKeys, includedColumns, outputColumns);
        for (OrcReaderSettings entry : settings) {
            assertTrue(entry.getFilterFunctions().isEmpty(), "Filter functions are not supported yet");
            assertTrue(entry.getFilterFunctionInputMapping().isEmpty(), "Filter functions are not supported yet");
            Map<Integer, Map<Subfield, TupleDomainFilter>> columnFilters = entry.getColumnFilters();
            List<List<?>> prunedAndFilteredRows = pruneValues(types, filterRows(types, expectedValues, columnFilters), entry.getRequiredSubfields());
            Optional<TupleDomainFilterOrderChecker> orderChecker = Optional.empty();
            List<Integer> expectedFilterOrder = entry.getExpectedFilterOrder();
            if (!expectedFilterOrder.isEmpty()) {
                orderChecker = Optional.of(new TupleDomainFilterOrderChecker(expectedFilterOrder));
            }
            Optional<Map<Integer, Map<Subfield, TupleDomainFilter>>> transformedFilters = Optional.of(orderChecker.map(checker -> addOrderTracking(columnFilters, checker)).orElse(columnFilters));
            assertFileContentsPresto(types, tempFile.getFile(), prunedAndFilteredRows, orcEncoding, orcPredicate, transformedFilters, entry.getFilterFunctions(), entry.getFilterFunctionInputMapping(), entry.getRequiredSubfields());
            orderChecker.ifPresent(TupleDomainFilterOrderChecker::assertOrder);
        }
        return;
    }
    try (OrcBatchRecordReader recordReader = createCustomOrcRecordReader(tempFile, orcEncoding, orcPredicate, types, MAX_BATCH_SIZE, new StorageOrcFileTailSource(), new StorageStripeMetadataSource(), false, intermediateEncryptionKeys, false)) {
        assertEquals(recordReader.getReaderPosition(), 0);
        assertEquals(recordReader.getFilePosition(), 0);
        boolean isFirst = true;
        int rowsProcessed = 0;
        for (int batchSize = toIntExact(recordReader.nextBatch()); batchSize >= 0; batchSize = toIntExact(recordReader.nextBatch())) {
            if (skipStripe && rowsProcessed < 10000) {
            // skip recordReader.readBlock
            } else if (skipFirstBatch && isFirst) {
                // skip recordReader.readBlock
                isFirst = false;
            } else {
                for (int i = 0; i < types.size(); i++) {
                    Type type = types.get(i);
                    Block block = recordReader.readBlock(i);
                    assertEquals(block.getPositionCount(), batchSize);
                    checkNullValues(type, block);
                    assertBlockEquals(type, block, expectedValues.get(i), rowsProcessed);
                }
            }
            assertEquals(recordReader.getReaderPosition(), rowsProcessed);
            assertEquals(recordReader.getFilePosition(), rowsProcessed);
            rowsProcessed += batchSize;
        }
        assertEquals(rowsProcessed, expectedValues.get(0).size());
        assertEquals(recordReader.getReaderPosition(), rowsProcessed);
        assertEquals(recordReader.getFilePosition(), rowsProcessed);
    }
}

Example 5 with TupleDomainFilter

use of com.facebook.presto.common.predicate.TupleDomainFilter in project presto by prestodb.

the class TestPositionalFilter method test.

@Test
public void test() {
    PositionalFilter filter = new PositionalFilter();
    // a[1] = '1' and a[3] = '3' The test data is converted to byte[]'s and the comparison is done using testLength()
    // followed by testBytes() so as to cover the double use of the position when testLength succeeeds and testBytes
    // fails.
    TupleDomainFilter[] filters = new TupleDomainFilter[] { equals(1), null, equals(3), null, equals(1), null, equals(3), null, equals(1), null, equals(3), null, null, equals(1), null, equals(3), null, null, null, equals(1), null, equals(3), null, null, null, null };
    long[] numbers = new long[] { // pass
    1, // pass
    2, // pass
    3, // pass
    4, // fail
    0, // fail
    2, // fail
    3, // fail
    4, // fail testLength()
    1, // fail testLength()
    2, // fail testLength()
    0, // fail testLength()
    4, // fail testLength()
    55, // pass
    1, // pass
    0, // pass
    3, // pass
    0, // pass
    5, // pass
    6, // fail testBytes()
    1, // fail testBytes()
    1, // fail testBytes()
    2, // fail testBytes()
    2, // fail testBytes()
    3, // fail testBytes()
    3, // fail testBytes()
    4 };
    // Convert the values to byte[][].
    byte[][] values = Arrays.stream(numbers).mapToObj(n -> toBytes(Long.valueOf(n).toString())).toArray(byte[][]::new);
    boolean[] expectedResults = new boolean[] { true, true, true, true, false, true, true, false, true, true, true, true, true, true, true, true, false };
    int[] offsets = new int[] { 0, 4, 8, 13, 19, 26 };
    filter.setFilters(filters, offsets);
    int valuesIndex = 0;
    for (int i = 0; i < expectedResults.length; i++) {
        boolean result = filter.testLength(values[valuesIndex].length) && filter.testBytes(values[valuesIndex], 0, values[valuesIndex].length);
        assertEquals(expectedResults[i], result);
        valuesIndex++;
        if (expectedResults[i] == false) {
            valuesIndex += filter.getSucceedingPositionsToFail();
        }
    }
    assertEquals(new boolean[] { false, true, true, false, true, false }, filter.getFailed());
}