Examples with TupleDomainFilter com.facebook.presto.common.predicate.TupleDomainFilter used on opensource projects

Example 11 with TupleDomainFilter

use of com.facebook.presto.common.predicate.TupleDomainFilter in project presto by prestodb.

the class TestListFilter method assertPositionalFilter.

private void assertPositionalFilter(Map<RowAndColumn, TupleDomainFilter> filters, Integer[][][] data) {
    ListFilter listFilter = buildListFilter(filters, data);
    TestFilter2 testFilter = (i, j, value) -> Optional.ofNullable(filters.get(RowAndColumn.of(i + 1, j + 1))).map(filter -> value == null ? filter.testNull() : filter.testLong(value)).orElse(true);
    PositionalFilter positionalFilter = listFilter.getChild().getPositionalFilter();
    for (int i = 0; i < data.length; i++) {
        boolean expectedToPass = true;
        int passCount = 0;
        int failCount = 0;
        for (int j = 0; j < data[i].length; j++) {
            if (!expectedToPass) {
                failCount += data[i][j].length;
                continue;
            }
            for (int k = 0; k < data[i][j].length; k++) {
                Integer value = data[i][j][k];
                expectedToPass = testFilter.test(j, k, value);
                assertEquals(value == null ? positionalFilter.testNull() : positionalFilter.testLong(value), expectedToPass);
                if (!expectedToPass) {
                    assertEquals(positionalFilter.getPrecedingPositionsToFail(), passCount);
                    failCount = data[i][j].length - k - 1;
                    break;
                }
                passCount++;
            }
        }
        assertEquals(positionalFilter.getSucceedingPositionsToFail(), failCount);
    }
}

Example 12 with TupleDomainFilter

use of com.facebook.presto.common.predicate.TupleDomainFilter in project presto by prestodb.

the class TestListFilter method assertPositionalFilter.

private void assertPositionalFilter(Map<Integer, TupleDomainFilter> filters, Integer[][] data) {
    ListFilter listFilter = buildListFilter(filters, data);
    TestFilter1 testFilter = (i, value) -> Optional.ofNullable(filters.get(i + 1)).map(filter -> value == null ? filter.testNull() : filter.testLong(value)).orElse(true);
    PositionalFilter positionalFilter = listFilter.getPositionalFilter();
    for (int i = 0; i < data.length; i++) {
        for (int j = 0; j < data[i].length; j++) {
            Integer value = data[i][j];
            boolean expectedToPass = testFilter.test(j, value);
            assertEquals(value == null ? positionalFilter.testNull() : positionalFilter.testLong(value), expectedToPass);
            if (!expectedToPass) {
                assertEquals(positionalFilter.getPrecedingPositionsToFail(), j);
                assertEquals(positionalFilter.getSucceedingPositionsToFail(), data[i].length - j - 1);
                break;
            }
        }
    }
}

Example 13 with TupleDomainFilter

use of com.facebook.presto.common.predicate.TupleDomainFilter in project presto by prestodb.

the class FilteringPageSource method getNextPage.

@Override
public Page getNextPage() {
    Page page = delegate.getNextPage();
    if (page == null || page.getPositionCount() == 0) {
        return page;
    }
    int positionCount = page.getPositionCount();
    int[] positions = new int[positionCount];
    for (int i = 0; i < positionCount; i++) {
        positions[i] = i;
    }
    for (int i = 0; i < page.getChannelCount(); i++) {
        TupleDomainFilter domainFilter = domainFilters[i];
        if (domainFilter != null) {
            positionCount = filterBlock(page.getBlock(i), columnTypes[i], domainFilter, positions, positionCount);
            if (positionCount == 0) {
                return new Page(0);
            }
        }
    }
    if (filterFunction != null) {
        RuntimeException[] errors = new RuntimeException[positionCount];
        int[] inputChannels = filterFunction.getInputChannels();
        Block[] inputBlocks = new Block[inputChannels.length];
        for (int i = 0; i < inputChannels.length; i++) {
            inputBlocks[i] = page.getBlock(this.functionInputs.get(inputChannels[i]));
        }
        Page inputPage = new Page(page.getPositionCount(), inputBlocks);
        positionCount = filterFunction.filter(inputPage, positions, positionCount, errors);
        for (int i = 0; i < positionCount; i++) {
            if (errors[i] != null) {
                throw errors[i];
            }
        }
        if (positionCount == 0) {
            return new Page(0);
        }
    }
    if (outputBlockCount == page.getChannelCount()) {
        return page.getPositions(positions, 0, positionCount);
    }
    Block[] blocks = new Block[outputBlockCount];
    for (int i = 0; i < outputBlockCount; i++) {
        blocks[i] = page.getBlock(i);
    }
    return new Page(page.getPositionCount(), blocks).getPositions(positions, 0, positionCount);
}

Example 14 with TupleDomainFilter

use of com.facebook.presto.common.predicate.TupleDomainFilter in project presto by prestodb.

the class TestSelectiveOrcReader method testOutputNotRequired.

@Test
public void testOutputNotRequired() throws Exception {
    List<Type> types = ImmutableList.of(VARCHAR, VARCHAR);
    TempFile tempFile = new TempFile();
    List<String> varcharDirectValues = newArrayList(limit(cycle(ImmutableList.of("A", "B", "C")), NUM_ROWS));
    List<List<?>> values = ImmutableList.of(varcharDirectValues, varcharDirectValues);
    writeOrcColumnsPresto(tempFile.getFile(), DWRF, NONE, Optional.empty(), types, values, new OrcWriterStats());
    OrcPredicate orcPredicate = createOrcPredicate(types, values, DWRF, false);
    // ImmutableMap.of(1, stringIn(true, "10", "11"));
    Map<Subfield, TupleDomainFilter> filters = ImmutableMap.of(new Subfield("c"), stringIn(true, "A", "B", "C"));
    Map<Integer, Type> includedColumns = IntStream.range(0, types.size()).boxed().collect(toImmutableMap(Function.identity(), types::get));
    // Do not output column 0 but only column 1
    List<Integer> outputColumns = ImmutableList.of(1);
    try (OrcSelectiveRecordReader recordReader = createCustomOrcSelectiveRecordReader(tempFile.getFile(), DWRF.getOrcEncoding(), orcPredicate, types, MAX_BATCH_SIZE, ImmutableMap.of(0, filters), OrcReaderSettings.builder().build().getFilterFunctions(), OrcReaderSettings.builder().build().getFilterFunctionInputMapping(), OrcReaderSettings.builder().build().getRequiredSubfields(), ImmutableMap.of(), ImmutableMap.of(), includedColumns, outputColumns, false, new TestingHiveOrcAggregatedMemoryContext(), false)) {
        assertEquals(recordReader.getReaderPosition(), 0);
        assertEquals(recordReader.getFilePosition(), 0);
        int rowsProcessed = 0;
        while (true) {
            Page page = recordReader.getNextPage();
            if (page == null) {
                break;
            }
            int positionCount = page.getPositionCount();
            if (positionCount == 0) {
                continue;
            }
            page.getLoadedPage();
            // The output block should be the second block
            assertBlockPositions(page.getBlock(0), varcharDirectValues.subList(rowsProcessed, rowsProcessed + positionCount));
            rowsProcessed += positionCount;
        }
        assertEquals(rowsProcessed, NUM_ROWS);
    }
}

Example 15 with TupleDomainFilter

use of com.facebook.presto.common.predicate.TupleDomainFilter in project presto by prestodb.

the class TestListFilter method buildListFilter.

private static ListFilter buildListFilter(Map<Integer, TupleDomainFilter> filters, Integer[][] data) {
    Map<Subfield, TupleDomainFilter> subfieldFilters = filters.entrySet().stream().collect(toImmutableMap(entry -> toSubfield(entry.getKey()), Map.Entry::getValue));
    ListFilter filter = new ListFilter(makeStreamDescriptor(1), subfieldFilters);
    int[] lengths = Arrays.stream(data).mapToInt(v -> v.length).toArray();
    filter.populateElementFilters(data.length, null, lengths, Arrays.stream(lengths).sum());
    return filter;
}