Examples with Subfield com.facebook.presto.common.Subfield used on opensource projects

Example 31 with Subfield

use of com.facebook.presto.common.Subfield in project presto by prestodb.

the class TestSelectiveOrcReader method testOutputNotRequired.

@Test
public void testOutputNotRequired() throws Exception {
    List<Type> types = ImmutableList.of(VARCHAR, VARCHAR);
    TempFile tempFile = new TempFile();
    List<String> varcharDirectValues = newArrayList(limit(cycle(ImmutableList.of("A", "B", "C")), NUM_ROWS));
    List<List<?>> values = ImmutableList.of(varcharDirectValues, varcharDirectValues);
    writeOrcColumnsPresto(tempFile.getFile(), DWRF, NONE, Optional.empty(), types, values, new OrcWriterStats());
    OrcPredicate orcPredicate = createOrcPredicate(types, values, DWRF, false);
    // ImmutableMap.of(1, stringIn(true, "10", "11"));
    Map<Subfield, TupleDomainFilter> filters = ImmutableMap.of(new Subfield("c"), stringIn(true, "A", "B", "C"));
    Map<Integer, Type> includedColumns = IntStream.range(0, types.size()).boxed().collect(toImmutableMap(Function.identity(), types::get));
    // Do not output column 0 but only column 1
    List<Integer> outputColumns = ImmutableList.of(1);
    try (OrcSelectiveRecordReader recordReader = createCustomOrcSelectiveRecordReader(tempFile.getFile(), DWRF.getOrcEncoding(), orcPredicate, types, MAX_BATCH_SIZE, ImmutableMap.of(0, filters), OrcReaderSettings.builder().build().getFilterFunctions(), OrcReaderSettings.builder().build().getFilterFunctionInputMapping(), OrcReaderSettings.builder().build().getRequiredSubfields(), ImmutableMap.of(), ImmutableMap.of(), includedColumns, outputColumns, false, new TestingHiveOrcAggregatedMemoryContext(), false)) {
        assertEquals(recordReader.getReaderPosition(), 0);
        assertEquals(recordReader.getFilePosition(), 0);
        int rowsProcessed = 0;
        while (true) {
            Page page = recordReader.getNextPage();
            if (page == null) {
                break;
            }
            int positionCount = page.getPositionCount();
            if (positionCount == 0) {
                continue;
            }
            page.getLoadedPage();
            // The output block should be the second block
            assertBlockPositions(page.getBlock(0), varcharDirectValues.subList(rowsProcessed, rowsProcessed + positionCount));
            rowsProcessed += positionCount;
        }
        assertEquals(rowsProcessed, NUM_ROWS);
    }
}

Example 32 with Subfield

use of com.facebook.presto.common.Subfield in project presto by prestodb.

the class TestSelectiveOrcReader method testArrayIndexOutOfBounds.

@Test
public void testArrayIndexOutOfBounds() throws Exception {
    Random random = new Random(0);
    // non-null arrays of varying sizes
    try {
        tester.testRoundTrip(arrayType(INTEGER), createList(NUM_ROWS, i -> randomIntegers(random.nextInt(10), random)), ImmutableList.of(ImmutableMap.of(new Subfield("c[2]"), IS_NULL)));
        fail("Expected 'Array subscript out of bounds' exception");
    } catch (InvalidFunctionArgumentException e) {
        assertTrue(e.getMessage().contains("Array subscript out of bounds"));
    }
    // non-null nested arrays of varying sizes
    try {
        tester.testRoundTrip(arrayType(arrayType(INTEGER)), createList(NUM_ROWS, i -> ImmutableList.of(randomIntegers(random.nextInt(5), random), randomIntegers(random.nextInt(5), random))), ImmutableList.of(ImmutableMap.of(new Subfield("c[2][3]"), IS_NULL)));
        fail("Expected 'Array subscript out of bounds' exception");
    } catch (InvalidFunctionArgumentException e) {
        assertTrue(e.getMessage().contains("Array subscript out of bounds"));
    }
    // empty arrays
    try {
        tester.testRoundTrip(arrayType(INTEGER), nCopies(NUM_ROWS, ImmutableList.of()), ImmutableList.of(ImmutableMap.of(new Subfield("c[2]"), IS_NULL)));
        fail("Expected 'Array subscript out of bounds' exception");
    } catch (InvalidFunctionArgumentException e) {
        assertTrue(e.getMessage().contains("Array subscript out of bounds"));
    }
    // empty nested arrays
    try {
        tester.testRoundTrip(arrayType(arrayType(INTEGER)), nCopies(NUM_ROWS, ImmutableList.of()), ImmutableList.of(ImmutableMap.of(new Subfield("c[2][3]"), IS_NULL)));
        fail("Expected 'Array subscript out of bounds' exception");
    } catch (InvalidFunctionArgumentException e) {
        assertTrue(e.getMessage().contains("Array subscript out of bounds"));
    }
}

Example 33 with Subfield

use of com.facebook.presto.common.Subfield in project presto by prestodb.

the class TestSelectiveOrcReader method testArrays.

@Test
public void testArrays() throws Exception {
    Random random = new Random(0);
    // non-null arrays of varying sizes; some arrays may be empty
    tester.testRoundTrip(arrayType(INTEGER), createList(NUM_ROWS, i -> randomIntegers(random.nextInt(10), random)), IS_NULL, IS_NOT_NULL);
    BigintRange negative = BigintRange.of(Integer.MIN_VALUE, 0, false);
    BigintRange nonNegative = BigintRange.of(0, Integer.MAX_VALUE, false);
    // arrays of strings
    tester.testRoundTrip(arrayType(VARCHAR), createList(1000, i -> randomStrings(5 + random.nextInt(5), random)), ImmutableList.of(toSubfieldFilter("c[1]", IS_NULL), toSubfieldFilter("c[1]", stringIn(true, "a", "b", "c", "d"))));
    tester.testRoundTrip(arrayType(VARCHAR), createList(10, i -> randomStringsWithNulls(5 + random.nextInt(5), random)), ImmutableList.of(toSubfieldFilter("c[1]", IS_NULL), toSubfieldFilter("c[1]", stringIn(true, "a", "b", "c", "d"))));
    // non-empty non-null arrays of varying sizes
    tester.testRoundTrip(arrayType(INTEGER), createList(NUM_ROWS, i -> randomIntegers(5 + random.nextInt(5), random)), ImmutableList.of(toSubfieldFilter(IS_NULL), toSubfieldFilter(IS_NOT_NULL), // c[1] >= 0
    toSubfieldFilter("c[1]", nonNegative), // c[2] >= 0 AND c[4] >= 0
    ImmutableMap.of(new Subfield("c[2]"), nonNegative, new Subfield("c[4]"), nonNegative)));
    // non-null arrays of varying sizes; some arrays may be empty
    tester.testRoundTripTypes(ImmutableList.of(INTEGER, arrayType(INTEGER)), ImmutableList.of(randomIntegers(NUM_ROWS, random), createList(NUM_ROWS, i -> randomIntegers(random.nextInt(10), random))), toSubfieldFilters(ImmutableMap.of(0, nonNegative), ImmutableMap.of(0, nonNegative, 1, IS_NULL), ImmutableMap.of(0, nonNegative, 1, IS_NOT_NULL)));
    // non-empty non-null arrays of varying sizes
    tester.testRoundTripTypes(ImmutableList.of(INTEGER, arrayType(INTEGER)), ImmutableList.of(randomIntegers(NUM_ROWS, random), createList(NUM_ROWS, i -> randomIntegers(5 + random.nextInt(5), random))), ImmutableList.of(// c[1] >= 0
    ImmutableMap.of(0, toSubfieldFilter(nonNegative), 1, toSubfieldFilter("c[1]", nonNegative)), // c[3] >= 0
    ImmutableMap.of(0, toSubfieldFilter(nonNegative), 1, toSubfieldFilter("c[3]", nonNegative)), // c[2] >= 0 AND c[4] <= 0
    ImmutableMap.of(0, toSubfieldFilter(nonNegative), 1, ImmutableMap.of(new Subfield("c[2]"), nonNegative, new Subfield("c[4]"), negative))));
    // nested arrays
    tester.testRoundTripTypes(ImmutableList.of(INTEGER, arrayType(arrayType(INTEGER))), ImmutableList.of(randomIntegers(NUM_ROWS, random), createList(NUM_ROWS, i -> createList(random.nextInt(10), index -> randomIntegers(random.nextInt(5), random)))), toSubfieldFilters(ImmutableMap.of(0, nonNegative), ImmutableMap.of(1, IS_NULL), ImmutableMap.of(1, IS_NOT_NULL), ImmutableMap.of(0, nonNegative, 1, IS_NULL)));
    tester.testRoundTripTypes(ImmutableList.of(INTEGER, arrayType(arrayType(INTEGER))), ImmutableList.of(randomIntegers(NUM_ROWS, random), createList(NUM_ROWS, i -> createList(3 + random.nextInt(10), index -> randomIntegers(3 + random.nextInt(5), random)))), ImmutableList.of(// c[1] IS NULL
    ImmutableMap.of(1, ImmutableMap.of(new Subfield("c[1]"), IS_NULL)), // c[2] IS NOT NULL AND c[2][3] >= 0
    ImmutableMap.of(1, ImmutableMap.of(new Subfield("c[2]"), IS_NOT_NULL, new Subfield("c[2][3]"), nonNegative)), ImmutableMap.of(0, toSubfieldFilter(nonNegative), 1, ImmutableMap.of(new Subfield("c[1]"), IS_NULL))));
}

Example 34 with Subfield

use of com.facebook.presto.common.Subfield in project presto by prestodb.

the class TestListFilter method buildListFilter.

private static ListFilter buildListFilter(Map<Integer, TupleDomainFilter> filters, Integer[][] data) {
    Map<Subfield, TupleDomainFilter> subfieldFilters = filters.entrySet().stream().collect(toImmutableMap(entry -> toSubfield(entry.getKey()), Map.Entry::getValue));
    ListFilter filter = new ListFilter(makeStreamDescriptor(1), subfieldFilters);
    int[] lengths = Arrays.stream(data).mapToInt(v -> v.length).toArray();
    filter.populateElementFilters(data.length, null, lengths, Arrays.stream(lengths).sum());
    return filter;
}

Example 35 with Subfield

use of com.facebook.presto.common.Subfield in project presto by prestodb.

the class TestListFilter method buildListFilter.

private static ListFilter buildListFilter(Map<RowAndColumn, TupleDomainFilter> filters, Integer[][][] data) {
    Map<Subfield, TupleDomainFilter> subfieldFilters = filters.entrySet().stream().collect(toImmutableMap(entry -> toSubfield(entry.getKey()), Map.Entry::getValue));
    ListFilter filter = new ListFilter(makeStreamDescriptor(2), subfieldFilters);
    int[] lengths = Arrays.stream(data).mapToInt(v -> v.length).toArray();
    filter.populateElementFilters(data.length, null, lengths, Arrays.stream(lengths).sum());
    int[] nestedLenghts = Arrays.stream(data).flatMap(Arrays::stream).mapToInt(v -> v.length).toArray();
    ((ListFilter) filter.getChild()).populateElementFilters(Arrays.stream(lengths).sum(), null, nestedLenghts, Arrays.stream(nestedLenghts).sum());
    return filter;
}