Examples with INT com.facebook.presto.orc.metadata.OrcType.OrcTypeKind.INT used on opensource projects

Example 1 with INT

use of com.facebook.presto.orc.metadata.OrcType.OrcTypeKind.INT in project presto by prestodb.

the class TestDecryption method validateFileStatistics.

private static void validateFileStatistics(TempFile tempFile, Optional<DwrfWriterEncryption> dwrfWriterEncryption, Map<Integer, Slice> readerIntermediateKeys) throws IOException {
    OrcReader readerNoKeys = OrcTester.createCustomOrcReader(tempFile, DWRF, false, ImmutableMap.of());
    if (readerNoKeys.getFooter().getStripes().isEmpty()) {
        // files w/o stripes don't have stats
        assertEquals(readerNoKeys.getFooter().getFileStats().size(), 0);
        return;
    }
    if (dwrfWriterEncryption.isPresent()) {
        List<OrcType> types = readerNoKeys.getTypes();
        List<ColumnStatistics> fileStatsNoKey = readerNoKeys.getFooter().getFileStats();
        assertEquals(fileStatsNoKey.size(), types.size());
        Set<Integer> allEncryptedNodes = dwrfWriterEncryption.get().getWriterEncryptionGroups().stream().flatMap(group -> group.getNodes().stream()).flatMap(node -> collectNodeTree(types, node).stream()).collect(Collectors.toSet());
        for (Set<Integer> readerKeyNodes : Sets.powerSet(readerIntermediateKeys.keySet())) {
            Map<Integer, Slice> readerKeys = new HashMap<>();
            readerKeyNodes.forEach(node -> readerKeys.put(node, readerIntermediateKeys.get(node)));
            // nodes that are supposed to be decrypted by the reader
            Set<Integer> decryptedNodes = readerKeys.keySet().stream().flatMap(node -> collectNodeTree(types, node).stream()).collect(Collectors.toSet());
            // decryptedNodes should be a subset of encrypted nodes
            assertTrue(allEncryptedNodes.containsAll(decryptedNodes));
            OrcReader readerWithKeys = OrcTester.createCustomOrcReader(tempFile, DWRF, false, readerIntermediateKeys);
            List<ColumnStatistics> fileStatsWithKey = readerWithKeys.getFooter().getFileStats();
            assertEquals(fileStatsWithKey.size(), types.size());
            for (int node = 0; node < types.size(); node++) {
                ColumnStatistics statsWithKey = fileStatsWithKey.get(node);
                ColumnStatistics statsNoKey = fileStatsNoKey.get(node);
                OrcType type = types.get(node);
                // encrypted nodes should have no type info
                if (allEncryptedNodes.contains(node)) {
                    assertTrue(hasNoTypeStats(statsNoKey));
                } else {
                    assertStatsTypeMatch(statsNoKey, type);
                    assertStatsTypeMatch(statsWithKey, type);
                    assertEquals(statsNoKey, statsWithKey);
                }
                if (decryptedNodes.contains(node)) {
                    assertStatsTypeMatch(statsWithKey, type);
                }
            }
        }
    }
}

Example 2 with INT

use of com.facebook.presto.orc.metadata.OrcType.OrcTypeKind.INT in project presto by prestodb.

the class TestListFilter method assertPositionalFilter.

private void assertPositionalFilter(Map<RowAndColumn, TupleDomainFilter> filters, Integer[][][] data) {
    ListFilter listFilter = buildListFilter(filters, data);
    TestFilter2 testFilter = (i, j, value) -> Optional.ofNullable(filters.get(RowAndColumn.of(i + 1, j + 1))).map(filter -> value == null ? filter.testNull() : filter.testLong(value)).orElse(true);
    PositionalFilter positionalFilter = listFilter.getChild().getPositionalFilter();
    for (int i = 0; i < data.length; i++) {
        boolean expectedToPass = true;
        int passCount = 0;
        int failCount = 0;
        for (int j = 0; j < data[i].length; j++) {
            if (!expectedToPass) {
                failCount += data[i][j].length;
                continue;
            }
            for (int k = 0; k < data[i][j].length; k++) {
                Integer value = data[i][j][k];
                expectedToPass = testFilter.test(j, k, value);
                assertEquals(value == null ? positionalFilter.testNull() : positionalFilter.testLong(value), expectedToPass);
                if (!expectedToPass) {
                    assertEquals(positionalFilter.getPrecedingPositionsToFail(), passCount);
                    failCount = data[i][j].length - k - 1;
                    break;
                }
                passCount++;
            }
        }
        assertEquals(positionalFilter.getSucceedingPositionsToFail(), failCount);
    }
}

Example 3 with INT

use of com.facebook.presto.orc.metadata.OrcType.OrcTypeKind.INT in project presto by prestodb.

the class TestListFilter method assertPositionalFilter.

private void assertPositionalFilter(Map<Integer, TupleDomainFilter> filters, Integer[][] data) {
    ListFilter listFilter = buildListFilter(filters, data);
    TestFilter1 testFilter = (i, value) -> Optional.ofNullable(filters.get(i + 1)).map(filter -> value == null ? filter.testNull() : filter.testLong(value)).orElse(true);
    PositionalFilter positionalFilter = listFilter.getPositionalFilter();
    for (int i = 0; i < data.length; i++) {
        for (int j = 0; j < data[i].length; j++) {
            Integer value = data[i][j];
            boolean expectedToPass = testFilter.test(j, value);
            assertEquals(value == null ? positionalFilter.testNull() : positionalFilter.testLong(value), expectedToPass);
            if (!expectedToPass) {
                assertEquals(positionalFilter.getPrecedingPositionsToFail(), j);
                assertEquals(positionalFilter.getSucceedingPositionsToFail(), data[i].length - j - 1);
                break;
            }
        }
    }
}

Example 4 with INT

use of com.facebook.presto.orc.metadata.OrcType.OrcTypeKind.INT in project presto by prestodb.

the class TestListFilter method buildListFilter.

private static ListFilter buildListFilter(Map<Integer, TupleDomainFilter> filters, Integer[][] data) {
    Map<Subfield, TupleDomainFilter> subfieldFilters = filters.entrySet().stream().collect(toImmutableMap(entry -> toSubfield(entry.getKey()), Map.Entry::getValue));
    ListFilter filter = new ListFilter(makeStreamDescriptor(1), subfieldFilters);
    int[] lengths = Arrays.stream(data).mapToInt(v -> v.length).toArray();
    filter.populateElementFilters(data.length, null, lengths, Arrays.stream(lengths).sum());
    return filter;
}

Example 5 with INT

use of com.facebook.presto.orc.metadata.OrcType.OrcTypeKind.INT in project presto by prestodb.

the class TestListFilter method buildListFilter.

private static ListFilter buildListFilter(Map<RowAndColumn, TupleDomainFilter> filters, Integer[][][] data) {
    Map<Subfield, TupleDomainFilter> subfieldFilters = filters.entrySet().stream().collect(toImmutableMap(entry -> toSubfield(entry.getKey()), Map.Entry::getValue));
    ListFilter filter = new ListFilter(makeStreamDescriptor(2), subfieldFilters);
    int[] lengths = Arrays.stream(data).mapToInt(v -> v.length).toArray();
    filter.populateElementFilters(data.length, null, lengths, Arrays.stream(lengths).sum());
    int[] nestedLenghts = Arrays.stream(data).flatMap(Arrays::stream).mapToInt(v -> v.length).toArray();
    ((ListFilter) filter.getChild()).populateElementFilters(Arrays.stream(lengths).sum(), null, nestedLenghts, Arrays.stream(nestedLenghts).sum());
    return filter;
}