Examples with INTEGER com.facebook.presto.common.type.IntegerType.INTEGER used on opensource projects

Example 1 with INTEGER

use of com.facebook.presto.common.type.IntegerType.INTEGER in project presto by prestodb.

the class OrcTester method assertRoundTrip.

private void assertRoundTrip(List<Type> writeTypes, List<Type> readTypes, List<List<?>> writeValues, List<List<?>> readValues, boolean verifyWithHiveReader, List<OrcReaderSettings> settings) throws Exception {
    assertEquals(writeTypes.size(), readTypes.size());
    assertEquals(writeTypes.size(), writeValues.size());
    assertEquals(writeTypes.size(), readValues.size());
    OrcWriterStats stats = new OrcWriterStats();
    for (Format format : formats) {
        if (!readTypes.stream().allMatch(readType -> format.supportsType(readType))) {
            return;
        }
        OrcEncoding orcEncoding = format.getOrcEncoding();
        for (CompressionKind compression : compressions) {
            boolean hiveSupported = (compression != LZ4) && (compression != ZSTD);
            // write Hive, read Presto
            if (hiveSupported) {
                try (TempFile tempFile = new TempFile()) {
                    writeOrcColumnsHive(tempFile.getFile(), format, compression, writeTypes, writeValues);
                    assertFileContentsPresto(readTypes, tempFile, readValues, false, false, orcEncoding, format, true, useSelectiveOrcReader, settings, ImmutableMap.of());
                }
            }
            // write Presto, read Hive and Presto
            try (TempFile tempFile = new TempFile()) {
                writeOrcColumnsPresto(tempFile.getFile(), format, compression, Optional.empty(), writeTypes, writeValues, stats);
                if (verifyWithHiveReader && hiveSupported) {
                    assertFileContentsHive(readTypes, tempFile, format, readValues);
                }
                assertFileContentsPresto(readTypes, tempFile, readValues, false, false, orcEncoding, format, false, useSelectiveOrcReader, settings, ImmutableMap.of());
                if (skipBatchTestsEnabled) {
                    assertFileContentsPresto(readTypes, tempFile, readValues, true, false, orcEncoding, format, false, useSelectiveOrcReader, settings, ImmutableMap.of());
                }
                if (skipStripeTestsEnabled) {
                    assertFileContentsPresto(readTypes, tempFile, readValues, false, true, orcEncoding, format, false, useSelectiveOrcReader, settings, ImmutableMap.of());
                }
            }
            // write presto read presto
            if (dwrfEncryptionEnabled && format == DWRF) {
                try (TempFile tempFile = new TempFile()) {
                    DwrfWriterEncryption dwrfWriterEncryption = generateWriterEncryption();
                    writeOrcColumnsPresto(tempFile.getFile(), format, compression, Optional.of(dwrfWriterEncryption), writeTypes, writeValues, stats);
                    ImmutableMap.Builder<Integer, Slice> intermediateKeysBuilder = ImmutableMap.builder();
                    for (int i = 0; i < dwrfWriterEncryption.getWriterEncryptionGroups().size(); i++) {
                        for (Integer node : dwrfWriterEncryption.getWriterEncryptionGroups().get(i).getNodes()) {
                            intermediateKeysBuilder.put(node, dwrfWriterEncryption.getWriterEncryptionGroups().get(i).getIntermediateKeyMetadata());
                        }
                    }
                    Map<Integer, Slice> intermediateKeysMap = intermediateKeysBuilder.build();
                    assertFileContentsPresto(readTypes, tempFile, readValues, false, false, orcEncoding, format, false, useSelectiveOrcReader, settings, intermediateKeysMap);
                    if (skipBatchTestsEnabled) {
                        assertFileContentsPresto(readTypes, tempFile, readValues, true, false, orcEncoding, format, false, useSelectiveOrcReader, settings, intermediateKeysMap);
                    }
                    if (skipStripeTestsEnabled) {
                        assertFileContentsPresto(readTypes, tempFile, readValues, false, true, orcEncoding, format, false, useSelectiveOrcReader, settings, intermediateKeysMap);
                    }
                }
            }
        }
    }
    assertEquals(stats.getWriterSizeInBytes(), 0);
}

Example 2 with INTEGER

use of com.facebook.presto.common.type.IntegerType.INTEGER in project presto by prestodb.

the class TestSelectiveOrcReader method testArraysWithSubfieldPruning.

@Test
public void testArraysWithSubfieldPruning() throws Exception {
    tester.assertRoundTripWithSettings(arrayType(INTEGER), createList(NUM_ROWS, i -> ImmutableList.of(1, 2, 3, 4)), ImmutableList.of(OrcReaderSettings.builder().addRequiredSubfields(0, "c[1]").build(), OrcReaderSettings.builder().addRequiredSubfields(0, "c[1]", "c[2]").build(), OrcReaderSettings.builder().addRequiredSubfields(0, "c[2]").build()));
    Random random = new Random(0);
    tester.assertRoundTripWithSettings(arrayType(INTEGER), createList(NUM_ROWS, i -> ImmutableList.of(random.nextInt(10), random.nextInt(10), 3, 4)), ImmutableList.of(OrcReaderSettings.builder().addRequiredSubfields(0, "c[1]", "c[3]").setColumnFilters(ImmutableMap.of(0, ImmutableMap.of(new Subfield("c[1]"), BigintRange.of(0, 4, false)))).build(), OrcReaderSettings.builder().addRequiredSubfields(0, "c[2]", "c[3]").setColumnFilters(ImmutableMap.of(0, ImmutableMap.of(new Subfield("c[2]"), BigintRange.of(0, 4, false)))).build()));
    // arrays of arrays
    tester.assertRoundTripWithSettings(arrayType(arrayType(INTEGER)), createList(NUM_ROWS, i -> nCopies(1 + random.nextInt(5), ImmutableList.of(1, 2, 3))), ImmutableList.of(OrcReaderSettings.builder().addRequiredSubfields(0, "c[1][1]").build(), OrcReaderSettings.builder().addRequiredSubfields(0, "c[2][2]", "c[4][2]", "c[5][3]").build(), OrcReaderSettings.builder().addRequiredSubfields(0, "c[2][3]", "c[10][2]", "c[3][10]").build()));
    // arrays of maps
    tester.assertRoundTripWithSettings(arrayType(mapType(INTEGER, INTEGER)), createList(NUM_ROWS, i -> nCopies(5, ImmutableMap.of(1, 10, 2, 20))), ImmutableList.of(OrcReaderSettings.builder().addRequiredSubfields(0, "c[1][1]").build(), OrcReaderSettings.builder().addRequiredSubfields(0, "c[2][1]").build(), OrcReaderSettings.builder().addRequiredSubfields(0, "c[2][1]", "c[4][1]", "c[3][2]").build()));
}

Example 3 with INTEGER

use of com.facebook.presto.common.type.IntegerType.INTEGER in project presto by prestodb.

the class TestSelectiveOrcReader method testMaps.

@Test
public void testMaps() throws Exception {
    Random random = new Random(0);
    tester.testRoundTrip(mapType(INTEGER, INTEGER), createList(NUM_ROWS, i -> createMap(i)));
    // map column with no nulls
    tester.testRoundTripTypes(ImmutableList.of(INTEGER, mapType(INTEGER, INTEGER)), ImmutableList.of(createList(NUM_ROWS, i -> random.nextInt()), createList(NUM_ROWS, i -> createMap(i))), toSubfieldFilters(ImmutableMap.of(0, BigintRange.of(0, Integer.MAX_VALUE, false)), ImmutableMap.of(1, IS_NOT_NULL), ImmutableMap.of(1, IS_NULL)));
    // map column with nulls
    tester.testRoundTripTypes(ImmutableList.of(INTEGER, mapType(INTEGER, INTEGER)), ImmutableList.of(createList(NUM_ROWS, i -> random.nextInt()), createList(NUM_ROWS, i -> i % 5 == 0 ? null : createMap(i))), toSubfieldFilters(ImmutableMap.of(0, BigintRange.of(0, Integer.MAX_VALUE, false)), ImmutableMap.of(1, IS_NOT_NULL), ImmutableMap.of(1, IS_NULL), ImmutableMap.of(0, BigintRange.of(0, Integer.MAX_VALUE, false), 1, IS_NULL), ImmutableMap.of(0, BigintRange.of(0, Integer.MAX_VALUE, false), 1, IS_NOT_NULL)));
    // map column with filter, followed by another column with filter
    tester.testRoundTripTypes(ImmutableList.of(mapType(INTEGER, INTEGER), INTEGER), ImmutableList.of(createList(NUM_ROWS, i -> i % 5 == 0 ? null : createMap(i)), createList(NUM_ROWS, i -> random.nextInt())), toSubfieldFilters(ImmutableMap.of(0, IS_NULL, 1, BigintRange.of(0, Integer.MAX_VALUE, false)), ImmutableMap.of(0, IS_NOT_NULL, 1, BigintRange.of(0, Integer.MAX_VALUE, false))));
    // empty maps
    tester.testRoundTripTypes(ImmutableList.of(INTEGER, mapType(INTEGER, INTEGER)), ImmutableList.of(createList(NUM_ROWS, i -> random.nextInt()), Collections.nCopies(NUM_ROWS, ImmutableMap.of())), ImmutableList.of());
    // read selected positions from all nulls map column
    tester.testRoundTripTypes(ImmutableList.of(INTEGER, mapType(INTEGER, INTEGER)), ImmutableList.of(createList(NUM_ROWS, i -> random.nextInt(10)), createList(NUM_ROWS, i -> null)), toSubfieldFilters(ImmutableMap.of(0, BigintRange.of(0, 5, false))));
}

Example 4 with INTEGER

use of com.facebook.presto.common.type.IntegerType.INTEGER in project presto by prestodb.

the class TestMapFlatSelectiveStreamReader method runTest.

private <K, V> void runTest(String testOrcFileName, Type keyType, Type valueType, ExpectedValuesBuilder<K, V> expectedValuesBuilder) throws Exception {
    List<Map<K, V>> expectedValues = expectedValuesBuilder.build();
    Type mapType = mapType(keyType, valueType);
    OrcPredicate orcPredicate = createOrcPredicate(0, mapType, expectedValues, OrcTester.Format.DWRF, true);
    runTest(testOrcFileName, mapType, expectedValues, orcPredicate, Optional.empty(), ImmutableList.of());
    runTest(testOrcFileName, mapType, expectedValues.stream().filter(Objects::isNull).collect(toList()), orcPredicate, Optional.of(IS_NULL), ImmutableList.of());
    runTest(testOrcFileName, mapType, expectedValues.stream().filter(Objects::nonNull).collect(toList()), orcPredicate, Optional.of(IS_NOT_NULL), ImmutableList.of());
    if (keyType != VARBINARY) {
        // read only some keys
        List<K> keys = expectedValues.stream().filter(Objects::nonNull).flatMap(v -> v.keySet().stream()).distinct().collect(toImmutableList());
        if (!keys.isEmpty()) {
            List<K> requiredKeys = ImmutableList.of(keys.get(0));
            runTest(testOrcFileName, mapType, pruneMaps(expectedValues, requiredKeys), orcPredicate, Optional.empty(), toSubfields(keyType, requiredKeys));
            List<Integer> keyIndices = ImmutableList.of(1, 3, 7, 11);
            requiredKeys = keyIndices.stream().filter(k -> k < keys.size()).map(keys::get).collect(toList());
            runTest(testOrcFileName, mapType, pruneMaps(expectedValues, requiredKeys), orcPredicate, Optional.empty(), toSubfields(keyType, requiredKeys));
        }
    }
    // read only some rows
    List<Integer> ids = IntStream.range(0, expectedValues.size()).map(i -> i % 10).boxed().collect(toImmutableList());
    ImmutableList<Type> types = ImmutableList.of(mapType, INTEGER);
    Map<Integer, Map<Subfield, TupleDomainFilter>> filters = ImmutableMap.of(1, ImmutableMap.of(new Subfield("c"), toBigintValues(new long[] { 1, 5, 6 }, true)));
    assertFileContentsPresto(types, new File(getResource(testOrcFileName).getFile()), filterRows(types, ImmutableList.of(expectedValues, ids), filters), OrcEncoding.DWRF, OrcPredicate.TRUE, Optional.of(filters), ImmutableList.of(), ImmutableMap.of(), ImmutableMap.of());
    TestingFilterFunction filterFunction = new TestingFilterFunction(mapType);
    assertFileContentsPresto(types, new File(getResource(testOrcFileName).getFile()), filterFunction.filterRows(ImmutableList.of(expectedValues, ids)), OrcEncoding.DWRF, OrcPredicate.TRUE, Optional.empty(), ImmutableList.of(filterFunction), ImmutableMap.of(0, 0), ImmutableMap.of());
}

Example 5 with INTEGER

use of com.facebook.presto.common.type.IntegerType.INTEGER in project presto by prestodb.

the class TestMapFlatSelectiveStreamReader method runTest.

private <K, V> void runTest(String testOrcFileName, Type mapType, List<Map<K, V>> expectedValues, OrcPredicate orcPredicate, Optional<TupleDomainFilter> filter, List<Subfield> subfields) throws Exception {
    List<Type> types = ImmutableList.of(mapType);
    Optional<Map<Integer, Map<Subfield, TupleDomainFilter>>> filters = filter.map(f -> ImmutableMap.of(new Subfield("c"), f)).map(f -> ImmutableMap.of(0, f));
    assertFileContentsPresto(types, new File(getResource(testOrcFileName).getFile()), filters.map(f -> filterRows(types, ImmutableList.of(expectedValues), f)).orElse(ImmutableList.of(expectedValues)), OrcEncoding.DWRF, orcPredicate, filters, ImmutableList.of(), ImmutableMap.of(), ImmutableMap.of(0, subfields));
}