Examples with ZSTD com.facebook.presto.orc.metadata.CompressionKind.ZSTD used on opensource projects

Example 1 with ZSTD

use of com.facebook.presto.orc.metadata.CompressionKind.ZSTD in project presto by prestodb.

the class OrcTester method assertRoundTrip.

private void assertRoundTrip(List<Type> writeTypes, List<Type> readTypes, List<List<?>> writeValues, List<List<?>> readValues, boolean verifyWithHiveReader, List<OrcReaderSettings> settings) throws Exception {
    assertEquals(writeTypes.size(), readTypes.size());
    assertEquals(writeTypes.size(), writeValues.size());
    assertEquals(writeTypes.size(), readValues.size());
    OrcWriterStats stats = new OrcWriterStats();
    for (Format format : formats) {
        if (!readTypes.stream().allMatch(readType -> format.supportsType(readType))) {
            return;
        }
        OrcEncoding orcEncoding = format.getOrcEncoding();
        for (CompressionKind compression : compressions) {
            boolean hiveSupported = (compression != LZ4) && (compression != ZSTD);
            // write Hive, read Presto
            if (hiveSupported) {
                try (TempFile tempFile = new TempFile()) {
                    writeOrcColumnsHive(tempFile.getFile(), format, compression, writeTypes, writeValues);
                    assertFileContentsPresto(readTypes, tempFile, readValues, false, false, orcEncoding, format, true, useSelectiveOrcReader, settings, ImmutableMap.of());
                }
            }
            // write Presto, read Hive and Presto
            try (TempFile tempFile = new TempFile()) {
                writeOrcColumnsPresto(tempFile.getFile(), format, compression, Optional.empty(), writeTypes, writeValues, stats);
                if (verifyWithHiveReader && hiveSupported) {
                    assertFileContentsHive(readTypes, tempFile, format, readValues);
                }
                assertFileContentsPresto(readTypes, tempFile, readValues, false, false, orcEncoding, format, false, useSelectiveOrcReader, settings, ImmutableMap.of());
                if (skipBatchTestsEnabled) {
                    assertFileContentsPresto(readTypes, tempFile, readValues, true, false, orcEncoding, format, false, useSelectiveOrcReader, settings, ImmutableMap.of());
                }
                if (skipStripeTestsEnabled) {
                    assertFileContentsPresto(readTypes, tempFile, readValues, false, true, orcEncoding, format, false, useSelectiveOrcReader, settings, ImmutableMap.of());
                }
            }
            // write presto read presto
            if (dwrfEncryptionEnabled && format == DWRF) {
                try (TempFile tempFile = new TempFile()) {
                    DwrfWriterEncryption dwrfWriterEncryption = generateWriterEncryption();
                    writeOrcColumnsPresto(tempFile.getFile(), format, compression, Optional.of(dwrfWriterEncryption), writeTypes, writeValues, stats);
                    ImmutableMap.Builder<Integer, Slice> intermediateKeysBuilder = ImmutableMap.builder();
                    for (int i = 0; i < dwrfWriterEncryption.getWriterEncryptionGroups().size(); i++) {
                        for (Integer node : dwrfWriterEncryption.getWriterEncryptionGroups().get(i).getNodes()) {
                            intermediateKeysBuilder.put(node, dwrfWriterEncryption.getWriterEncryptionGroups().get(i).getIntermediateKeyMetadata());
                        }
                    }
                    Map<Integer, Slice> intermediateKeysMap = intermediateKeysBuilder.build();
                    assertFileContentsPresto(readTypes, tempFile, readValues, false, false, orcEncoding, format, false, useSelectiveOrcReader, settings, intermediateKeysMap);
                    if (skipBatchTestsEnabled) {
                        assertFileContentsPresto(readTypes, tempFile, readValues, true, false, orcEncoding, format, false, useSelectiveOrcReader, settings, intermediateKeysMap);
                    }
                    if (skipStripeTestsEnabled) {
                        assertFileContentsPresto(readTypes, tempFile, readValues, false, true, orcEncoding, format, false, useSelectiveOrcReader, settings, intermediateKeysMap);
                    }
                }
            }
        }
    }
    assertEquals(stats.getWriterSizeInBytes(), 0);
}