Examples with PagesSerde com.facebook.presto.spi.page.PagesSerde used on opensource projects

Example 6 with PagesSerde

use of com.facebook.presto.spi.page.PagesSerde in project presto by prestodb.

the class PageFileWriterFactory method createPagesSerdeForPageFile.

public static PagesSerde createPagesSerdeForPageFile(BlockEncodingSerde blockEncodingSerde, Optional<HiveCompressionCodec> compressionCodec) {
    if (!compressionCodec.isPresent()) {
        return new PagesSerde(blockEncodingSerde, Optional.empty(), Optional.empty(), Optional.empty());
    }
    PageCompressor pageCompressor = null;
    PageDecompressor pageDecompressor = null;
    switch(compressionCodec.get()) {
        case NONE:
            break;
        case SNAPPY:
            pageCompressor = new AirliftCompressorAdapter(new SnappyCompressor());
            pageDecompressor = new AirliftDecompressorAdapter(new SnappyDecompressor());
            break;
        case LZ4:
            pageCompressor = new AirliftCompressorAdapter(new Lz4Compressor());
            pageDecompressor = new AirliftDecompressorAdapter(new Lz4Decompressor());
            break;
        case GZIP:
            pageCompressor = new AirliftCompressorAdapter(new DeflateCompressor(OptionalInt.empty()));
            pageDecompressor = new AirliftDecompressorAdapter(new InflateDecompressor());
            break;
        case ZSTD:
            pageCompressor = new AirliftCompressorAdapter(new ZstdJniCompressor(OptionalInt.empty()));
            pageDecompressor = new AirliftDecompressorAdapter(new ZstdJniDecompressor());
            break;
        default:
            throw new PrestoException(GENERIC_INTERNAL_ERROR, format("%s compression is not supported for %s", compressionCodec.get().name(), PAGEFILE.getOutputFormat()));
    }
    return new PagesSerde(blockEncodingSerde, Optional.ofNullable(pageCompressor), Optional.ofNullable(pageDecompressor), Optional.empty());
}

Example 7 with PagesSerde

use of com.facebook.presto.spi.page.PagesSerde in project presto by prestodb.

the class GrpcUtils method toGrpcSerializedPage.

public static GrpcSerializedPage toGrpcSerializedPage(BlockEncodingSerde blockEncodingSerde, Page prestoPage) {
    PagesSerde pagesSerde = new PagesSerde(blockEncodingSerde, Optional.empty(), Optional.empty(), Optional.empty());
    SerializedPage serializedPage = pagesSerde.serialize(prestoPage);
    return GrpcSerializedPage.newBuilder().setSliceBytes(ByteString.copyFrom(serializedPage.getSlice().getBytes())).setPositionCount(serializedPage.getPositionCount()).setUncompressedSizeInBytes(serializedPage.getUncompressedSizeInBytes()).setPageCodecMarkers(ByteString.copyFrom(new byte[] { serializedPage.getPageCodecMarkers() })).setChecksum(serializedPage.getChecksum()).build();
}

Example 8 with PagesSerde

use of com.facebook.presto.spi.page.PagesSerde in project presto by prestodb.

the class PageFileWriterFactory method createFileWriter.

@Override
public Optional<HiveFileWriter> createFileWriter(Path path, List<String> inputColumnNames, StorageFormat storageFormat, Properties schema, JobConf configuration, ConnectorSession session, Optional<EncryptionInformation> encryptionInformation) {
    if (!storageFormat.getOutputFormat().equals(PAGEFILE.getOutputFormat())) {
        return Optional.empty();
    }
    HiveCompressionCodec compression = HiveCompressionCodec.valueOf(configuration.get(PAGE_FILE_COMPRESSION));
    if (!compression.isSupportedStorageFormat(PAGEFILE)) {
        throw new PrestoException(GENERIC_USER_ERROR, format("%s compression is not supported for %s", compression.name(), PAGEFILE.getOutputFormat()));
    }
    PagesSerde pagesSerde = createPagesSerdeForPageFile(blockEncodingSerde, Optional.of(compression));
    try {
        FileSystem fileSystem = hdfsEnvironment.getFileSystem(session.getUser(), path, configuration);
        DataSink dataSink = dataSinkFactory.createDataSink(session, fileSystem, path);
        Callable<Void> rollbackAction = () -> {
            fileSystem.delete(path, false);
            return null;
        };
        return Optional.of(new PageFileWriter(dataSink, pagesSerde, compression, getPageFileStripeMaxSize(session), rollbackAction));
    } catch (IOException e) {
        throw new PrestoException(HIVE_WRITER_OPEN_ERROR, "Error creating pagefile", e);
    }
}

Example 9 with PagesSerde

use of com.facebook.presto.spi.page.PagesSerde in project presto by prestodb.

the class TestSpillCipherPagesSerde method test.

@Test
public void test() {
    SpillCipher cipher = new AesSpillCipher();
    PagesSerde serde = new TestingPagesSerdeFactory().createPagesSerdeForSpill(Optional.of(cipher));
    List<Type> types = ImmutableList.of(VARCHAR);
    Page emptyPage = new Page(VARCHAR.createBlockBuilder(null, 0).build());
    assertPageEquals(types, serde.deserialize(serde.serialize(emptyPage)), emptyPage);
    BlockBuilder blockBuilder = VARCHAR.createBlockBuilder(null, 2);
    VARCHAR.writeString(blockBuilder, "hello");
    VARCHAR.writeString(blockBuilder, "world");
    Page helloWorldPage = new Page(blockBuilder.build());
    SerializedPage serialized = serde.serialize(helloWorldPage);
    assertPageEquals(types, serde.deserialize(serialized), helloWorldPage);
    assertTrue(ENCRYPTED.isSet(serialized.getPageCodecMarkers()), "page should be encrypted");
    cipher.destroy();
    assertFailure(() -> serde.serialize(helloWorldPage), "Spill cipher already destroyed");
    assertFailure(() -> serde.deserialize(serialized), "Spill cipher already destroyed");
}

Example 10 with PagesSerde

use of com.facebook.presto.spi.page.PagesSerde in project presto by prestodb.

the class TestPagesSerde method testRoundTrip.

@Test
public void testRoundTrip() {
    PagesSerde serde = new TestingPagesSerdeFactory().createPagesSerde();
    BlockBuilder expectedBlockBuilder = VARCHAR.createBlockBuilder(null, 5);
    VARCHAR.writeString(expectedBlockBuilder, "alice");
    VARCHAR.writeString(expectedBlockBuilder, "bob");
    VARCHAR.writeString(expectedBlockBuilder, "charlie");
    VARCHAR.writeString(expectedBlockBuilder, "dave");
    Block expectedBlock = expectedBlockBuilder.build();
    Page expectedPage = new Page(expectedBlock, expectedBlock, expectedBlock);
    DynamicSliceOutput sliceOutput = new DynamicSliceOutput(1024);
    writePages(serde, sliceOutput, expectedPage, expectedPage, expectedPage);
    List<Type> types = ImmutableList.of(VARCHAR, VARCHAR, VARCHAR);
    Iterator<Page> pageIterator = readPages(serde, sliceOutput.slice().getInput());
    assertPageEquals(types, pageIterator.next(), expectedPage);
    assertPageEquals(types, pageIterator.next(), expectedPage);
    assertPageEquals(types, pageIterator.next(), expectedPage);
    assertFalse(pageIterator.hasNext());
}