Search in sources :

Example 1 with PrestoSparkTaskOutput

use of com.facebook.presto.spark.classloader_interface.PrestoSparkTaskOutput in project presto by prestodb.

the class PrestoSparkTaskExecutorFactory method configureOutput.

@SuppressWarnings("unchecked")
private static <T extends PrestoSparkTaskOutput> Output<T> configureOutput(Class<T> outputType, BlockEncodingManager blockEncodingManager, OutputBufferMemoryManager memoryManager, DataSize targetAverageRowSize, Optional<OutputPartitioning> preDeterminedPartition, TempStorage tempStorage, TempDataOperationContext tempDataOperationContext, DataSize writeBufferSize) {
    if (outputType.equals(PrestoSparkMutableRow.class)) {
        PrestoSparkOutputBuffer<PrestoSparkRowBatch> outputBuffer = new PrestoSparkOutputBuffer<>(memoryManager);
        OutputFactory outputFactory = new PrestoSparkRowOutputFactory(outputBuffer, targetAverageRowSize, preDeterminedPartition);
        OutputSupplier<T> outputSupplier = (OutputSupplier<T>) new RowOutputSupplier(outputBuffer);
        return new Output<>(OutputBufferType.SPARK_ROW_OUTPUT_BUFFER, outputBuffer, outputFactory, outputSupplier);
    } else if (outputType.equals(PrestoSparkSerializedPage.class)) {
        PrestoSparkOutputBuffer<PrestoSparkBufferedSerializedPage> outputBuffer = new PrestoSparkOutputBuffer<>(memoryManager);
        OutputFactory outputFactory = new PrestoSparkPageOutputFactory(outputBuffer, blockEncodingManager);
        OutputSupplier<T> outputSupplier = (OutputSupplier<T>) new PageOutputSupplier(outputBuffer);
        return new Output<>(OutputBufferType.SPARK_PAGE_OUTPUT_BUFFER, outputBuffer, outputFactory, outputSupplier);
    } else if (outputType.equals(PrestoSparkStorageHandle.class)) {
        PrestoSparkOutputBuffer<PrestoSparkBufferedSerializedPage> outputBuffer = new PrestoSparkOutputBuffer<>(memoryManager);
        OutputFactory outputFactory = new PrestoSparkPageOutputFactory(outputBuffer, blockEncodingManager);
        OutputSupplier<T> outputSupplier = (OutputSupplier<T>) new DiskPageOutputSupplier(outputBuffer, tempStorage, tempDataOperationContext, writeBufferSize);
        return new Output<>(OutputBufferType.SPARK_DISK_PAGE_OUTPUT_BUFFER, outputBuffer, outputFactory, outputSupplier);
    } else {
        throw new IllegalArgumentException("Unexpected output type: " + outputType.getName());
    }
}
Also used : PrestoSparkRowOutputFactory(com.facebook.presto.spark.execution.PrestoSparkRowOutputOperator.PrestoSparkRowOutputFactory) PrestoSparkUtils.toPrestoSparkSerializedPage(com.facebook.presto.spark.util.PrestoSparkUtils.toPrestoSparkSerializedPage) PrestoSparkSerializedPage(com.facebook.presto.spark.classloader_interface.PrestoSparkSerializedPage) PrestoSparkPageOutputFactory(com.facebook.presto.spark.execution.PrestoSparkPageOutputOperator.PrestoSparkPageOutputFactory) PageDataOutput(com.facebook.presto.spi.page.PageDataOutput) PrestoSparkTaskOutput(com.facebook.presto.spark.classloader_interface.PrestoSparkTaskOutput) DataOutput(com.facebook.presto.common.io.DataOutput) PrestoSparkPageOutputFactory(com.facebook.presto.spark.execution.PrestoSparkPageOutputOperator.PrestoSparkPageOutputFactory) PrestoSparkRowOutputFactory(com.facebook.presto.spark.execution.PrestoSparkRowOutputOperator.PrestoSparkRowOutputFactory) OutputFactory(com.facebook.presto.operator.OutputFactory)

Example 2 with PrestoSparkTaskOutput

use of com.facebook.presto.spark.classloader_interface.PrestoSparkTaskOutput in project presto by prestodb.

the class PrestoSparkDiskPageInput method loadBroadcastTable.

private List<Page> loadBroadcastTable(List<PrestoSparkStorageHandle> broadcastTaskFilesInfo, TempStorage tempStorage, TempDataOperationContext tempDataOperationContext, UpdateMemory updateMemory) {
    try {
        CRC32 checksum = new CRC32();
        ImmutableList.Builder<Page> pages = ImmutableList.builder();
        List<PrestoSparkStorageHandle> broadcastTaskFilesInfoCopy = new ArrayList<>(broadcastTaskFilesInfo);
        shuffle(broadcastTaskFilesInfoCopy);
        for (PrestoSparkTaskOutput taskFileInfo : broadcastTaskFilesInfoCopy) {
            checksum.reset();
            PrestoSparkStorageHandle prestoSparkStorageHandle = (PrestoSparkStorageHandle) taskFileInfo;
            TempStorageHandle tempStorageHandle = tempStorage.deserialize(prestoSparkStorageHandle.getSerializedStorageHandle());
            log.info("Reading path: " + tempStorageHandle.toString());
            try (InputStream inputStream = tempStorage.open(tempDataOperationContext, tempStorageHandle);
                InputStreamSliceInput inputStreamSliceInput = new InputStreamSliceInput(inputStream)) {
                Iterator<SerializedPage> pagesIterator = readSerializedPages(inputStreamSliceInput);
                while (pagesIterator.hasNext()) {
                    SerializedPage serializedPage = pagesIterator.next();
                    checksum.update(serializedPage.getSlice().byteArray(), serializedPage.getSlice().byteArrayOffset(), serializedPage.getSlice().length());
                    Page deserializedPage = pagesSerde.deserialize(serializedPage);
                    pages.add(deserializedPage);
                    stagingBroadcastTableSizeInBytes += deserializedPage.getRetainedSizeInBytes();
                }
                updateMemory.update();
            }
            if (checksum.getValue() != prestoSparkStorageHandle.getChecksum()) {
                throw new PrestoException(STORAGE_ERROR, "Disk page checksum does not match. " + "Data seems to be corrupted on disk for file " + tempStorageHandle.toString());
            }
        }
        return pages.build();
    } catch (UncheckedIOException | IOException e) {
        throw new PrestoException(STORAGE_ERROR, "Unable to read data from disk: ", e);
    }
}
Also used : PrestoSparkTaskOutput(com.facebook.presto.spark.classloader_interface.PrestoSparkTaskOutput) CRC32(java.util.zip.CRC32) ImmutableList(com.google.common.collect.ImmutableList) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) InputStream(java.io.InputStream) ArrayList(java.util.ArrayList) Page(com.facebook.presto.common.Page) SerializedPage(com.facebook.presto.spi.page.SerializedPage) PrestoException(com.facebook.presto.spi.PrestoException) UncheckedIOException(java.io.UncheckedIOException) IOException(java.io.IOException) UncheckedIOException(java.io.UncheckedIOException) TempStorageHandle(com.facebook.presto.spi.storage.TempStorageHandle) PrestoSparkStorageHandle(com.facebook.presto.spark.classloader_interface.PrestoSparkStorageHandle) InputStreamSliceInput(io.airlift.slice.InputStreamSliceInput) SerializedPage(com.facebook.presto.spi.page.SerializedPage)

Aggregations

PrestoSparkTaskOutput (com.facebook.presto.spark.classloader_interface.PrestoSparkTaskOutput)2 Page (com.facebook.presto.common.Page)1 DataOutput (com.facebook.presto.common.io.DataOutput)1 OutputFactory (com.facebook.presto.operator.OutputFactory)1 PrestoSparkSerializedPage (com.facebook.presto.spark.classloader_interface.PrestoSparkSerializedPage)1 PrestoSparkStorageHandle (com.facebook.presto.spark.classloader_interface.PrestoSparkStorageHandle)1 PrestoSparkPageOutputFactory (com.facebook.presto.spark.execution.PrestoSparkPageOutputOperator.PrestoSparkPageOutputFactory)1 PrestoSparkRowOutputFactory (com.facebook.presto.spark.execution.PrestoSparkRowOutputOperator.PrestoSparkRowOutputFactory)1 PrestoSparkUtils.toPrestoSparkSerializedPage (com.facebook.presto.spark.util.PrestoSparkUtils.toPrestoSparkSerializedPage)1 PrestoException (com.facebook.presto.spi.PrestoException)1 PageDataOutput (com.facebook.presto.spi.page.PageDataOutput)1 SerializedPage (com.facebook.presto.spi.page.SerializedPage)1 TempStorageHandle (com.facebook.presto.spi.storage.TempStorageHandle)1 ImmutableList (com.google.common.collect.ImmutableList)1 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)1 InputStreamSliceInput (io.airlift.slice.InputStreamSliceInput)1 IOException (java.io.IOException)1 InputStream (java.io.InputStream)1 UncheckedIOException (java.io.UncheckedIOException)1 ArrayList (java.util.ArrayList)1