use of com.facebook.presto.spark.classloader_interface.PrestoSparkTaskOutput in project presto by prestodb.
the class PrestoSparkTaskExecutorFactory method configureOutput.
@SuppressWarnings("unchecked")
private static <T extends PrestoSparkTaskOutput> Output<T> configureOutput(Class<T> outputType, BlockEncodingManager blockEncodingManager, OutputBufferMemoryManager memoryManager, DataSize targetAverageRowSize, Optional<OutputPartitioning> preDeterminedPartition, TempStorage tempStorage, TempDataOperationContext tempDataOperationContext, DataSize writeBufferSize) {
if (outputType.equals(PrestoSparkMutableRow.class)) {
PrestoSparkOutputBuffer<PrestoSparkRowBatch> outputBuffer = new PrestoSparkOutputBuffer<>(memoryManager);
OutputFactory outputFactory = new PrestoSparkRowOutputFactory(outputBuffer, targetAverageRowSize, preDeterminedPartition);
OutputSupplier<T> outputSupplier = (OutputSupplier<T>) new RowOutputSupplier(outputBuffer);
return new Output<>(OutputBufferType.SPARK_ROW_OUTPUT_BUFFER, outputBuffer, outputFactory, outputSupplier);
} else if (outputType.equals(PrestoSparkSerializedPage.class)) {
PrestoSparkOutputBuffer<PrestoSparkBufferedSerializedPage> outputBuffer = new PrestoSparkOutputBuffer<>(memoryManager);
OutputFactory outputFactory = new PrestoSparkPageOutputFactory(outputBuffer, blockEncodingManager);
OutputSupplier<T> outputSupplier = (OutputSupplier<T>) new PageOutputSupplier(outputBuffer);
return new Output<>(OutputBufferType.SPARK_PAGE_OUTPUT_BUFFER, outputBuffer, outputFactory, outputSupplier);
} else if (outputType.equals(PrestoSparkStorageHandle.class)) {
PrestoSparkOutputBuffer<PrestoSparkBufferedSerializedPage> outputBuffer = new PrestoSparkOutputBuffer<>(memoryManager);
OutputFactory outputFactory = new PrestoSparkPageOutputFactory(outputBuffer, blockEncodingManager);
OutputSupplier<T> outputSupplier = (OutputSupplier<T>) new DiskPageOutputSupplier(outputBuffer, tempStorage, tempDataOperationContext, writeBufferSize);
return new Output<>(OutputBufferType.SPARK_DISK_PAGE_OUTPUT_BUFFER, outputBuffer, outputFactory, outputSupplier);
} else {
throw new IllegalArgumentException("Unexpected output type: " + outputType.getName());
}
}
use of com.facebook.presto.spark.classloader_interface.PrestoSparkTaskOutput in project presto by prestodb.
the class PrestoSparkDiskPageInput method loadBroadcastTable.
private List<Page> loadBroadcastTable(List<PrestoSparkStorageHandle> broadcastTaskFilesInfo, TempStorage tempStorage, TempDataOperationContext tempDataOperationContext, UpdateMemory updateMemory) {
try {
CRC32 checksum = new CRC32();
ImmutableList.Builder<Page> pages = ImmutableList.builder();
List<PrestoSparkStorageHandle> broadcastTaskFilesInfoCopy = new ArrayList<>(broadcastTaskFilesInfo);
shuffle(broadcastTaskFilesInfoCopy);
for (PrestoSparkTaskOutput taskFileInfo : broadcastTaskFilesInfoCopy) {
checksum.reset();
PrestoSparkStorageHandle prestoSparkStorageHandle = (PrestoSparkStorageHandle) taskFileInfo;
TempStorageHandle tempStorageHandle = tempStorage.deserialize(prestoSparkStorageHandle.getSerializedStorageHandle());
log.info("Reading path: " + tempStorageHandle.toString());
try (InputStream inputStream = tempStorage.open(tempDataOperationContext, tempStorageHandle);
InputStreamSliceInput inputStreamSliceInput = new InputStreamSliceInput(inputStream)) {
Iterator<SerializedPage> pagesIterator = readSerializedPages(inputStreamSliceInput);
while (pagesIterator.hasNext()) {
SerializedPage serializedPage = pagesIterator.next();
checksum.update(serializedPage.getSlice().byteArray(), serializedPage.getSlice().byteArrayOffset(), serializedPage.getSlice().length());
Page deserializedPage = pagesSerde.deserialize(serializedPage);
pages.add(deserializedPage);
stagingBroadcastTableSizeInBytes += deserializedPage.getRetainedSizeInBytes();
}
updateMemory.update();
}
if (checksum.getValue() != prestoSparkStorageHandle.getChecksum()) {
throw new PrestoException(STORAGE_ERROR, "Disk page checksum does not match. " + "Data seems to be corrupted on disk for file " + tempStorageHandle.toString());
}
}
return pages.build();
} catch (UncheckedIOException | IOException e) {
throw new PrestoException(STORAGE_ERROR, "Unable to read data from disk: ", e);
}
}
Aggregations