use of com.facebook.presto.spark.classloader_interface.PrestoSparkStorageHandle in project presto by prestodb.
the class PrestoSparkDiskPageInput method loadBroadcastTable.
private List<Page> loadBroadcastTable(List<PrestoSparkStorageHandle> broadcastTaskFilesInfo, TempStorage tempStorage, TempDataOperationContext tempDataOperationContext, UpdateMemory updateMemory) {
try {
CRC32 checksum = new CRC32();
ImmutableList.Builder<Page> pages = ImmutableList.builder();
List<PrestoSparkStorageHandle> broadcastTaskFilesInfoCopy = new ArrayList<>(broadcastTaskFilesInfo);
shuffle(broadcastTaskFilesInfoCopy);
for (PrestoSparkTaskOutput taskFileInfo : broadcastTaskFilesInfoCopy) {
checksum.reset();
PrestoSparkStorageHandle prestoSparkStorageHandle = (PrestoSparkStorageHandle) taskFileInfo;
TempStorageHandle tempStorageHandle = tempStorage.deserialize(prestoSparkStorageHandle.getSerializedStorageHandle());
log.info("Reading path: " + tempStorageHandle.toString());
try (InputStream inputStream = tempStorage.open(tempDataOperationContext, tempStorageHandle);
InputStreamSliceInput inputStreamSliceInput = new InputStreamSliceInput(inputStream)) {
Iterator<SerializedPage> pagesIterator = readSerializedPages(inputStreamSliceInput);
while (pagesIterator.hasNext()) {
SerializedPage serializedPage = pagesIterator.next();
checksum.update(serializedPage.getSlice().byteArray(), serializedPage.getSlice().byteArrayOffset(), serializedPage.getSlice().length());
Page deserializedPage = pagesSerde.deserialize(serializedPage);
pages.add(deserializedPage);
stagingBroadcastTableSizeInBytes += deserializedPage.getRetainedSizeInBytes();
}
updateMemory.update();
}
if (checksum.getValue() != prestoSparkStorageHandle.getChecksum()) {
throw new PrestoException(STORAGE_ERROR, "Disk page checksum does not match. " + "Data seems to be corrupted on disk for file " + tempStorageHandle.toString());
}
}
return pages.build();
} catch (UncheckedIOException | IOException e) {
throw new PrestoException(STORAGE_ERROR, "Unable to read data from disk: ", e);
}
}
use of com.facebook.presto.spark.classloader_interface.PrestoSparkStorageHandle in project presto by prestodb.
the class PrestoSparkStorageBasedBroadcastDependency method destroy.
@Override
public void destroy() {
if (broadcastVariable == null) {
return;
}
try {
// Delete the files
for (PrestoSparkStorageHandle diskPage : broadcastVariable.getValue()) {
TempStorageHandle storageHandle = tempStorage.deserialize(diskPage.getSerializedStorageHandle());
tempStorage.remove(tempDataOperationContext, storageHandle);
log.info("Deleted broadcast spill file: " + storageHandle.toString());
}
} catch (IOException e) {
throw new PrestoException(STORAGE_ERROR, "Unable to delete broadcast spill file", e);
}
// Destroy broadcast variable
broadcastVariable.destroy();
}
Aggregations