Examples with TempStorageHandle com.facebook.presto.spi.storage.TempStorageHandle used on opensource projects

Example 1 with TempStorageHandle

use of com.facebook.presto.spi.storage.TempStorageHandle in project presto by prestodb.

the class PrestoSparkDiskPageInput method loadBroadcastTable.

private List<Page> loadBroadcastTable(List<PrestoSparkStorageHandle> broadcastTaskFilesInfo, TempStorage tempStorage, TempDataOperationContext tempDataOperationContext, UpdateMemory updateMemory) {
    try {
        CRC32 checksum = new CRC32();
        ImmutableList.Builder<Page> pages = ImmutableList.builder();
        List<PrestoSparkStorageHandle> broadcastTaskFilesInfoCopy = new ArrayList<>(broadcastTaskFilesInfo);
        shuffle(broadcastTaskFilesInfoCopy);
        for (PrestoSparkTaskOutput taskFileInfo : broadcastTaskFilesInfoCopy) {
            checksum.reset();
            PrestoSparkStorageHandle prestoSparkStorageHandle = (PrestoSparkStorageHandle) taskFileInfo;
            TempStorageHandle tempStorageHandle = tempStorage.deserialize(prestoSparkStorageHandle.getSerializedStorageHandle());
            log.info("Reading path: " + tempStorageHandle.toString());
            try (InputStream inputStream = tempStorage.open(tempDataOperationContext, tempStorageHandle);
                InputStreamSliceInput inputStreamSliceInput = new InputStreamSliceInput(inputStream)) {
                Iterator<SerializedPage> pagesIterator = readSerializedPages(inputStreamSliceInput);
                while (pagesIterator.hasNext()) {
                    SerializedPage serializedPage = pagesIterator.next();
                    checksum.update(serializedPage.getSlice().byteArray(), serializedPage.getSlice().byteArrayOffset(), serializedPage.getSlice().length());
                    Page deserializedPage = pagesSerde.deserialize(serializedPage);
                    pages.add(deserializedPage);
                    stagingBroadcastTableSizeInBytes += deserializedPage.getRetainedSizeInBytes();
                }
                updateMemory.update();
            }
            if (checksum.getValue() != prestoSparkStorageHandle.getChecksum()) {
                throw new PrestoException(STORAGE_ERROR, "Disk page checksum does not match. " + "Data seems to be corrupted on disk for file " + tempStorageHandle.toString());
            }
        }
        return pages.build();
    } catch (UncheckedIOException | IOException e) {
        throw new PrestoException(STORAGE_ERROR, "Unable to read data from disk: ", e);
    }
}

Example 2 with TempStorageHandle

use of com.facebook.presto.spi.storage.TempStorageHandle in project presto by prestodb.

the class TempStorageStandaloneSpiller method getSpilledPages.

public Iterator<Page> getSpilledPages(SerializedStorageHandle storageHandle) {
    try {
        TempStorageHandle tempStorageHandle = tempStorage.deserialize(storageHandle.getSerializedStorageHandle());
        InputStream inputStream = tempStorage.open(tempDataOperationContext, tempStorageHandle);
        Iterator<Page> deserializedPages = PagesSerdeUtil.readPages(serde, new InputStreamSliceInput(inputStream));
        return transform(deserializedPages, Page::compact);
    } catch (IOException e) {
        throw new PrestoException(GENERIC_SPILL_FAILURE, "Failed to read spilled pages", e);
    }
}

Example 3 with TempStorageHandle

use of com.facebook.presto.spi.storage.TempStorageHandle in project presto by prestodb.

the class SpoolingOutputBuffer method flush.

private synchronized void flush() {
    List<DataOutput> dataOutputs = pages.stream().map(PageDataOutput::new).collect(toImmutableList());
    // create a future that will hold the handle
    ListenableFuture<TempStorageHandle> handleFuture = executor.submit(() -> {
        TempDataSink dataSink = tempStorage.create(tempDataOperationContext);
        dataSink.write(dataOutputs);
        return dataSink.commit();
    });
    // store the handleFuture and file information
    long bytes = totalInMemoryBytes.get();
    int pageCount = pages.size();
    HandleInfo handleInfo = new HandleInfo(closedOpen(currentMemorySequenceId.get(), currentMemorySequenceId.get() + pageCount), handleFuture, bytes, pageCount);
    handleInfoQueue.add(handleInfo);
    // update cutoff for file pages
    currentMemorySequenceId.addAndGet(pageCount);
    // clear the pages in memory
    pages.clear();
    // update info about storage
    totalStorageBytesAdded.addAndGet(bytes);
    totalStoragePagesAdded.addAndGet(pageCount);
    totalInMemoryBytes.set(0);
}

Example 4 with TempStorageHandle

use of com.facebook.presto.spi.storage.TempStorageHandle in project presto by prestodb.

the class TempStorageStandaloneSpiller method spill.

public SerializedStorageHandle spill(Iterator<Page> pageIterator) {
    List<DataOutput> bufferedPages = new ArrayList<>();
    int bufferedBytes = 0;
    IOException ioException = null;
    TempDataSink tempDataSink = null;
    try {
        tempDataSink = tempStorage.create(tempDataOperationContext);
        while (pageIterator.hasNext()) {
            Page page = pageIterator.next();
            List<Page> splitPages = splitPage(page, DEFAULT_MAX_PAGE_SIZE_IN_BYTES);
            for (Page splitPage : splitPages) {
                SerializedPage serializedPage = serde.serialize(splitPage);
                spillerStats.addToTotalSpilledBytes(serializedPage.getSizeInBytes());
                PageDataOutput pageDataOutput = new PageDataOutput(serializedPage);
                bufferedBytes += toIntExact(pageDataOutput.size());
                bufferedPages.add(pageDataOutput);
                if (bufferedBytes > maxBufferSizeInBytes) {
                    flushBufferedPages(tempDataSink, bufferedPages);
                    bufferedBytes = 0;
                }
            }
        }
        // Flush remaining buffered pages
        if (!bufferedPages.isEmpty()) {
            flushBufferedPages(tempDataSink, bufferedPages);
        }
        TempStorageHandle tempStorageHandle = tempDataSink.commit();
        return new SerializedStorageHandle(tempStorage.serializeHandle(tempStorageHandle));
    } catch (IOException e) {
        ioException = e;
        try {
            if (tempDataSink != null) {
                tempDataSink.rollback();
            }
        } catch (IOException exception) {
            if (ioException != exception) {
                ioException.addSuppressed(exception);
            }
        }
    } finally {
        try {
            if (tempDataSink != null) {
                tempDataSink.close();
            }
        } catch (IOException e) {
            if (ioException == null) {
                ioException = e;
            } else if (ioException != e) {
                ioException.addSuppressed(e);
            }
            throw new PrestoException(GENERIC_SPILL_FAILURE, "Failed to spill pages", ioException);
        }
    }
    throw new PrestoException(GENERIC_SPILL_FAILURE, "Failed to spill pages", ioException);
}

Example 5 with TempStorageHandle

use of com.facebook.presto.spi.storage.TempStorageHandle in project presto by prestodb.

the class PrestoSparkStorageBasedBroadcastDependency method destroy.

@Override
public void destroy() {
    if (broadcastVariable == null) {
        return;
    }
    try {
        // Delete the files
        for (PrestoSparkStorageHandle diskPage : broadcastVariable.getValue()) {
            TempStorageHandle storageHandle = tempStorage.deserialize(diskPage.getSerializedStorageHandle());
            tempStorage.remove(tempDataOperationContext, storageHandle);
            log.info("Deleted broadcast spill file: " + storageHandle.toString());
        }
    } catch (IOException e) {
        throw new PrestoException(STORAGE_ERROR, "Unable to delete broadcast spill file", e);
    }
    // Destroy broadcast variable
    broadcastVariable.destroy();
}