Search in sources :

Example 1 with SerializedStorageHandle

use of com.facebook.presto.spi.storage.SerializedStorageHandle in project presto by prestodb.

the class TempStorageStandaloneSpiller method spill.

public SerializedStorageHandle spill(Iterator<Page> pageIterator) {
    List<DataOutput> bufferedPages = new ArrayList<>();
    int bufferedBytes = 0;
    IOException ioException = null;
    TempDataSink tempDataSink = null;
    try {
        tempDataSink = tempStorage.create(tempDataOperationContext);
        while (pageIterator.hasNext()) {
            Page page = pageIterator.next();
            List<Page> splitPages = splitPage(page, DEFAULT_MAX_PAGE_SIZE_IN_BYTES);
            for (Page splitPage : splitPages) {
                SerializedPage serializedPage = serde.serialize(splitPage);
                spillerStats.addToTotalSpilledBytes(serializedPage.getSizeInBytes());
                PageDataOutput pageDataOutput = new PageDataOutput(serializedPage);
                bufferedBytes += toIntExact(pageDataOutput.size());
                bufferedPages.add(pageDataOutput);
                if (bufferedBytes > maxBufferSizeInBytes) {
                    flushBufferedPages(tempDataSink, bufferedPages);
                    bufferedBytes = 0;
                }
            }
        }
        // Flush remaining buffered pages
        if (!bufferedPages.isEmpty()) {
            flushBufferedPages(tempDataSink, bufferedPages);
        }
        TempStorageHandle tempStorageHandle = tempDataSink.commit();
        return new SerializedStorageHandle(tempStorage.serializeHandle(tempStorageHandle));
    } catch (IOException e) {
        ioException = e;
        try {
            if (tempDataSink != null) {
                tempDataSink.rollback();
            }
        } catch (IOException exception) {
            if (ioException != exception) {
                ioException.addSuppressed(exception);
            }
        }
    } finally {
        try {
            if (tempDataSink != null) {
                tempDataSink.close();
            }
        } catch (IOException e) {
            if (ioException == null) {
                ioException = e;
            } else if (ioException != e) {
                ioException.addSuppressed(e);
            }
            throw new PrestoException(GENERIC_SPILL_FAILURE, "Failed to spill pages", ioException);
        }
    }
    throw new PrestoException(GENERIC_SPILL_FAILURE, "Failed to spill pages", ioException);
}
Also used : PageDataOutput(com.facebook.presto.spi.page.PageDataOutput) DataOutput(com.facebook.presto.common.io.DataOutput) TempDataSink(com.facebook.presto.spi.storage.TempDataSink) ArrayList(java.util.ArrayList) PageDataOutput(com.facebook.presto.spi.page.PageDataOutput) Page(com.facebook.presto.common.Page) SerializedPage(com.facebook.presto.spi.page.SerializedPage) PageSplitterUtil.splitPage(com.facebook.presto.execution.buffer.PageSplitterUtil.splitPage) PrestoException(com.facebook.presto.spi.PrestoException) IOException(java.io.IOException) UncheckedIOException(java.io.UncheckedIOException) SerializedStorageHandle(com.facebook.presto.spi.storage.SerializedStorageHandle) TempStorageHandle(com.facebook.presto.spi.storage.TempStorageHandle) SerializedPage(com.facebook.presto.spi.page.SerializedPage)

Aggregations

Page (com.facebook.presto.common.Page)1 DataOutput (com.facebook.presto.common.io.DataOutput)1 PageSplitterUtil.splitPage (com.facebook.presto.execution.buffer.PageSplitterUtil.splitPage)1 PrestoException (com.facebook.presto.spi.PrestoException)1 PageDataOutput (com.facebook.presto.spi.page.PageDataOutput)1 SerializedPage (com.facebook.presto.spi.page.SerializedPage)1 SerializedStorageHandle (com.facebook.presto.spi.storage.SerializedStorageHandle)1 TempDataSink (com.facebook.presto.spi.storage.TempDataSink)1 TempStorageHandle (com.facebook.presto.spi.storage.TempStorageHandle)1 IOException (java.io.IOException)1 UncheckedIOException (java.io.UncheckedIOException)1 ArrayList (java.util.ArrayList)1