use of com.facebook.presto.spi.page.PageDataOutput in project presto by prestodb.
the class TempStorageSingleStreamSpiller method writePages.
private void writePages(Iterator<Page> pageIterator) {
checkState(writable, "Spilling no longer allowed. The spiller has been made non-writable on first read for subsequent reads to be consistent");
checkState(!committed, "Spilling no longer allowed. Spill file is already committed");
while (pageIterator.hasNext()) {
Page page = pageIterator.next();
spilledPagesInMemorySize += page.getSizeInBytes();
// page serialization requires page.getSizeInBytes() + Integer.BYTES to fit in an integer
splitPage(page, DEFAULT_MAX_PAGE_SIZE_IN_BYTES).stream().map(serde::serialize).forEach(serializedPage -> {
long pageSize = serializedPage.getSizeInBytes();
localSpillContext.updateBytes(pageSize);
spillerStats.addToTotalSpilledBytes(pageSize);
PageDataOutput pageDataOutput = new PageDataOutput(serializedPage);
bufferedBytes += toIntExact(pageDataOutput.size());
bufferedPages.add(pageDataOutput);
if (bufferedBytes > maxBufferSizeInBytes) {
flushBufferedPages();
}
});
}
memoryContext.setBytes(bufferedBytes + dataSink.getRetainedSizeInBytes());
}
use of com.facebook.presto.spi.page.PageDataOutput in project presto by prestodb.
the class SpoolingOutputBuffer method flush.
private synchronized void flush() {
List<DataOutput> dataOutputs = pages.stream().map(PageDataOutput::new).collect(toImmutableList());
// create a future that will hold the handle
ListenableFuture<TempStorageHandle> handleFuture = executor.submit(() -> {
TempDataSink dataSink = tempStorage.create(tempDataOperationContext);
dataSink.write(dataOutputs);
return dataSink.commit();
});
// store the handleFuture and file information
long bytes = totalInMemoryBytes.get();
int pageCount = pages.size();
HandleInfo handleInfo = new HandleInfo(closedOpen(currentMemorySequenceId.get(), currentMemorySequenceId.get() + pageCount), handleFuture, bytes, pageCount);
handleInfoQueue.add(handleInfo);
// update cutoff for file pages
currentMemorySequenceId.addAndGet(pageCount);
// clear the pages in memory
pages.clear();
// update info about storage
totalStorageBytesAdded.addAndGet(bytes);
totalStoragePagesAdded.addAndGet(pageCount);
totalInMemoryBytes.set(0);
}
use of com.facebook.presto.spi.page.PageDataOutput in project presto by prestodb.
the class PageWriter method write.
public void write(SerializedPage page) throws IOException {
PageDataOutput pageDataOutput = new PageDataOutput(page);
long writtenSize = pageDataOutput.size();
if (maxBufferedBytes - bufferedBytes < writtenSize) {
flushStripe();
}
bufferedPages.add(pageDataOutput);
bufferedBytes += writtenSize;
retainedBytes += page.getRetainedSizeInBytes();
}
use of com.facebook.presto.spi.page.PageDataOutput in project presto by prestodb.
the class TempStorageStandaloneSpiller method spill.
public SerializedStorageHandle spill(Iterator<Page> pageIterator) {
List<DataOutput> bufferedPages = new ArrayList<>();
int bufferedBytes = 0;
IOException ioException = null;
TempDataSink tempDataSink = null;
try {
tempDataSink = tempStorage.create(tempDataOperationContext);
while (pageIterator.hasNext()) {
Page page = pageIterator.next();
List<Page> splitPages = splitPage(page, DEFAULT_MAX_PAGE_SIZE_IN_BYTES);
for (Page splitPage : splitPages) {
SerializedPage serializedPage = serde.serialize(splitPage);
spillerStats.addToTotalSpilledBytes(serializedPage.getSizeInBytes());
PageDataOutput pageDataOutput = new PageDataOutput(serializedPage);
bufferedBytes += toIntExact(pageDataOutput.size());
bufferedPages.add(pageDataOutput);
if (bufferedBytes > maxBufferSizeInBytes) {
flushBufferedPages(tempDataSink, bufferedPages);
bufferedBytes = 0;
}
}
}
// Flush remaining buffered pages
if (!bufferedPages.isEmpty()) {
flushBufferedPages(tempDataSink, bufferedPages);
}
TempStorageHandle tempStorageHandle = tempDataSink.commit();
return new SerializedStorageHandle(tempStorage.serializeHandle(tempStorageHandle));
} catch (IOException e) {
ioException = e;
try {
if (tempDataSink != null) {
tempDataSink.rollback();
}
} catch (IOException exception) {
if (ioException != exception) {
ioException.addSuppressed(exception);
}
}
} finally {
try {
if (tempDataSink != null) {
tempDataSink.close();
}
} catch (IOException e) {
if (ioException == null) {
ioException = e;
} else if (ioException != e) {
ioException.addSuppressed(e);
}
throw new PrestoException(GENERIC_SPILL_FAILURE, "Failed to spill pages", ioException);
}
}
throw new PrestoException(GENERIC_SPILL_FAILURE, "Failed to spill pages", ioException);
}
Aggregations