Examples with SerializedPage com.facebook.presto.spi.page.SerializedPage used on opensource projects

Example 16 with SerializedPage

use of com.facebook.presto.spi.page.SerializedPage in project presto by prestodb.

the class TempStorageStandaloneSpiller method spill.

public SerializedStorageHandle spill(Iterator<Page> pageIterator) {
    List<DataOutput> bufferedPages = new ArrayList<>();
    int bufferedBytes = 0;
    IOException ioException = null;
    TempDataSink tempDataSink = null;
    try {
        tempDataSink = tempStorage.create(tempDataOperationContext);
        while (pageIterator.hasNext()) {
            Page page = pageIterator.next();
            List<Page> splitPages = splitPage(page, DEFAULT_MAX_PAGE_SIZE_IN_BYTES);
            for (Page splitPage : splitPages) {
                SerializedPage serializedPage = serde.serialize(splitPage);
                spillerStats.addToTotalSpilledBytes(serializedPage.getSizeInBytes());
                PageDataOutput pageDataOutput = new PageDataOutput(serializedPage);
                bufferedBytes += toIntExact(pageDataOutput.size());
                bufferedPages.add(pageDataOutput);
                if (bufferedBytes > maxBufferSizeInBytes) {
                    flushBufferedPages(tempDataSink, bufferedPages);
                    bufferedBytes = 0;
                }
            }
        }
        // Flush remaining buffered pages
        if (!bufferedPages.isEmpty()) {
            flushBufferedPages(tempDataSink, bufferedPages);
        }
        TempStorageHandle tempStorageHandle = tempDataSink.commit();
        return new SerializedStorageHandle(tempStorage.serializeHandle(tempStorageHandle));
    } catch (IOException e) {
        ioException = e;
        try {
            if (tempDataSink != null) {
                tempDataSink.rollback();
            }
        } catch (IOException exception) {
            if (ioException != exception) {
                ioException.addSuppressed(exception);
            }
        }
    } finally {
        try {
            if (tempDataSink != null) {
                tempDataSink.close();
            }
        } catch (IOException e) {
            if (ioException == null) {
                ioException = e;
            } else if (ioException != e) {
                ioException.addSuppressed(e);
            }
            throw new PrestoException(GENERIC_SPILL_FAILURE, "Failed to spill pages", ioException);
        }
    }
    throw new PrestoException(GENERIC_SPILL_FAILURE, "Failed to spill pages", ioException);
}

Example 17 with SerializedPage

use of com.facebook.presto.spi.page.SerializedPage in project presto by prestodb.

the class TestFileSingleStreamSpiller method assertSpill.

private void assertSpill(boolean compression, boolean encryption) throws Exception {
    File spillPath = new File(tempDirectory, UUID.randomUUID().toString());
    FileSingleStreamSpillerFactory spillerFactory = new FileSingleStreamSpillerFactory(// executor won't be closed, because we don't call destroy() on the spiller factory
    executor, new BlockEncodingManager(), new SpillerStats(), ImmutableList.of(spillPath.toPath()), 1.0, compression, encryption);
    LocalMemoryContext memoryContext = newSimpleAggregatedMemoryContext().newLocalMemoryContext("test");
    SingleStreamSpiller singleStreamSpiller = spillerFactory.create(TYPES, new TestingSpillContext(), memoryContext);
    assertTrue(singleStreamSpiller instanceof FileSingleStreamSpiller);
    FileSingleStreamSpiller spiller = (FileSingleStreamSpiller) singleStreamSpiller;
    Page page = buildPage();
    // The spillers will reserve memory in their constructors
    assertEquals(memoryContext.getBytes(), 4096);
    spiller.spill(page).get();
    spiller.spill(Iterators.forArray(page, page, page)).get();
    assertEquals(listFiles(spillPath.toPath()).size(), 1);
    // Assert the spill codec flags match the expected configuration
    try (InputStream is = newInputStream(listFiles(spillPath.toPath()).get(0))) {
        Iterator<SerializedPage> serializedPages = PagesSerdeUtil.readSerializedPages(new InputStreamSliceInput(is));
        assertTrue(serializedPages.hasNext(), "at least one page should be successfully read back");
        byte markers = serializedPages.next().getPageCodecMarkers();
        assertEquals(PageCodecMarker.COMPRESSED.isSet(markers), compression);
        assertEquals(PageCodecMarker.ENCRYPTED.isSet(markers), encryption);
    }
    // The spillers release their memory reservations when they are closed, therefore at this point
    // they will have non-zero memory reservation.
    // assertEquals(memoryContext.getBytes(), 0);
    Iterator<Page> spilledPagesIterator = spiller.getSpilledPages();
    assertEquals(memoryContext.getBytes(), FileSingleStreamSpiller.BUFFER_SIZE);
    ImmutableList<Page> spilledPages = ImmutableList.copyOf(spilledPagesIterator);
    // The spillers release their memory reservations when they are closed, therefore at this point
    // they will have non-zero memory reservation.
    // assertEquals(memoryContext.getBytes(), 0);
    assertEquals(4, spilledPages.size());
    for (int i = 0; i < 4; ++i) {
        PageAssertions.assertPageEquals(TYPES, page, spilledPages.get(i));
    }
    spiller.close();
    assertEquals(listFiles(spillPath.toPath()).size(), 0);
    assertEquals(memoryContext.getBytes(), 0);
}

Example 18 with SerializedPage

use of com.facebook.presto.spi.page.SerializedPage in project presto by prestodb.

the class ExchangeClient method pollPage.

@Nullable
public SerializedPage pollPage() {
    checkState(!Thread.holdsLock(this), "Can not get next page while holding a lock on this");
    throwIfFailed();
    if (closed.get()) {
        return null;
    }
    SerializedPage page = pageBuffer.poll();
    if (page == null) {
        return null;
    }
    if (page == NO_MORE_PAGES) {
        // mark client closed; close() will add the end marker
        close();
        notifyBlockedCallers();
        // don't return end of stream marker
        return null;
    }
    synchronized (this) {
        if (!closed.get()) {
            bufferRetainedSizeInBytes -= page.getRetainedSizeInBytes();
            systemMemoryContext.setBytes(bufferRetainedSizeInBytes);
        }
        scheduleRequestIfNecessary();
    }
    return page;
}