Examples with LocalMemoryContext io.trino.memory.context.LocalMemoryContext used on opensource projects

Example 11 with LocalMemoryContext

use of io.trino.memory.context.LocalMemoryContext in project trino by trinodb.

the class TestPageProcessor method testFilterNoColumns.

@Test
public void testFilterNoColumns() {
    PageProcessor pageProcessor = new PageProcessor(Optional.of(new TestingPageFilter(positionsRange(0, 50))), ImmutableList.of());
    Page inputPage = new Page(createLongSequenceBlock(0, 100));
    LocalMemoryContext memoryContext = newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName());
    Iterator<Optional<Page>> output = pageProcessor.process(SESSION, new DriverYieldSignal(), memoryContext, inputPage);
    assertEquals(memoryContext.getBytes(), 0);
    List<Optional<Page>> outputPages = ImmutableList.copyOf(output);
    assertEquals(outputPages.size(), 1);
    Page outputPage = outputPages.get(0).orElse(null);
    assertEquals(outputPage.getChannelCount(), 0);
    assertEquals(outputPage.getPositionCount(), 50);
}

Example 12 with LocalMemoryContext

use of io.trino.memory.context.LocalMemoryContext in project trino by trinodb.

the class TestFileSingleStreamSpiller method assertSpill.

private void assertSpill(boolean compression, boolean encryption) throws Exception {
    FileSingleStreamSpillerFactory spillerFactory = new FileSingleStreamSpillerFactory(// executor won't be closed, because we don't call destroy() on the spiller factory
    executor, new TestingBlockEncodingSerde(), new SpillerStats(), ImmutableList.of(spillPath.toPath()), 1.0, compression, encryption);
    LocalMemoryContext memoryContext = newSimpleAggregatedMemoryContext().newLocalMemoryContext("test");
    SingleStreamSpiller singleStreamSpiller = spillerFactory.create(TYPES, bytes -> {
    }, memoryContext);
    assertTrue(singleStreamSpiller instanceof FileSingleStreamSpiller);
    FileSingleStreamSpiller spiller = (FileSingleStreamSpiller) singleStreamSpiller;
    Page page = buildPage();
    // The spillers will reserve memory in their constructors
    assertEquals(memoryContext.getBytes(), 4096);
    spiller.spill(page).get();
    spiller.spill(Iterators.forArray(page, page, page)).get();
    assertEquals(listFiles(spillPath.toPath()).size(), 1);
    // Assert the spill codec flags match the expected configuration
    try (InputStream is = newInputStream(listFiles(spillPath.toPath()).get(0))) {
        Iterator<Slice> serializedPages = PagesSerdeUtil.readSerializedPages(is);
        assertTrue(serializedPages.hasNext(), "at least one page should be successfully read back");
        Slice serializedPage = serializedPages.next();
        assertEquals(isSerializedPageCompressed(serializedPage), compression);
        assertEquals(isSerializedPageEncrypted(serializedPage), encryption);
    }
    // The spillers release their memory reservations when they are closed, therefore at this point
    // they will have non-zero memory reservation.
    // assertEquals(memoryContext.getBytes(), 0);
    Iterator<Page> spilledPagesIterator = spiller.getSpilledPages();
    assertEquals(memoryContext.getBytes(), FileSingleStreamSpiller.BUFFER_SIZE);
    ImmutableList<Page> spilledPages = ImmutableList.copyOf(spilledPagesIterator);
    // The spillers release their memory reservations when they are closed, therefore at this point
    // they will have non-zero memory reservation.
    // assertEquals(memoryContext.getBytes(), 0);
    assertEquals(4, spilledPages.size());
    for (int i = 0; i < 4; ++i) {
        PageAssertions.assertPageEquals(TYPES, page, spilledPages.get(i));
    }
    spiller.close();
    assertEquals(listFiles(spillPath.toPath()).size(), 0);
    assertEquals(memoryContext.getBytes(), 0);
}

Example 13 with LocalMemoryContext

use of io.trino.memory.context.LocalMemoryContext in project trino by trinodb.

the class SpoolingExchangeOutputBuffer method getInfo.

@Override
public OutputBufferInfo getInfo() {
    BufferState state = stateMachine.getState();
    LocalMemoryContext memoryContext = getSystemMemoryContextOrNull();
    return new OutputBufferInfo("EXTERNAL", state, false, state.canAddPages(), memoryContext == null ? 0 : memoryContext.getBytes(), totalPagesAdded.get(), totalRowsAdded.get(), totalPagesAdded.get(), ImmutableList.of());
}

Example 14 with LocalMemoryContext

use of io.trino.memory.context.LocalMemoryContext in project trino by trinodb.

the class TestMemoryTracking method testStats.

@Test
public void testStats() {
    LocalMemoryContext userMemory = operatorContext.localUserMemoryContext();
    userMemory.setBytes(100_000_000);
    assertStats(operatorContext.getNestedOperatorStats(), driverContext.getDriverStats(), pipelineContext.getPipelineStats(), taskContext.getTaskStats(), 100_000_000, 0);
    // allocate more and check peak memory reservation
    userMemory.setBytes(600_000_000);
    assertStats(operatorContext.getNestedOperatorStats(), driverContext.getDriverStats(), pipelineContext.getPipelineStats(), taskContext.getTaskStats(), 600_000_000, 0);
    userMemory.setBytes(userMemory.getBytes() - 300_000_000);
    assertStats(operatorContext.getNestedOperatorStats(), driverContext.getDriverStats(), pipelineContext.getPipelineStats(), taskContext.getTaskStats(), 300_000_000, 0);
    userMemory.setBytes(userMemory.getBytes() - 300_000_000);
    assertStats(operatorContext.getNestedOperatorStats(), driverContext.getDriverStats(), pipelineContext.getPipelineStats(), taskContext.getTaskStats(), 0, 0);
    operatorContext.destroy();
    assertStats(operatorContext.getNestedOperatorStats(), driverContext.getDriverStats(), pipelineContext.getPipelineStats(), taskContext.getTaskStats(), 0, 0);
}

Example 15 with LocalMemoryContext

use of io.trino.memory.context.LocalMemoryContext in project trino by trinodb.

the class TestMemoryTracking method testDestroy.

@Test
public void testDestroy() {
    LocalMemoryContext newLocalUserMemoryContext = operatorContext.localUserMemoryContext();
    LocalMemoryContext newLocalRevocableMemoryContext = operatorContext.localRevocableMemoryContext();
    newLocalRevocableMemoryContext.setBytes(200_000);
    newLocalUserMemoryContext.setBytes(400_000);
    assertEquals(operatorContext.getOperatorMemoryContext().getUserMemory(), 400_000);
    operatorContext.destroy();
    assertOperatorMemoryAllocations(operatorContext.getOperatorMemoryContext(), 0, 0);
}