Examples with LocalMemoryContext com.facebook.presto.memory.context.LocalMemoryContext used on opensource projects

Example 26 with LocalMemoryContext

use of com.facebook.presto.memory.context.LocalMemoryContext in project presto by prestodb.

the class TestMemoryTracking method testTrySetZeroBytesFullPool.

@Test
public void testTrySetZeroBytesFullPool() {
    LocalMemoryContext localMemoryContext = operatorContext.localUserMemoryContext();
    // fill up the pool
    memoryPool.reserve(new QueryId("test_query"), "test", memoryPool.getFreeBytes());
    // try to reserve 0 bytes in the full pool
    assertTrue(localMemoryContext.trySetBytes(localMemoryContext.getBytes()));
}

Example 27 with LocalMemoryContext

use of com.facebook.presto.memory.context.LocalMemoryContext in project presto by prestodb.

the class TestMemoryTracking method testCumulativeUserMemoryEstimation.

@Test
public void testCumulativeUserMemoryEstimation() {
    LocalMemoryContext userMemory = operatorContext.localUserMemoryContext();
    long userMemoryBytes = 100_000_000;
    userMemory.setBytes(userMemoryBytes);
    long startTime = System.nanoTime();
    double cumulativeUserMemory = taskContext.getTaskStats().getCumulativeUserMemory();
    long endTime = System.nanoTime();
    double elapsedTimeInMillis = (endTime - startTime) / 1_000_000.0;
    long averageMemoryForLastPeriod = userMemoryBytes / 2;
    assertTrue(cumulativeUserMemory < elapsedTimeInMillis * averageMemoryForLastPeriod);
}

Example 28 with LocalMemoryContext

use of com.facebook.presto.memory.context.LocalMemoryContext in project presto by prestodb.

the class TestMemoryTracking method testLocalRevocableMemoryLimitExceeded.

@Test
public void testLocalRevocableMemoryLimitExceeded() {
    LocalMemoryContext revocableMemoryContext = operatorContext.localRevocableMemoryContext();
    revocableMemoryContext.setBytes(100);
    assertOperatorMemoryAllocations(operatorContext.getOperatorMemoryContext(), 0, 0, 100);
    revocableMemoryContext.setBytes(queryMaxRevocableMemory.toBytes());
    assertOperatorMemoryAllocations(operatorContext.getOperatorMemoryContext(), 0, 0, queryMaxRevocableMemory.toBytes());
    try {
        revocableMemoryContext.setBytes(queryMaxRevocableMemory.toBytes() + 1);
        fail("allocation should hit the per-node revocable memory limit");
    } catch (ExceededMemoryLimitException e) {
        assertEquals(e.getMessage(), format("Query exceeded per-node revocable memory limit of %1$s [Allocated: %1$s, Delta: 1B]", queryMaxRevocableMemory));
    }
}

Example 29 with LocalMemoryContext

use of com.facebook.presto.memory.context.LocalMemoryContext in project presto by prestodb.

the class MergeSortedPages method pageWithPositions.

private static WorkProcessor<PageWithPosition> pageWithPositions(WorkProcessor<Page> pages, AggregatedMemoryContext aggregatedMemoryContext) {
    return pages.flatMap(page -> {
        LocalMemoryContext memoryContext = aggregatedMemoryContext.newLocalMemoryContext(MergeSortedPages.class.getSimpleName());
        memoryContext.setBytes(page.getRetainedSizeInBytes());
        return WorkProcessor.create(new WorkProcessor.Process<PageWithPosition>() {

            int position;

            @Override
            public ProcessState<PageWithPosition> process() {
                if (position >= page.getPositionCount()) {
                    memoryContext.close();
                    return ProcessState.finished();
                }
                return ProcessState.ofResult(new PageWithPosition(page, position++));
            }
        });
    });
}

Example 30 with LocalMemoryContext

use of com.facebook.presto.memory.context.LocalMemoryContext in project presto by prestodb.

the class MergeSortedPages method buildPage.

private static WorkProcessor<Page> buildPage(WorkProcessor<PageWithPosition> pageWithPositions, List<Integer> outputChannels, List<Type> outputTypes, BiPredicate<PageBuilder, PageWithPosition> pageBreakPredicate, boolean updateMemoryAfterEveryPosition, AggregatedMemoryContext aggregatedMemoryContext, DriverYieldSignal yieldSignal) {
    LocalMemoryContext memoryContext = aggregatedMemoryContext.newLocalMemoryContext(MergeSortedPages.class.getSimpleName());
    PageBuilder pageBuilder = new PageBuilder(outputTypes);
    return pageWithPositions.yielding(yieldSignal::isSet).transform(pageWithPositionOptional -> {
        boolean finished = !pageWithPositionOptional.isPresent();
        if (finished && pageBuilder.isEmpty()) {
            memoryContext.close();
            return TransformationState.finished();
        }
        if (finished || pageBreakPredicate.test(pageBuilder, pageWithPositionOptional.get())) {
            if (!updateMemoryAfterEveryPosition) {
                // update memory usage just before producing page to cap from top
                memoryContext.setBytes(pageBuilder.getRetainedSizeInBytes());
            }
            Page page = pageBuilder.build();
            pageBuilder.reset();
            if (!finished) {
                pageWithPositionOptional.get().appendTo(pageBuilder, outputChannels, outputTypes);
            }
            if (updateMemoryAfterEveryPosition) {
                memoryContext.setBytes(pageBuilder.getRetainedSizeInBytes());
            }
            return TransformationState.ofResult(page, !finished);
        }
        pageWithPositionOptional.get().appendTo(pageBuilder, outputChannels, outputTypes);
        if (updateMemoryAfterEveryPosition) {
            memoryContext.setBytes(pageBuilder.getRetainedSizeInBytes());
        }
        return TransformationState.needsMoreData();
    });
}