Examples with LocalMemoryContext com.facebook.presto.memory.context.LocalMemoryContext used on opensource projects

Example 6 with LocalMemoryContext

use of com.facebook.presto.memory.context.LocalMemoryContext in project presto by prestodb.

the class TestTempStorageSingleStreamSpiller method assertSpill.

private void assertSpill(boolean compression, boolean encryption) throws Exception {
    File spillPath = new File(tempDirectory, UUID.randomUUID().toString());
    TempStorageSingleStreamSpillerFactory spillerFactory = new TempStorageSingleStreamSpillerFactory(new TestingTempStorageManager(spillPath.toString()), // executor won't be closed, because we don't call destroy() on the spiller factory
    executor, new BlockEncodingManager(), new SpillerStats(), compression, encryption, LocalTempStorage.NAME);
    LocalMemoryContext memoryContext = newSimpleAggregatedMemoryContext().newLocalMemoryContext("test");
    SingleStreamSpiller singleStreamSpiller = spillerFactory.create(TYPES, new TestingSpillContext(), memoryContext);
    assertTrue(singleStreamSpiller instanceof TempStorageSingleStreamSpiller);
    TempStorageSingleStreamSpiller spiller = (TempStorageSingleStreamSpiller) singleStreamSpiller;
    Page page = buildPage();
    // The spillers will reserve memory in their constructors
    int retainedSizeForEmptyDataSink = toIntExact(new OutputStreamDataSink(new DynamicSliceOutput(0)).getRetainedSizeInBytes());
    assertEquals(memoryContext.getBytes(), retainedSizeForEmptyDataSink);
    spiller.spill(page).get();
    spiller.spill(Iterators.forArray(page, page, page)).get();
    assertEquals(listFiles(spillPath.toPath()).size(), 1);
    // The spillers release their memory reservations when they are closed, therefore at this point
    // they will have non-zero memory reservation.
    // assertEquals(memoryContext.getBytes(), 0);
    Iterator<Page> spilledPagesIterator = spiller.getSpilledPages();
    assertEquals(memoryContext.getBytes(), retainedSizeForEmptyDataSink);
    ImmutableList<Page> spilledPages = ImmutableList.copyOf(spilledPagesIterator);
    // The spillers release their memory reservations when they are closed, therefore at this point
    // they will have non-zero memory reservation.
    // assertEquals(memoryContext.getBytes(), 0);
    assertEquals(4, spilledPages.size());
    for (int i = 0; i < 4; ++i) {
        PageAssertions.assertPageEquals(TYPES, page, spilledPages.get(i));
    }
    // Assert the spill codec flags match the expected configuration
    try (InputStream is = newInputStream(listFiles(spillPath.toPath()).get(0))) {
        Iterator<SerializedPage> serializedPages = PagesSerdeUtil.readSerializedPages(new InputStreamSliceInput(is));
        assertTrue(serializedPages.hasNext(), "at least one page should be successfully read back");
        byte markers = serializedPages.next().getPageCodecMarkers();
        assertEquals(PageCodecMarker.COMPRESSED.isSet(markers), compression);
        assertEquals(PageCodecMarker.ENCRYPTED.isSet(markers), encryption);
    }
    spiller.close();
    assertEquals(listFiles(spillPath.toPath()).size(), 0);
    assertEquals(memoryContext.getBytes(), 0);
}

Example 7 with LocalMemoryContext

use of com.facebook.presto.memory.context.LocalMemoryContext in project presto by prestodb.

the class PagesSpatialIndexSupplier method buildRTree.

private static Flatbush<GeometryWithPosition> buildRTree(AdaptiveLongBigArray addresses, int positionCount, List<List<Block>> channels, int geometryChannel, Optional<Integer> radiusChannel, Optional<Integer> partitionChannel, LocalMemoryContext localUserMemoryContext) {
    Operator relateOperator = OperatorFactoryLocal.getInstance().getOperator(Operator.Type.Relate);
    ObjectArrayList<GeometryWithPosition> geometries = new ObjectArrayList<>();
    long recordedSizeInBytes = localUserMemoryContext.getBytes();
    long addedSizeInBytes = 0;
    for (int position = 0; position < positionCount; position++) {
        long pageAddress = addresses.get(position);
        int blockIndex = decodeSliceIndex(pageAddress);
        int blockPosition = decodePosition(pageAddress);
        Block block = channels.get(geometryChannel).get(blockIndex);
        // TODO Consider pushing is-null and is-empty checks into a filter below the join
        if (block.isNull(blockPosition)) {
            continue;
        }
        Slice slice = block.getSlice(blockPosition, 0, block.getSliceLength(blockPosition));
        OGCGeometry ogcGeometry = deserialize(slice);
        verify(ogcGeometry != null);
        if (ogcGeometry.isEmpty()) {
            continue;
        }
        double radius = radiusChannel.map(channel -> DOUBLE.getDouble(channels.get(channel).get(blockIndex), blockPosition)).orElse(0.0);
        if (radius < 0) {
            continue;
        }
        if (!radiusChannel.isPresent()) {
            // If radiusChannel is supplied, this is a distance query, for which our acceleration won't help.
            accelerateGeometry(ogcGeometry, relateOperator);
        }
        int partition = -1;
        if (partitionChannel.isPresent()) {
            Block partitionBlock = channels.get(partitionChannel.get()).get(blockIndex);
            partition = toIntExact(INTEGER.getLong(partitionBlock, blockPosition));
        }
        GeometryWithPosition geometryWithPosition = new GeometryWithPosition(ogcGeometry, partition, position, radius);
        geometries.add(geometryWithPosition);
        addedSizeInBytes += geometryWithPosition.getEstimatedSizeInBytes();
        if (addedSizeInBytes >= MEMORY_USAGE_UPDATE_INCREMENT_BYTES) {
            localUserMemoryContext.setBytes(recordedSizeInBytes + addedSizeInBytes);
            recordedSizeInBytes += addedSizeInBytes;
            addedSizeInBytes = 0;
        }
    }
    return new Flatbush<>(geometries.toArray(new GeometryWithPosition[] {}));
}

Example 8 with LocalMemoryContext

use of com.facebook.presto.memory.context.LocalMemoryContext in project presto by prestodb.

the class TestMemoryRevokingScheduler method testMemoryPoolRevoking.

@Test
public void testMemoryPoolRevoking() throws Exception {
    QueryContext q1 = getOrCreateQueryContext(new QueryId("q1"), memoryPool);
    QueryContext q2 = getOrCreateQueryContext(new QueryId("q2"), memoryPool);
    SqlTask sqlTask1 = newSqlTask(q1.getQueryId(), memoryPool);
    SqlTask sqlTask2 = newSqlTask(q2.getQueryId(), memoryPool);
    TaskContext taskContext1 = getOrCreateTaskContext(sqlTask1);
    PipelineContext pipelineContext11 = taskContext1.addPipelineContext(0, false, false, false);
    DriverContext driverContext111 = pipelineContext11.addDriverContext();
    OperatorContext operatorContext1 = driverContext111.addOperatorContext(1, new PlanNodeId("na"), "na");
    OperatorContext operatorContext2 = driverContext111.addOperatorContext(2, new PlanNodeId("na"), "na");
    DriverContext driverContext112 = pipelineContext11.addDriverContext();
    OperatorContext operatorContext3 = driverContext112.addOperatorContext(3, new PlanNodeId("na"), "na");
    TaskContext taskContext2 = getOrCreateTaskContext(sqlTask2);
    PipelineContext pipelineContext21 = taskContext2.addPipelineContext(1, false, false, false);
    DriverContext driverContext211 = pipelineContext21.addDriverContext();
    OperatorContext operatorContext4 = driverContext211.addOperatorContext(4, new PlanNodeId("na"), "na");
    OperatorContext operatorContext5 = driverContext211.addOperatorContext(5, new PlanNodeId("na"), "na");
    List<SqlTask> tasks = ImmutableList.of(sqlTask1, sqlTask2);
    MemoryRevokingScheduler scheduler = new MemoryRevokingScheduler(singletonList(memoryPool), () -> tasks, queryContexts::get, 1.0, 1.0, ORDER_BY_CREATE_TIME, false);
    try {
        scheduler.start();
        allOperatorContexts = ImmutableSet.of(operatorContext1, operatorContext2, operatorContext3, operatorContext4, operatorContext5);
        assertMemoryRevokingNotRequested();
        assertEquals(10, memoryPool.getFreeBytes());
        scheduler.awaitAsynchronousCallbacksRun();
        assertMemoryRevokingNotRequested();
        LocalMemoryContext revocableMemory1 = operatorContext1.localRevocableMemoryContext();
        LocalMemoryContext revocableMemory3 = operatorContext3.localRevocableMemoryContext();
        LocalMemoryContext revocableMemory4 = operatorContext4.localRevocableMemoryContext();
        LocalMemoryContext revocableMemory5 = operatorContext5.localRevocableMemoryContext();
        revocableMemory1.setBytes(3);
        revocableMemory3.setBytes(6);
        assertEquals(1, memoryPool.getFreeBytes());
        scheduler.awaitAsynchronousCallbacksRun();
        // we are still good - no revoking needed
        assertMemoryRevokingNotRequested();
        revocableMemory4.setBytes(7);
        assertEquals(-6, memoryPool.getFreeBytes());
        scheduler.awaitAsynchronousCallbacksRun();
        // we need to revoke 3 and 6
        assertMemoryRevokingRequestedFor(operatorContext1, operatorContext3);
        // lets revoke some bytes
        revocableMemory1.setBytes(0);
        operatorContext1.resetMemoryRevokingRequested();
        scheduler.awaitAsynchronousCallbacksRun();
        assertMemoryRevokingRequestedFor(operatorContext3);
        assertEquals(-3, memoryPool.getFreeBytes());
        // and allocate some more
        revocableMemory5.setBytes(3);
        assertEquals(-6, memoryPool.getFreeBytes());
        scheduler.awaitAsynchronousCallbacksRun();
        // we are still good with just OC3 in process of revoking
        assertMemoryRevokingRequestedFor(operatorContext3);
        // and allocate some more
        revocableMemory5.setBytes(4);
        assertEquals(-7, memoryPool.getFreeBytes());
        scheduler.awaitAsynchronousCallbacksRun();
        // now we have to trigger revoking for OC4
        assertMemoryRevokingRequestedFor(operatorContext3, operatorContext4);
    } finally {
        scheduler.stop();
    }
}

Example 9 with LocalMemoryContext

use of com.facebook.presto.memory.context.LocalMemoryContext in project presto by prestodb.

the class TestMemoryRevokingScheduler method testRevokesPoolWhenFullBeforeQueryLimit.

@Test
public void testRevokesPoolWhenFullBeforeQueryLimit() throws Exception {
    QueryContext q1 = getOrCreateQueryContext(new QueryId("q1"), memoryPool);
    QueryContext q2 = getOrCreateQueryContext(new QueryId("q2"), memoryPool);
    SqlTask sqlTask1 = newSqlTask(q1.getQueryId(), memoryPool);
    SqlTask sqlTask2 = newSqlTask(q2.getQueryId(), memoryPool);
    TaskContext taskContext1 = getOrCreateTaskContext(sqlTask1);
    PipelineContext pipelineContext11 = taskContext1.addPipelineContext(0, false, false, false);
    DriverContext driverContext111 = pipelineContext11.addDriverContext();
    OperatorContext operatorContext1 = driverContext111.addOperatorContext(1, new PlanNodeId("na"), "na");
    OperatorContext operatorContext2 = driverContext111.addOperatorContext(2, new PlanNodeId("na"), "na");
    DriverContext driverContext112 = pipelineContext11.addDriverContext();
    OperatorContext operatorContext3 = driverContext112.addOperatorContext(3, new PlanNodeId("na"), "na");
    TaskContext taskContext2 = getOrCreateTaskContext(sqlTask2);
    PipelineContext pipelineContext21 = taskContext2.addPipelineContext(1, false, false, false);
    DriverContext driverContext211 = pipelineContext21.addDriverContext();
    OperatorContext operatorContext4 = driverContext211.addOperatorContext(4, new PlanNodeId("na"), "na");
    List<SqlTask> tasks = ImmutableList.of(sqlTask1, sqlTask2);
    MemoryRevokingScheduler scheduler = new MemoryRevokingScheduler(singletonList(memoryPool), () -> tasks, queryContexts::get, 1.0, 1.0, ORDER_BY_CREATE_TIME, true);
    try {
        scheduler.start();
        allOperatorContexts = ImmutableSet.of(operatorContext1, operatorContext2, operatorContext3, operatorContext4);
        assertMemoryRevokingNotRequested();
        assertEquals(10, memoryPool.getFreeBytes());
        scheduler.awaitAsynchronousCallbacksRun();
        assertMemoryRevokingNotRequested();
        LocalMemoryContext revocableMemory1 = operatorContext1.localRevocableMemoryContext();
        LocalMemoryContext revocableMemory3 = operatorContext3.localRevocableMemoryContext();
        LocalMemoryContext revocableMemory4 = operatorContext4.localRevocableMemoryContext();
        revocableMemory1.setBytes(3);
        revocableMemory3.setBytes(6);
        assertEquals(1, memoryPool.getFreeBytes());
        scheduler.awaitAsynchronousCallbacksRun();
        // we are still good - no revoking needed
        assertMemoryRevokingNotRequested();
        revocableMemory4.setBytes(7);
        assertEquals(-6, memoryPool.getFreeBytes());
        scheduler.awaitAsynchronousCallbacksRun();
        // we need to revoke 3 and 6
        assertMemoryRevokingRequestedFor(operatorContext1, operatorContext3);
    } finally {
        scheduler.stop();
    }
}

Example 10 with LocalMemoryContext

use of com.facebook.presto.memory.context.LocalMemoryContext in project presto by prestodb.

the class TestPageProcessor method testProjectLazyLoad.

@Test
public void testProjectLazyLoad() {
    PageProcessor pageProcessor = new PageProcessor(Optional.of(new SelectAllFilter()), ImmutableList.of(new PageProjectionWithOutputs(new LazyPagePageProjection(), new int[] { 0 })), OptionalInt.of(MAX_BATCH_SIZE));
    // if channel 1 is loaded, test will fail
    Page inputPage = new Page(createLongSequenceBlock(0, 100), new LazyBlock(100, lazyBlock -> {
        throw new AssertionError("Lazy block should not be loaded");
    }));
    LocalMemoryContext memoryContext = newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName());
    Iterator<Optional<Page>> output = pageProcessor.process(SESSION.getSqlFunctionProperties(), new DriverYieldSignal(), memoryContext, inputPage);
    List<Optional<Page>> outputPages = ImmutableList.copyOf(output);
    assertEquals(outputPages.size(), 1);
    assertPageEquals(ImmutableList.of(BIGINT), outputPages.get(0).orElse(null), new Page(createLongSequenceBlock(0, 100)));
}