Examples with OperatorContext io.trino.operator.OperatorContext used on opensource projects

Example 1 with OperatorContext

use of io.trino.operator.OperatorContext in project trino by trinodb.

the class MemoryRevokingScheduler method requestRevoking.

private void requestRevoking(List<SqlTask> sqlTasks, long remainingBytesToRevoke) {
    VoidTraversingQueryContextVisitor<AtomicLong> visitor = new VoidTraversingQueryContextVisitor<>() {

        @Override
        public Void visitPipelineContext(PipelineContext pipelineContext, AtomicLong remainingBytesToRevoke) {
            if (remainingBytesToRevoke.get() <= 0) {
                // exit immediately if no work needs to be done
                return null;
            }
            return super.visitPipelineContext(pipelineContext, remainingBytesToRevoke);
        }

        @Override
        public Void visitOperatorContext(OperatorContext operatorContext, AtomicLong remainingBytesToRevoke) {
            if (remainingBytesToRevoke.get() > 0) {
                long revokedBytes = operatorContext.requestMemoryRevoking();
                if (revokedBytes > 0) {
                    remainingBytesToRevoke.addAndGet(-revokedBytes);
                    log.debug("requested revoking %s; remaining %s", revokedBytes, remainingBytesToRevoke.get());
                }
            }
            return null;
        }
    };
    AtomicLong remainingBytesToRevokeAtomic = new AtomicLong(remainingBytesToRevoke);
    for (SqlTask task : sqlTasks) {
        Optional<TaskContext> taskContext = task.getTaskContext();
        if (taskContext.isPresent()) {
            taskContext.get().accept(visitor, remainingBytesToRevokeAtomic);
            if (remainingBytesToRevokeAtomic.get() <= 0) {
                // No further revoking required
                return;
            }
        }
    }
}

Example 2 with OperatorContext

use of io.trino.operator.OperatorContext in project trino by trinodb.

the class LocalQueryRunner method executeInternal.

private MaterializedResultWithPlan executeInternal(Session session, @Language("SQL") String sql) {
    lock.readLock().lock();
    try (Closer closer = Closer.create()) {
        accessControl.checkCanExecuteQuery(session.getIdentity());
        AtomicReference<MaterializedResult.Builder> builder = new AtomicReference<>();
        PageConsumerOutputFactory outputFactory = new PageConsumerOutputFactory(types -> {
            builder.compareAndSet(null, MaterializedResult.resultBuilder(session, types));
            return builder.get()::page;
        });
        TaskContext taskContext = TestingTaskContext.builder(notificationExecutor, yieldExecutor, session).setMaxSpillSize(nodeSpillConfig.getMaxSpillPerNode()).setQueryMaxSpillSize(nodeSpillConfig.getQueryMaxSpillPerNode()).build();
        Plan plan = createPlan(session, sql, WarningCollector.NOOP);
        List<Driver> drivers = createDrivers(session, plan, outputFactory, taskContext);
        drivers.forEach(closer::register);
        boolean done = false;
        while (!done) {
            boolean processed = false;
            for (Driver driver : drivers) {
                if (alwaysRevokeMemory) {
                    driver.getDriverContext().getOperatorContexts().stream().filter(operatorContext -> operatorContext.getNestedOperatorStats().stream().mapToLong(stats -> stats.getRevocableMemoryReservation().toBytes()).sum() > 0).forEach(OperatorContext::requestMemoryRevoking);
                }
                if (!driver.isFinished()) {
                    driver.process();
                    processed = true;
                }
            }
            done = !processed;
        }
        verify(builder.get() != null, "Output operator was not created");
        return new MaterializedResultWithPlan(builder.get().build(), plan);
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    } finally {
        lock.readLock().unlock();
    }
}

Example 3 with OperatorContext

use of io.trino.operator.OperatorContext in project trino by trinodb.

the class TestMemoryRevokingScheduler method testScheduleMemoryRevoking.

@Test
public void testScheduleMemoryRevoking() throws Exception {
    QueryContext q1 = getOrCreateQueryContext(new QueryId("q1"));
    QueryContext q2 = getOrCreateQueryContext(new QueryId("q2"));
    SqlTask sqlTask1 = newSqlTask(q1.getQueryId());
    SqlTask sqlTask2 = newSqlTask(q2.getQueryId());
    TaskContext taskContext1 = getOrCreateTaskContext(sqlTask1);
    PipelineContext pipelineContext11 = taskContext1.addPipelineContext(0, false, false, false);
    DriverContext driverContext111 = pipelineContext11.addDriverContext();
    OperatorContext operatorContext1 = driverContext111.addOperatorContext(1, new PlanNodeId("na"), "na");
    OperatorContext operatorContext2 = driverContext111.addOperatorContext(2, new PlanNodeId("na"), "na");
    DriverContext driverContext112 = pipelineContext11.addDriverContext();
    OperatorContext operatorContext3 = driverContext112.addOperatorContext(3, new PlanNodeId("na"), "na");
    TaskContext taskContext2 = getOrCreateTaskContext(sqlTask2);
    PipelineContext pipelineContext21 = taskContext2.addPipelineContext(1, false, false, false);
    DriverContext driverContext211 = pipelineContext21.addDriverContext();
    OperatorContext operatorContext4 = driverContext211.addOperatorContext(4, new PlanNodeId("na"), "na");
    OperatorContext operatorContext5 = driverContext211.addOperatorContext(5, new PlanNodeId("na"), "na");
    Collection<SqlTask> tasks = ImmutableList.of(sqlTask1, sqlTask2);
    MemoryRevokingScheduler scheduler = new MemoryRevokingScheduler(memoryPool, () -> tasks, executor, 1.0, 1.0);
    allOperatorContexts = ImmutableSet.of(operatorContext1, operatorContext2, operatorContext3, operatorContext4, operatorContext5);
    assertMemoryRevokingNotRequested();
    requestMemoryRevoking(scheduler);
    assertEquals(10, memoryPool.getFreeBytes());
    assertMemoryRevokingNotRequested();
    LocalMemoryContext revocableMemory1 = operatorContext1.localRevocableMemoryContext();
    LocalMemoryContext revocableMemory3 = operatorContext3.localRevocableMemoryContext();
    LocalMemoryContext revocableMemory4 = operatorContext4.localRevocableMemoryContext();
    LocalMemoryContext revocableMemory5 = operatorContext5.localRevocableMemoryContext();
    revocableMemory1.setBytes(3);
    revocableMemory3.setBytes(6);
    assertEquals(1, memoryPool.getFreeBytes());
    requestMemoryRevoking(scheduler);
    // we are still good - no revoking needed
    assertMemoryRevokingNotRequested();
    revocableMemory4.setBytes(7);
    assertEquals(-6, memoryPool.getFreeBytes());
    requestMemoryRevoking(scheduler);
    // we need to revoke 3 and 6
    assertMemoryRevokingRequestedFor(operatorContext1, operatorContext3);
    // yet another revoking request should not change anything
    requestMemoryRevoking(scheduler);
    assertMemoryRevokingRequestedFor(operatorContext1, operatorContext3);
    // lets revoke some bytes
    revocableMemory1.setBytes(0);
    operatorContext1.resetMemoryRevokingRequested();
    requestMemoryRevoking(scheduler);
    assertMemoryRevokingRequestedFor(operatorContext3);
    assertEquals(-3, memoryPool.getFreeBytes());
    // and allocate some more
    revocableMemory5.setBytes(3);
    assertEquals(-6, memoryPool.getFreeBytes());
    requestMemoryRevoking(scheduler);
    // we are still good with just OC3 in process of revoking
    assertMemoryRevokingRequestedFor(operatorContext3);
    // and allocate some more
    revocableMemory5.setBytes(4);
    assertEquals(-7, memoryPool.getFreeBytes());
    requestMemoryRevoking(scheduler);
    // no we have to trigger revoking for OC4
    assertMemoryRevokingRequestedFor(operatorContext3, operatorContext4);
}

Example 4 with OperatorContext

use of io.trino.operator.OperatorContext in project trino by trinodb.

the class TestMemoryRevokingScheduler method testImmediateMemoryRevoking.

/**
 * Test that when a {@link MemoryPool} is over-allocated, revocable memory is revoked without delay (although asynchronously).
 */
@Test
public void testImmediateMemoryRevoking() throws Exception {
    // Given
    SqlTask sqlTask = newSqlTask(new QueryId("query"));
    OperatorContext operatorContext = createContexts(sqlTask);
    allOperatorContexts = ImmutableSet.of(operatorContext);
    List<SqlTask> tasks = ImmutableList.of(sqlTask);
    MemoryRevokingScheduler scheduler = new MemoryRevokingScheduler(memoryPool, () -> tasks, executor, 1.0, 1.0);
    // no periodic check initiated
    scheduler.registerPoolListeners();
    // When
    operatorContext.localRevocableMemoryContext().setBytes(12);
    awaitAsynchronousCallbacksRun();
    // Then
    assertMemoryRevokingRequestedFor(operatorContext);
}

Example 5 with OperatorContext

use of io.trino.operator.OperatorContext in project trino by trinodb.

the class TestMemoryRevokingScheduler method createContexts.

private OperatorContext createContexts(SqlTask sqlTask) {
    TaskContext taskContext = getOrCreateTaskContext(sqlTask);
    PipelineContext pipelineContext = taskContext.addPipelineContext(0, false, false, false);
    DriverContext driverContext = pipelineContext.addDriverContext();
    OperatorContext operatorContext = driverContext.addOperatorContext(1, new PlanNodeId("na"), "na");
    return operatorContext;
}