Examples with PipelineContext io.trino.operator.PipelineContext used on opensource projects

Example 1 with PipelineContext

use of io.trino.operator.PipelineContext in project trino by trinodb.

the class SqlTask method createTaskStatus.

private TaskStatus createTaskStatus(TaskHolder taskHolder) {
    // Obtain task status version before building actual TaskStatus object.
    // This way any task updates won't be lost since all updates happen
    // before version number is increased.
    long versionNumber = taskStatusVersion.get();
    TaskState state = taskStateMachine.getState();
    List<ExecutionFailureInfo> failures = ImmutableList.of();
    if (state == FAILED) {
        failures = toFailures(taskStateMachine.getFailureCauses());
    }
    int queuedPartitionedDrivers = 0;
    long queuedPartitionedSplitsWeight = 0L;
    int runningPartitionedDrivers = 0;
    long runningPartitionedSplitsWeight = 0L;
    DataSize physicalWrittenDataSize = DataSize.ofBytes(0);
    DataSize userMemoryReservation = DataSize.ofBytes(0);
    DataSize revocableMemoryReservation = DataSize.ofBytes(0);
    // TODO: add a mechanism to avoid sending the whole completedDriverGroups set over the wire for every task status reply
    Set<Lifespan> completedDriverGroups = ImmutableSet.of();
    long fullGcCount = 0;
    Duration fullGcTime = new Duration(0, MILLISECONDS);
    long dynamicFiltersVersion = INITIAL_DYNAMIC_FILTERS_VERSION;
    if (taskHolder.getFinalTaskInfo() != null) {
        TaskInfo taskInfo = taskHolder.getFinalTaskInfo();
        TaskStats taskStats = taskInfo.getStats();
        queuedPartitionedDrivers = taskStats.getQueuedPartitionedDrivers();
        queuedPartitionedSplitsWeight = taskStats.getQueuedPartitionedSplitsWeight();
        runningPartitionedDrivers = taskStats.getRunningPartitionedDrivers();
        runningPartitionedSplitsWeight = taskStats.getRunningPartitionedSplitsWeight();
        physicalWrittenDataSize = taskStats.getPhysicalWrittenDataSize();
        userMemoryReservation = taskStats.getUserMemoryReservation();
        revocableMemoryReservation = taskStats.getRevocableMemoryReservation();
        fullGcCount = taskStats.getFullGcCount();
        fullGcTime = taskStats.getFullGcTime();
    } else if (taskHolder.getTaskExecution() != null) {
        long physicalWrittenBytes = 0;
        TaskContext taskContext = taskHolder.getTaskExecution().getTaskContext();
        for (PipelineContext pipelineContext : taskContext.getPipelineContexts()) {
            PipelineStatus pipelineStatus = pipelineContext.getPipelineStatus();
            queuedPartitionedDrivers += pipelineStatus.getQueuedPartitionedDrivers();
            queuedPartitionedSplitsWeight += pipelineStatus.getQueuedPartitionedSplitsWeight();
            runningPartitionedDrivers += pipelineStatus.getRunningPartitionedDrivers();
            runningPartitionedSplitsWeight += pipelineStatus.getRunningPartitionedSplitsWeight();
            physicalWrittenBytes += pipelineContext.getPhysicalWrittenDataSize();
        }
        physicalWrittenDataSize = succinctBytes(physicalWrittenBytes);
        userMemoryReservation = taskContext.getMemoryReservation();
        revocableMemoryReservation = taskContext.getRevocableMemoryReservation();
        completedDriverGroups = taskContext.getCompletedDriverGroups();
        fullGcCount = taskContext.getFullGcCount();
        fullGcTime = taskContext.getFullGcTime();
        dynamicFiltersVersion = taskContext.getDynamicFiltersVersion();
    }
    return new TaskStatus(taskStateMachine.getTaskId(), taskInstanceId, versionNumber, state, location, nodeId, completedDriverGroups, failures, queuedPartitionedDrivers, runningPartitionedDrivers, isOutputBufferOverutilized(), physicalWrittenDataSize, userMemoryReservation, revocableMemoryReservation, fullGcCount, fullGcTime, dynamicFiltersVersion, queuedPartitionedSplitsWeight, runningPartitionedSplitsWeight);
}

Example 2 with PipelineContext

use of io.trino.operator.PipelineContext in project trino by trinodb.

the class MemoryRevokingScheduler method requestRevoking.

private void requestRevoking(List<SqlTask> sqlTasks, long remainingBytesToRevoke) {
    VoidTraversingQueryContextVisitor<AtomicLong> visitor = new VoidTraversingQueryContextVisitor<>() {

        @Override
        public Void visitPipelineContext(PipelineContext pipelineContext, AtomicLong remainingBytesToRevoke) {
            if (remainingBytesToRevoke.get() <= 0) {
                // exit immediately if no work needs to be done
                return null;
            }
            return super.visitPipelineContext(pipelineContext, remainingBytesToRevoke);
        }

        @Override
        public Void visitOperatorContext(OperatorContext operatorContext, AtomicLong remainingBytesToRevoke) {
            if (remainingBytesToRevoke.get() > 0) {
                long revokedBytes = operatorContext.requestMemoryRevoking();
                if (revokedBytes > 0) {
                    remainingBytesToRevoke.addAndGet(-revokedBytes);
                    log.debug("requested revoking %s; remaining %s", revokedBytes, remainingBytesToRevoke.get());
                }
            }
            return null;
        }
    };
    AtomicLong remainingBytesToRevokeAtomic = new AtomicLong(remainingBytesToRevoke);
    for (SqlTask task : sqlTasks) {
        Optional<TaskContext> taskContext = task.getTaskContext();
        if (taskContext.isPresent()) {
            taskContext.get().accept(visitor, remainingBytesToRevokeAtomic);
            if (remainingBytesToRevokeAtomic.get() <= 0) {
                // No further revoking required
                return;
            }
        }
    }
}

Example 3 with PipelineContext

use of io.trino.operator.PipelineContext in project trino by trinodb.

the class TestSpatialJoinOperator method testDuplicateProbeFactory.

@Test(dataProvider = "testDuplicateProbeFactoryDataProvider")
public void testDuplicateProbeFactory(boolean createSecondaryOperators) throws Exception {
    TaskContext taskContext = createTaskContext();
    PipelineContext pipelineContext = taskContext.addPipelineContext(0, true, true, false);
    DriverContext probeDriver = pipelineContext.addDriverContext();
    DriverContext buildDriver = pipelineContext.addDriverContext();
    RowPagesBuilder buildPages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR, DOUBLE)).row(stPoint(0, 0), "0_0", 1.5);
    PagesSpatialIndexFactory pagesSpatialIndexFactory = buildIndex(buildDriver, (build, probe, r) -> build.distance(probe) <= r.getAsDouble(), Optional.of(2), Optional.empty(), buildPages);
    RowPagesBuilder probePages = rowPagesBuilder(ImmutableList.of(GEOMETRY, VARCHAR)).row(stPoint(0, 1), "0_1");
    OperatorFactory firstFactory = new SpatialJoinOperatorFactory(2, new PlanNodeId("test"), INNER, probePages.getTypes(), Ints.asList(1), 0, Optional.empty(), pagesSpatialIndexFactory);
    for (int i = 0; i < 3; i++) {
        DriverContext secondDriver = pipelineContext.addDriverContext();
        OperatorFactory secondFactory = firstFactory.duplicate();
        if (createSecondaryOperators) {
            try (Operator secondOperator = secondFactory.createOperator(secondDriver)) {
                assertEquals(toPages(secondOperator, emptyIterator()), ImmutableList.of());
            }
        }
        secondFactory.noMoreOperators();
    }
    MaterializedResult expected = resultBuilder(taskContext.getSession(), ImmutableList.of(VARCHAR, VARCHAR)).row("0_1", "0_0").build();
    assertOperatorEquals(firstFactory, probeDriver, probePages.build(), expected);
}

Example 4 with PipelineContext

use of io.trino.operator.PipelineContext in project trino by trinodb.

the class TestMemoryRevokingScheduler method testScheduleMemoryRevoking.

@Test
public void testScheduleMemoryRevoking() throws Exception {
    QueryContext q1 = getOrCreateQueryContext(new QueryId("q1"));
    QueryContext q2 = getOrCreateQueryContext(new QueryId("q2"));
    SqlTask sqlTask1 = newSqlTask(q1.getQueryId());
    SqlTask sqlTask2 = newSqlTask(q2.getQueryId());
    TaskContext taskContext1 = getOrCreateTaskContext(sqlTask1);
    PipelineContext pipelineContext11 = taskContext1.addPipelineContext(0, false, false, false);
    DriverContext driverContext111 = pipelineContext11.addDriverContext();
    OperatorContext operatorContext1 = driverContext111.addOperatorContext(1, new PlanNodeId("na"), "na");
    OperatorContext operatorContext2 = driverContext111.addOperatorContext(2, new PlanNodeId("na"), "na");
    DriverContext driverContext112 = pipelineContext11.addDriverContext();
    OperatorContext operatorContext3 = driverContext112.addOperatorContext(3, new PlanNodeId("na"), "na");
    TaskContext taskContext2 = getOrCreateTaskContext(sqlTask2);
    PipelineContext pipelineContext21 = taskContext2.addPipelineContext(1, false, false, false);
    DriverContext driverContext211 = pipelineContext21.addDriverContext();
    OperatorContext operatorContext4 = driverContext211.addOperatorContext(4, new PlanNodeId("na"), "na");
    OperatorContext operatorContext5 = driverContext211.addOperatorContext(5, new PlanNodeId("na"), "na");
    Collection<SqlTask> tasks = ImmutableList.of(sqlTask1, sqlTask2);
    MemoryRevokingScheduler scheduler = new MemoryRevokingScheduler(memoryPool, () -> tasks, executor, 1.0, 1.0);
    allOperatorContexts = ImmutableSet.of(operatorContext1, operatorContext2, operatorContext3, operatorContext4, operatorContext5);
    assertMemoryRevokingNotRequested();
    requestMemoryRevoking(scheduler);
    assertEquals(10, memoryPool.getFreeBytes());
    assertMemoryRevokingNotRequested();
    LocalMemoryContext revocableMemory1 = operatorContext1.localRevocableMemoryContext();
    LocalMemoryContext revocableMemory3 = operatorContext3.localRevocableMemoryContext();
    LocalMemoryContext revocableMemory4 = operatorContext4.localRevocableMemoryContext();
    LocalMemoryContext revocableMemory5 = operatorContext5.localRevocableMemoryContext();
    revocableMemory1.setBytes(3);
    revocableMemory3.setBytes(6);
    assertEquals(1, memoryPool.getFreeBytes());
    requestMemoryRevoking(scheduler);
    // we are still good - no revoking needed
    assertMemoryRevokingNotRequested();
    revocableMemory4.setBytes(7);
    assertEquals(-6, memoryPool.getFreeBytes());
    requestMemoryRevoking(scheduler);
    // we need to revoke 3 and 6
    assertMemoryRevokingRequestedFor(operatorContext1, operatorContext3);
    // yet another revoking request should not change anything
    requestMemoryRevoking(scheduler);
    assertMemoryRevokingRequestedFor(operatorContext1, operatorContext3);
    // lets revoke some bytes
    revocableMemory1.setBytes(0);
    operatorContext1.resetMemoryRevokingRequested();
    requestMemoryRevoking(scheduler);
    assertMemoryRevokingRequestedFor(operatorContext3);
    assertEquals(-3, memoryPool.getFreeBytes());
    // and allocate some more
    revocableMemory5.setBytes(3);
    assertEquals(-6, memoryPool.getFreeBytes());
    requestMemoryRevoking(scheduler);
    // we are still good with just OC3 in process of revoking
    assertMemoryRevokingRequestedFor(operatorContext3);
    // and allocate some more
    revocableMemory5.setBytes(4);
    assertEquals(-7, memoryPool.getFreeBytes());
    requestMemoryRevoking(scheduler);
    // no we have to trigger revoking for OC4
    assertMemoryRevokingRequestedFor(operatorContext3, operatorContext4);
}

Example 5 with PipelineContext

use of io.trino.operator.PipelineContext in project trino by trinodb.

the class TestMemoryRevokingScheduler method createContexts.

private OperatorContext createContexts(SqlTask sqlTask) {
    TaskContext taskContext = getOrCreateTaskContext(sqlTask);
    PipelineContext pipelineContext = taskContext.addPipelineContext(0, false, false, false);
    DriverContext driverContext = pipelineContext.addDriverContext();
    OperatorContext operatorContext = driverContext.addOperatorContext(1, new PlanNodeId("na"), "na");
    return operatorContext;
}