Examples with TaskContext com.facebook.presto.operator.TaskContext used on opensource projects

Example 16 with TaskContext

use of com.facebook.presto.operator.TaskContext in project presto by prestodb.

the class IndexLoader method initializeStateIfNecessary.

private synchronized void initializeStateIfNecessary() {
    if (pipelineContext == null) {
        TaskContext taskContext = taskContextReference.get();
        checkState(taskContext != null, "Task context must be set before index can be built");
        pipelineContext = taskContext.addPipelineContext(indexBuildDriverFactoryProvider.getPipelineId(), true, true, false);
    }
    if (indexSnapshotLoader == null) {
        indexSnapshotLoader = new IndexSnapshotLoader(indexBuildDriverFactoryProvider, pipelineContext, indexSnapshotReference, lookupSourceInputChannels, keyTypes, keyOutputChannels, keyOutputHashChannel, expectedPositions, maxIndexMemorySize, pagesIndexFactory, joinCompiler, indexLoaderTimeout);
    }
}

Example 17 with TaskContext

use of com.facebook.presto.operator.TaskContext in project presto by prestodb.

the class AbstractOperatorBenchmark method runOnce.

@Override
protected Map<String, Long> runOnce() {
    Session session = testSessionBuilder().setSystemProperty("optimizer.optimize-hash-generation", "true").build();
    MemoryPool memoryPool = new MemoryPool(new MemoryPoolId("test"), new DataSize(1, GIGABYTE));
    SpillSpaceTracker spillSpaceTracker = new SpillSpaceTracker(new DataSize(1, GIGABYTE));
    TaskContext taskContext = new QueryContext(new QueryId("test"), new DataSize(256, MEGABYTE), new DataSize(512, MEGABYTE), new DataSize(256, MEGABYTE), new DataSize(1, GIGABYTE), memoryPool, new TestingGcMonitor(), localQueryRunner.getExecutor(), localQueryRunner.getScheduler(), new DataSize(256, MEGABYTE), spillSpaceTracker, listJsonCodec(TaskMemoryReservationSummary.class)).addTaskContext(new TaskStateMachine(new TaskId("query", 0, 0, 0), localQueryRunner.getExecutor()), session, Optional.empty(), false, false, false, false, false);
    CpuTimer cpuTimer = new CpuTimer();
    Map<String, Long> executionStats = execute(taskContext);
    CpuDuration executionTime = cpuTimer.elapsedTime();
    TaskStats taskStats = taskContext.getTaskStats();
    long inputRows = taskStats.getRawInputPositions();
    long inputBytes = taskStats.getRawInputDataSizeInBytes();
    long outputRows = taskStats.getOutputPositions();
    long outputBytes = taskStats.getOutputDataSizeInBytes();
    double inputMegaBytes = new DataSize(inputBytes, BYTE).getValue(MEGABYTE);
    return ImmutableMap.<String, Long>builder().putAll(executionStats).put("elapsed_millis", executionTime.getWall().toMillis()).put("input_rows_per_second", (long) (inputRows / executionTime.getWall().getValue(SECONDS))).put("output_rows_per_second", (long) (outputRows / executionTime.getWall().getValue(SECONDS))).put("input_megabytes", (long) inputMegaBytes).put("input_megabytes_per_second", (long) (inputMegaBytes / executionTime.getWall().getValue(SECONDS))).put("wall_nanos", executionTime.getWall().roundTo(NANOSECONDS)).put("cpu_nanos", executionTime.getCpu().roundTo(NANOSECONDS)).put("user_nanos", executionTime.getUser().roundTo(NANOSECONDS)).put("input_rows", inputRows).put("input_bytes", inputBytes).put("output_rows", outputRows).put("output_bytes", outputBytes).build();
}

Example 18 with TaskContext

use of com.facebook.presto.operator.TaskContext in project presto by prestodb.

the class SqlTaskExecutionFactory method create.

public SqlTaskExecution create(Session session, QueryContext queryContext, TaskStateMachine taskStateMachine, OutputBuffer outputBuffer, TaskExchangeClientManager taskExchangeClientManager, PlanFragment fragment, List<TaskSource> sources, TableWriteInfo tableWriteInfo) {
    TaskContext taskContext = queryContext.addTaskContext(taskStateMachine, session, // Plan has to be retained only if verbose memory exceeded errors are requested
    isVerboseExceededMemoryLimitErrorsEnabled(session) ? Optional.of(fragment.getRoot()) : Optional.empty(), perOperatorCpuTimerEnabled, cpuTimerEnabled, perOperatorAllocationTrackingEnabled, allocationTrackingEnabled, legacyLifespanCompletionCondition);
    LocalExecutionPlan localExecutionPlan;
    try (SetThreadName ignored = new SetThreadName("Task-%s", taskStateMachine.getTaskId())) {
        try {
            localExecutionPlan = planner.plan(taskContext, fragment.getRoot(), fragment.getPartitioningScheme(), fragment.getStageExecutionDescriptor(), fragment.getTableScanSchedulingOrder(), outputBuffer, new HttpRemoteSourceFactory(blockEncodingSerde, taskExchangeClientManager, orderingCompiler), tableWriteInfo);
        } catch (Throwable e) {
            // planning failed
            taskStateMachine.failed(e);
            throwIfUnchecked(e);
            throw new RuntimeException(e);
        }
    }
    return createSqlTaskExecution(taskStateMachine, taskContext, outputBuffer, sources, localExecutionPlan, taskExecutor, taskNotificationExecutor, splitMonitor);
}

Example 19 with TaskContext

use of com.facebook.presto.operator.TaskContext in project presto by prestodb.

the class TaskThresholdMemoryRevokingScheduler method revokeHighMemoryTasks.

private synchronized void revokeHighMemoryTasks() {
    if (checkPending.getAndSet(false)) {
        Collection<SqlTask> sqlTasks = requireNonNull(allTasksSupplier.get());
        for (SqlTask task : sqlTasks) {
            Optional<TaskContext> taskContext = task.getTaskContext();
            if (!taskContext.isPresent()) {
                continue;
            }
            long currentTaskRevocableMemory = taskContext.get().getTaskMemoryContext().getRevocableMemory();
            if (currentTaskRevocableMemory < maxRevocableMemoryPerTask) {
                continue;
            }
            AtomicLong remainingBytesToRevokeAtomic = new AtomicLong(currentTaskRevocableMemory - maxRevocableMemoryPerTask);
            taskContext.get().accept(new VoidTraversingQueryContextVisitor<AtomicLong>() {

                @Override
                public Void visitOperatorContext(OperatorContext operatorContext, AtomicLong remainingBytesToRevoke) {
                    if (remainingBytesToRevoke.get() > 0) {
                        long revokedBytes = operatorContext.requestMemoryRevoking();
                        if (revokedBytes > 0) {
                            remainingBytesToRevoke.addAndGet(-revokedBytes);
                            log.debug("taskId=%s: requested revoking %s; remaining %s", task.getTaskId(), revokedBytes, remainingBytesToRevoke.get());
                        }
                    }
                    return null;
                }
            }, remainingBytesToRevokeAtomic);
        }
    }
}

Example 20 with TaskContext

use of com.facebook.presto.operator.TaskContext in project presto by prestodb.

the class MemoryRevokingScheduler method requestRevoking.

private void requestRevoking(MemoryPoolId memoryPoolId, ArrayList<SqlTask> sqlTasks, long remainingBytesToRevoke) {
    VoidTraversingQueryContextVisitor<AtomicLong> visitor = new VoidTraversingQueryContextVisitor<AtomicLong>() {

        @Override
        public Void visitPipelineContext(PipelineContext pipelineContext, AtomicLong remainingBytesToRevoke) {
            if (remainingBytesToRevoke.get() <= 0) {
                // exit immediately if no work needs to be done
                return null;
            }
            return super.visitPipelineContext(pipelineContext, remainingBytesToRevoke);
        }

        @Override
        public Void visitOperatorContext(OperatorContext operatorContext, AtomicLong remainingBytesToRevoke) {
            if (remainingBytesToRevoke.get() > 0) {
                long revokedBytes = operatorContext.requestMemoryRevoking();
                if (revokedBytes > 0) {
                    remainingBytesToRevoke.addAndGet(-revokedBytes);
                    log.debug("memoryPool=%s: requested revoking %s; remaining %s", memoryPoolId, revokedBytes, remainingBytesToRevoke.get());
                }
            }
            return null;
        }
    };
    // Sort the tasks into their traversal order
    log.debug("Ordering by %s", spillingStrategy);
    sortTasksToTraversalOrder(sqlTasks, spillingStrategy);
    AtomicLong remainingBytesToRevokeAtomic = new AtomicLong(remainingBytesToRevoke);
    for (SqlTask task : sqlTasks) {
        Optional<TaskContext> taskContext = task.getTaskContext();
        if (taskContext.isPresent()) {
            taskContext.get().accept(visitor, remainingBytesToRevokeAtomic);
            if (remainingBytesToRevokeAtomic.get() <= 0) {
                // No further revoking required
                return;
            }
        }
    }
}