Examples with QueryExecution io.trino.execution.QueryExecution used on opensource projects

Example 1 with QueryExecution

use of io.trino.execution.QueryExecution in project trino by trinodb.

the class LocalDispatchQueryFactory method createDispatchQuery.

@Override
public DispatchQuery createDispatchQuery(Session session, Optional<TransactionId> existingTransactionId, String query, PreparedQuery preparedQuery, Slug slug, ResourceGroupId resourceGroup) {
    WarningCollector warningCollector = warningCollectorFactory.create();
    QueryStateMachine stateMachine = QueryStateMachine.begin(existingTransactionId, query, preparedQuery.getPrepareSql(), session, locationFactory.createQueryLocation(session.getQueryId()), resourceGroup, isTransactionControlStatement(preparedQuery.getStatement()), transactionManager, accessControl, executor, metadata, warningCollector, getQueryType(preparedQuery.getStatement()));
    // It is important that `queryCreatedEvent` is called here. Moving it past the `executor.submit` below
    // can result in delivering query-created event after query analysis has already started.
    // That can result in misbehaviour of plugins called during analysis phase (e.g. access control auditing)
    // which depend on the contract that event was already delivered.
    // 
    // Note that for immediate and in-order delivery of query events we depend on synchronous nature of
    // QueryMonitor and EventListenerManager.
    queryMonitor.queryCreatedEvent(stateMachine.getBasicQueryInfo(Optional.empty()));
    ListenableFuture<QueryExecution> queryExecutionFuture = executor.submit(() -> {
        QueryExecutionFactory<?> queryExecutionFactory = executionFactories.get(preparedQuery.getStatement().getClass());
        if (queryExecutionFactory == null) {
            throw new TrinoException(NOT_SUPPORTED, "Unsupported statement type: " + preparedQuery.getStatement().getClass().getSimpleName());
        }
        try {
            return queryExecutionFactory.createQueryExecution(preparedQuery, stateMachine, slug, warningCollector);
        } catch (Throwable e) {
            if (e instanceof Error) {
                if (e instanceof StackOverflowError) {
                    log.error(e, "Unhandled StackOverFlowError; should be handled earlier; to investigate full stacktrace you may need to enable -XX:MaxJavaStackTraceDepth=0 JVM flag");
                } else {
                    log.error(e, "Unhandled Error");
                }
                // wrapping as RuntimeException to guard us from problem that code downstream which investigates queryExecutionFuture may not necessarily handle
                // Error subclass of Throwable well.
                RuntimeException wrappedError = new RuntimeException(e);
                stateMachine.transitionToFailed(wrappedError);
                throw wrappedError;
            }
            stateMachine.transitionToFailed(e);
            throw e;
        }
    });
    return new LocalDispatchQuery(stateMachine, queryExecutionFuture, queryMonitor, clusterSizeMonitor, executor, queryManager::createQuery);
}

Example 2 with QueryExecution

use of io.trino.execution.QueryExecution in project trino by trinodb.

the class ClusterMemoryManager method process.

public synchronized void process(Iterable<QueryExecution> runningQueries, Supplier<List<BasicQueryInfo>> allQueryInfoSupplier) {
    // TODO revocable memory reservations can also leak and may need to be detected in the future
    // We are only concerned about the leaks in the memory pool.
    memoryLeakDetector.checkForMemoryLeaks(allQueryInfoSupplier, pool.getQueryMemoryReservations());
    boolean outOfMemory = isClusterOutOfMemory();
    if (!outOfMemory) {
        lastTimeNotOutOfMemory = System.nanoTime();
    }
    boolean queryKilled = false;
    long totalUserMemoryBytes = 0L;
    long totalMemoryBytes = 0L;
    for (QueryExecution query : runningQueries) {
        boolean resourceOvercommit = resourceOvercommit(query.getSession());
        long userMemoryReservation = query.getUserMemoryReservation().toBytes();
        long totalMemoryReservation = query.getTotalMemoryReservation().toBytes();
        if (resourceOvercommit && outOfMemory) {
            // If a query has requested resource overcommit, only kill it if the cluster has run out of memory
            DataSize memory = succinctBytes(getQueryMemoryReservation(query));
            query.fail(new TrinoException(CLUSTER_OUT_OF_MEMORY, format("The cluster is out of memory and %s=true, so this query was killed. It was using %s of memory", RESOURCE_OVERCOMMIT, memory)));
            queryKilled = true;
        }
        if (!resourceOvercommit) {
            long userMemoryLimit = min(maxQueryMemory.toBytes(), getQueryMaxMemory(query.getSession()).toBytes());
            if (userMemoryReservation > userMemoryLimit) {
                query.fail(exceededGlobalUserLimit(succinctBytes(userMemoryLimit)));
                queryKilled = true;
            }
            long totalMemoryLimit = min(maxQueryTotalMemory.toBytes(), getQueryMaxTotalMemory(query.getSession()).toBytes());
            if (totalMemoryReservation > totalMemoryLimit) {
                query.fail(exceededGlobalTotalLimit(succinctBytes(totalMemoryLimit)));
                queryKilled = true;
            }
        }
        totalUserMemoryBytes += userMemoryReservation;
        totalMemoryBytes += totalMemoryReservation;
    }
    clusterUserMemoryReservation.set(totalUserMemoryBytes);
    clusterTotalMemoryReservation.set(totalMemoryBytes);
    if (!(lowMemoryKiller instanceof NoneLowMemoryKiller) && outOfMemory && !queryKilled && nanosSince(lastTimeNotOutOfMemory).compareTo(killOnOutOfMemoryDelay) > 0) {
        if (isLastKillTargetGone(runningQueries)) {
            callOomKiller(runningQueries);
        } else {
            log.debug("Last killed target is still not gone: %s", lastKillTarget);
        }
    }
    updateMemoryPool(Iterables.size(runningQueries));
    updateNodes();
}

Example 3 with QueryExecution

use of io.trino.execution.QueryExecution in project trino by trinodb.

the class ClusterMemoryManager method callOomKiller.

private synchronized void callOomKiller(Iterable<QueryExecution> runningQueries) {
    List<QueryMemoryInfo> queryMemoryInfoList = Streams.stream(runningQueries).map(this::createQueryMemoryInfo).collect(toImmutableList());
    List<MemoryInfo> nodeMemoryInfos = nodes.values().stream().map(RemoteNodeMemory::getInfo).filter(Optional::isPresent).map(Optional::get).collect(toImmutableList());
    Optional<KillTarget> killTarget = lowMemoryKiller.chooseQueryToKill(queryMemoryInfoList, nodeMemoryInfos);
    if (killTarget.isPresent()) {
        if (killTarget.get().isWholeQuery()) {
            QueryId queryId = killTarget.get().getQuery();
            log.debug("Low memory killer chose %s", queryId);
            Optional<QueryExecution> chosenQuery = findRunningQuery(runningQueries, killTarget.get().getQuery());
            if (chosenQuery.isPresent()) {
                // See comments in  isQueryGone for why chosenQuery might be absent.
                chosenQuery.get().fail(new TrinoException(CLUSTER_OUT_OF_MEMORY, "Query killed because the cluster is out of memory. Please try again in a few minutes."));
                queriesKilledDueToOutOfMemory.incrementAndGet();
                lastKillTarget = killTarget.get();
                logQueryKill(queryId, nodeMemoryInfos);
            }
        } else {
            Set<TaskId> tasks = killTarget.get().getTasks();
            log.debug("Low memory killer chose %s", tasks);
            ImmutableSet.Builder<TaskId> killedTasksBuilder = ImmutableSet.builder();
            for (TaskId task : tasks) {
                Optional<QueryExecution> runningQuery = findRunningQuery(runningQueries, task.getQueryId());
                if (runningQuery.isPresent()) {
                    runningQuery.get().failTask(task, new TrinoException(CLUSTER_OUT_OF_MEMORY, "Task killed because the cluster is out of memory."));
                    tasksKilledDueToOutOfMemory.incrementAndGet();
                    killedTasksBuilder.add(task);
                }
            }
            // only record tasks actually killed
            ImmutableSet<TaskId> killedTasks = killedTasksBuilder.build();
            if (!killedTasks.isEmpty()) {
                lastKillTarget = KillTarget.selectedTasks(killedTasks);
                logTasksKill(killedTasks, nodeMemoryInfos);
            }
        }
    }
}