Examples with PrestoException com.facebook.presto.spi.PrestoException used on opensource projects

Example 51 with PrestoException

use of com.facebook.presto.spi.PrestoException in project presto by prestodb.

the class FunctionRegistry method resolveFunction.

public Signature resolveFunction(QualifiedName name, List<TypeSignatureProvider> parameterTypes) {
    Collection<SqlFunction> allCandidates = functions.get(name);
    List<SqlFunction> exactCandidates = allCandidates.stream().filter(function -> function.getSignature().getTypeVariableConstraints().isEmpty()).collect(Collectors.toList());
    Optional<Signature> match = matchFunctionExact(exactCandidates, parameterTypes);
    if (match.isPresent()) {
        return match.get();
    }
    List<SqlFunction> genericCandidates = allCandidates.stream().filter(function -> !function.getSignature().getTypeVariableConstraints().isEmpty()).collect(Collectors.toList());
    match = matchFunctionExact(genericCandidates, parameterTypes);
    if (match.isPresent()) {
        return match.get();
    }
    match = matchFunctionWithCoercion(allCandidates, parameterTypes);
    if (match.isPresent()) {
        return match.get();
    }
    List<String> expectedParameters = new ArrayList<>();
    for (SqlFunction function : allCandidates) {
        expectedParameters.add(format("%s(%s) %s", name, Joiner.on(", ").join(function.getSignature().getArgumentTypes()), Joiner.on(", ").join(function.getSignature().getTypeVariableConstraints())));
    }
    String parameters = Joiner.on(", ").join(parameterTypes);
    String message = format("Function %s not registered", name);
    if (!expectedParameters.isEmpty()) {
        String expected = Joiner.on(", ").join(expectedParameters);
        message = format("Unexpected parameters (%s) for function %s. Expected: %s", parameters, name, expected);
    }
    if (name.getSuffix().startsWith(MAGIC_LITERAL_FUNCTION_PREFIX)) {
        // extract type from function name
        String typeName = name.getSuffix().substring(MAGIC_LITERAL_FUNCTION_PREFIX.length());
        // lookup the type
        Type type = typeManager.getType(parseTypeSignature(typeName));
        requireNonNull(type, format("Type %s not registered", typeName));
        // verify we have one parameter of the proper type
        checkArgument(parameterTypes.size() == 1, "Expected one argument to literal function, but got %s", parameterTypes);
        Type parameterType = typeManager.getType(parameterTypes.get(0).getTypeSignature());
        requireNonNull(parameterType, format("Type %s not found", parameterTypes.get(0)));
        return getMagicLiteralFunctionSignature(type);
    }
    throw new PrestoException(FUNCTION_NOT_FOUND, message);
}

Example 52 with PrestoException

use of com.facebook.presto.spi.PrestoException in project presto by prestodb.

the class SimpleNodeSelector method computeAssignments.

@Override
public SplitPlacementResult computeAssignments(Set<Split> splits, List<RemoteTask> existingTasks) {
    Multimap<Node, Split> assignment = HashMultimap.create();
    NodeMap nodeMap = this.nodeMap.get().get();
    NodeAssignmentStats assignmentStats = new NodeAssignmentStats(nodeTaskMap, nodeMap, existingTasks);
    ResettableRandomizedIterator<Node> randomCandidates = randomizedNodes(nodeMap, includeCoordinator);
    Set<Node> blockedExactNodes = new HashSet<>();
    boolean splitWaitingForAnyNode = false;
    for (Split split : splits) {
        randomCandidates.reset();
        List<Node> candidateNodes;
        if (!split.isRemotelyAccessible()) {
            candidateNodes = selectExactNodes(nodeMap, split.getAddresses(), includeCoordinator);
        } else {
            candidateNodes = selectNodes(minCandidates, randomCandidates);
        }
        if (candidateNodes.isEmpty()) {
            log.debug("No nodes available to schedule %s. Available nodes %s", split, nodeMap.getNodesByHost().keys());
            throw new PrestoException(NO_NODES_AVAILABLE, "No nodes available to run query");
        }
        Node chosenNode = null;
        int min = Integer.MAX_VALUE;
        for (Node node : candidateNodes) {
            int totalSplitCount = assignmentStats.getTotalSplitCount(node);
            if (totalSplitCount < min && totalSplitCount < maxSplitsPerNode) {
                chosenNode = node;
                min = totalSplitCount;
            }
        }
        if (chosenNode == null) {
            // min is guaranteed to be MAX_VALUE at this line
            for (Node node : candidateNodes) {
                int totalSplitCount = assignmentStats.getQueuedSplitCountForStage(node);
                if (totalSplitCount < min && totalSplitCount < maxPendingSplitsPerTask) {
                    chosenNode = node;
                    min = totalSplitCount;
                }
            }
        }
        if (chosenNode != null) {
            assignment.put(chosenNode, split);
            assignmentStats.addAssignedSplit(chosenNode);
        } else {
            if (split.isRemotelyAccessible()) {
                splitWaitingForAnyNode = true;
            } else // Exact node set won't matter, if a split is waiting for any node
            if (!splitWaitingForAnyNode) {
                blockedExactNodes.addAll(candidateNodes);
            }
        }
    }
    ListenableFuture<?> blocked;
    if (splitWaitingForAnyNode) {
        blocked = toWhenHasSplitQueueSpaceFuture(existingTasks, calculateLowWatermark(maxPendingSplitsPerTask));
    } else {
        blocked = toWhenHasSplitQueueSpaceFuture(blockedExactNodes, existingTasks, calculateLowWatermark(maxPendingSplitsPerTask));
    }
    return new SplitPlacementResult(blocked, assignment);
}

Example 53 with PrestoException

use of com.facebook.presto.spi.PrestoException in project presto by prestodb.

the class SqlQueryScheduler method schedule.

private void schedule() {
    try (SetThreadName ignored = new SetThreadName("Query-%s", queryStateMachine.getQueryId())) {
        Set<StageId> completedStages = new HashSet<>();
        ExecutionSchedule executionSchedule = executionPolicy.createExecutionSchedule(stages.values());
        while (!executionSchedule.isFinished()) {
            List<ListenableFuture<?>> blockedStages = new ArrayList<>();
            for (SqlStageExecution stage : executionSchedule.getStagesToSchedule()) {
                stage.beginScheduling();
                // perform some scheduling work
                ScheduleResult result = stageSchedulers.get(stage.getStageId()).schedule();
                // modify parent and children based on the results of the scheduling
                if (result.isFinished()) {
                    stage.schedulingComplete();
                } else if (!result.getBlocked().isDone()) {
                    blockedStages.add(result.getBlocked());
                }
                stageLinkages.get(stage.getStageId()).processScheduleResults(stage.getState(), result.getNewTasks());
                schedulerStats.getSplitsScheduledPerIteration().add(result.getSplitsScheduled());
                if (result.getBlockedReason().isPresent()) {
                    switch(result.getBlockedReason().get()) {
                        case WAITING_FOR_SOURCE:
                            schedulerStats.getWaitingForSource().update(1);
                            break;
                        case SPLIT_QUEUES_FULL:
                            schedulerStats.getSplitQueuesFull().update(1);
                            break;
                        default:
                            throw new UnsupportedOperationException("Unknown blocked reason: " + result.getBlockedReason().get());
                    }
                }
            }
            // make sure to update stage linkage at least once per loop to catch async state changes (e.g., partial cancel)
            for (SqlStageExecution stage : stages.values()) {
                if (!completedStages.contains(stage.getStageId()) && stage.getState().isDone()) {
                    stageLinkages.get(stage.getStageId()).processScheduleResults(stage.getState(), ImmutableSet.of());
                    completedStages.add(stage.getStageId());
                }
            }
            // wait for a state change and then schedule again
            if (!blockedStages.isEmpty()) {
                try (TimeStat.BlockTimer timer = schedulerStats.getSleepTime().time()) {
                    tryGetFutureValue(whenAnyComplete(blockedStages), 1, SECONDS);
                }
                for (ListenableFuture<?> blockedStage : blockedStages) {
                    blockedStage.cancel(true);
                }
            }
        }
        for (SqlStageExecution stage : stages.values()) {
            StageState state = stage.getState();
            if (state != SCHEDULED && state != RUNNING && !state.isDone()) {
                throw new PrestoException(GENERIC_INTERNAL_ERROR, format("Scheduling is complete, but stage %s is in state %s", stage.getStageId(), state));
            }
        }
    } catch (Throwable t) {
        queryStateMachine.transitionToFailed(t);
        throw Throwables.propagate(t);
    } finally {
        RuntimeException closeError = new RuntimeException();
        for (StageScheduler scheduler : stageSchedulers.values()) {
            try {
                scheduler.close();
            } catch (Throwable t) {
                queryStateMachine.transitionToFailed(t);
                // Self-suppression not permitted
                if (closeError != t) {
                    closeError.addSuppressed(t);
                }
            }
        }
        if (closeError.getSuppressed().length > 0) {
            throw closeError;
        }
    }
}

Example 54 with PrestoException

use of com.facebook.presto.spi.PrestoException in project presto by prestodb.

the class ClusterMemoryManager method process.

public synchronized void process(Iterable<QueryExecution> queries) {
    if (!enabled) {
        return;
    }
    boolean outOfMemory = isClusterOutOfMemory();
    if (!outOfMemory) {
        lastTimeNotOutOfMemory = System.nanoTime();
    }
    boolean queryKilled = false;
    long totalBytes = 0;
    for (QueryExecution query : queries) {
        long bytes = query.getTotalMemoryReservation();
        DataSize sessionMaxQueryMemory = getQueryMaxMemory(query.getSession());
        long queryMemoryLimit = Math.min(maxQueryMemory.toBytes(), sessionMaxQueryMemory.toBytes());
        totalBytes += bytes;
        if (resourceOvercommit(query.getSession()) && outOfMemory) {
            // If a query has requested resource overcommit, only kill it if the cluster has run out of memory
            DataSize memory = succinctBytes(bytes);
            query.fail(new PrestoException(CLUSTER_OUT_OF_MEMORY, format("The cluster is out of memory and %s=true, so this query was killed. It was using %s of memory", RESOURCE_OVERCOMMIT, memory)));
            queryKilled = true;
        }
        if (!resourceOvercommit(query.getSession()) && bytes > queryMemoryLimit) {
            DataSize maxMemory = succinctBytes(queryMemoryLimit);
            query.fail(exceededGlobalLimit(maxMemory));
            queryKilled = true;
        }
    }
    clusterMemoryUsageBytes.set(totalBytes);
    if (killOnOutOfMemory) {
        boolean shouldKillQuery = nanosSince(lastTimeNotOutOfMemory).compareTo(killOnOutOfMemoryDelay) > 0 && outOfMemory;
        boolean lastKilledQueryIsGone = (lastKilledQuery == null);
        if (!lastKilledQueryIsGone) {
            ClusterMemoryPool generalPool = pools.get(GENERAL_POOL);
            if (generalPool != null) {
                lastKilledQueryIsGone = generalPool.getQueryMemoryReservations().containsKey(lastKilledQuery);
            }
        }
        if (shouldKillQuery && lastKilledQueryIsGone && !queryKilled) {
            // Kill the biggest query in the general pool
            QueryExecution biggestQuery = null;
            long maxMemory = -1;
            for (QueryExecution query : queries) {
                long bytesUsed = query.getTotalMemoryReservation();
                if (bytesUsed > maxMemory && query.getMemoryPool().getId().equals(GENERAL_POOL)) {
                    biggestQuery = query;
                    maxMemory = bytesUsed;
                }
            }
            if (biggestQuery != null) {
                biggestQuery.fail(new PrestoException(CLUSTER_OUT_OF_MEMORY, "The cluster is out of memory, and your query was killed. Please try again in a few minutes."));
                queriesKilledDueToOutOfMemory.incrementAndGet();
                lastKilledQuery = biggestQuery.getQueryId();
            }
        }
    }
    Map<MemoryPoolId, Integer> countByPool = new HashMap<>();
    for (QueryExecution query : queries) {
        MemoryPoolId id = query.getMemoryPool().getId();
        countByPool.put(id, countByPool.getOrDefault(id, 0) + 1);
    }
    updatePools(countByPool);
    updateNodes(updateAssignments(queries));
    // check if CPU usage is over limit
    for (QueryExecution query : queries) {
        Duration cpuTime = query.getTotalCpuTime();
        Duration sessionLimit = getQueryMaxCpuTime(query.getSession());
        Duration limit = maxQueryCpuTime.compareTo(sessionLimit) < 0 ? maxQueryCpuTime : sessionLimit;
        if (cpuTime.compareTo(limit) > 0) {
            query.fail(new ExceededCpuLimitException(limit));
        }
    }
}

Example 55 with PrestoException

use of com.facebook.presto.spi.PrestoException in project presto by prestodb.

the class SqlTaskManager method failAbandonedTasks.

public void failAbandonedTasks() {
    DateTime now = DateTime.now();
    DateTime oldestAllowedHeartbeat = now.minus(clientTimeout.toMillis());
    for (SqlTask sqlTask : tasks.asMap().values()) {
        try {
            TaskInfo taskInfo = sqlTask.getTaskInfo();
            TaskStatus taskStatus = taskInfo.getTaskStatus();
            if (taskStatus.getState().isDone()) {
                continue;
            }
            DateTime lastHeartbeat = taskInfo.getLastHeartbeat();
            if (lastHeartbeat != null && lastHeartbeat.isBefore(oldestAllowedHeartbeat)) {
                log.info("Failing abandoned task %s", taskStatus.getTaskId());
                sqlTask.failed(new PrestoException(ABANDONED_TASK, format("Task %s has not been accessed since %s: currentTime %s", taskStatus.getTaskId(), lastHeartbeat, now)));
            }
        } catch (RuntimeException e) {
            log.warn(e, "Error while inspecting age of task %s", sqlTask.getTaskId());
        }
    }
}