Search in sources :

Example 1 with QueryMemoryInfo

use of com.facebook.presto.memory.LowMemoryKiller.QueryMemoryInfo in project presto by prestodb.

the class ClusterMemoryManager method callOomKiller.

private synchronized void callOomKiller(Iterable<QueryExecution> runningQueries) {
    List<QueryMemoryInfo> queryMemoryInfoList = Streams.stream(runningQueries).map(this::createQueryMemoryInfo).collect(toImmutableList());
    List<MemoryInfo> nodeMemoryInfos = nodes.values().stream().map(RemoteNodeMemory::getInfo).filter(Optional::isPresent).map(Optional::get).collect(toImmutableList());
    Optional<QueryId> chosenQueryId = lowMemoryKiller.chooseQueryToKill(queryMemoryInfoList, nodeMemoryInfos);
    if (chosenQueryId.isPresent()) {
        log.debug("Low memory killer chose %s", chosenQueryId.get());
        Optional<QueryExecution> chosenQuery = Streams.stream(runningQueries).filter(query -> chosenQueryId.get().equals(query.getQueryId())).collect(toOptional());
        if (chosenQuery.isPresent()) {
            // See comments in  isLastKilledQueryGone for why chosenQuery might be absent.
            chosenQuery.get().fail(new PrestoException(CLUSTER_OUT_OF_MEMORY, "Query killed because the cluster is out of memory. Please try again in a few minutes."));
            queriesKilledDueToOutOfMemory.incrementAndGet();
            lastKilledQuery = chosenQueryId.get();
            logQueryKill(chosenQueryId.get(), nodeMemoryInfos);
        }
    }
}
Also used : QueryMemoryInfo(com.facebook.presto.memory.LowMemoryKiller.QueryMemoryInfo) JsonCodec(com.facebook.airlift.json.JsonCodec) SYSTEM(com.facebook.presto.execution.QueryLimit.Source.SYSTEM) QUERY(com.facebook.presto.execution.QueryLimit.Source.QUERY) Duration(io.airlift.units.Duration) ACTIVE(com.facebook.presto.spi.NodeState.ACTIVE) RESERVED_POOL(com.facebook.presto.memory.LocalMemoryManager.RESERVED_POOL) PreDestroy(javax.annotation.PreDestroy) Sets.difference(com.google.common.collect.Sets.difference) InternalNodeManager(com.facebook.presto.metadata.InternalNodeManager) QueryMemoryInfo(com.facebook.presto.memory.LowMemoryKiller.QueryMemoryInfo) QueryLimit.getMinimum(com.facebook.presto.execution.QueryLimit.getMinimum) Duration.nanosSince(io.airlift.units.Duration.nanosSince) Map(java.util.Map) GENERAL_POOL(com.facebook.presto.memory.LocalMemoryManager.GENERAL_POOL) BasicQueryInfo(com.facebook.presto.server.BasicQueryInfo) ServerConfig(com.facebook.presto.server.ServerConfig) QueryLimit(com.facebook.presto.execution.QueryLimit) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) ExceededMemoryLimitException.exceededGlobalUserLimit(com.facebook.presto.ExceededMemoryLimitException.exceededGlobalUserLimit) Math.min(java.lang.Math.min) Streams(com.google.common.collect.Streams) GuardedBy(javax.annotation.concurrent.GuardedBy) Codec(com.facebook.airlift.json.Codec) Executors(java.util.concurrent.Executors) String.format(java.lang.String.format) QueryIdGenerator(com.facebook.presto.execution.QueryIdGenerator) QueryExecution(com.facebook.presto.execution.QueryExecution) DataSize(io.airlift.units.DataSize) List(java.util.List) Stream(java.util.stream.Stream) Comparator.comparingLong(java.util.Comparator.comparingLong) Entry(java.util.Map.Entry) Optional(java.util.Optional) JmxException(org.weakref.jmx.JmxException) RESOURCE_GROUP(com.facebook.presto.execution.QueryLimit.Source.RESOURCE_GROUP) SystemSessionProperties.resourceOvercommit(com.facebook.presto.SystemSessionProperties.resourceOvercommit) Joiner(com.google.common.base.Joiner) QueryLimit.createDataSizeLimit(com.facebook.presto.execution.QueryLimit.createDataSizeLimit) NodeSchedulerConfig(com.facebook.presto.execution.scheduler.NodeSchedulerConfig) RESOURCE_OVERCOMMIT(com.facebook.presto.SystemSessionProperties.RESOURCE_OVERCOMMIT) Logger(com.facebook.airlift.log.Logger) DataSize.succinctBytes(io.airlift.units.DataSize.succinctBytes) HashMap(java.util.HashMap) PrestoException(com.facebook.presto.spi.PrestoException) CLUSTER_OUT_OF_MEMORY(com.facebook.presto.spi.StandardErrorCode.CLUSTER_OUT_OF_MEMORY) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) Inject(javax.inject.Inject) HashSet(java.util.HashSet) ResourceGroupQueryLimits(com.facebook.presto.spi.resourceGroups.ResourceGroupQueryLimits) SystemSessionProperties.getQueryMaxTotalMemory(com.facebook.presto.SystemSessionProperties.getQueryMaxTotalMemory) ImmutableList(com.google.common.collect.ImmutableList) Managed(org.weakref.jmx.Managed) Closer(com.google.common.io.Closer) Verify.verify(com.google.common.base.Verify.verify) SmileCodec(com.facebook.airlift.json.smile.SmileCodec) Objects.requireNonNull(java.util.Objects.requireNonNull) MemoryPoolInfo(com.facebook.presto.spi.memory.MemoryPoolInfo) SystemSessionProperties.getQueryMaxMemory(com.facebook.presto.SystemSessionProperties.getQueryMaxMemory) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) SimpleEntry(java.util.AbstractMap.SimpleEntry) ExecutorService(java.util.concurrent.ExecutorService) MoreCollectors.toOptional(com.google.common.collect.MoreCollectors.toOptional) ClusterMemoryPoolManager(com.facebook.presto.spi.memory.ClusterMemoryPoolManager) ObjectNames.generatedNameOf(org.weakref.jmx.ObjectNames.generatedNameOf) SHUTTING_DOWN(com.facebook.presto.spi.NodeState.SHUTTING_DOWN) InternalCommunicationConfig(com.facebook.presto.server.InternalCommunicationConfig) IOException(java.io.IOException) LocationFactory(com.facebook.presto.execution.LocationFactory) HttpClient(com.facebook.airlift.http.client.HttpClient) ExceededMemoryLimitException.exceededGlobalTotalLimit(com.facebook.presto.ExceededMemoryLimitException.exceededGlobalTotalLimit) InternalNode(com.facebook.presto.metadata.InternalNode) Consumer(java.util.function.Consumer) MemoryPoolId(com.facebook.presto.spi.memory.MemoryPoolId) AtomicLong(java.util.concurrent.atomic.AtomicLong) ClusterMemoryPoolInfo(com.facebook.presto.spi.memory.ClusterMemoryPoolInfo) QueryId(com.facebook.presto.spi.QueryId) VisibleForTesting(com.google.common.annotations.VisibleForTesting) MBeanExporter(org.weakref.jmx.MBeanExporter) Comparator(java.util.Comparator) ClusterMemoryManagerService(com.facebook.presto.resourcemanager.ClusterMemoryManagerService) Optional(java.util.Optional) MoreCollectors.toOptional(com.google.common.collect.MoreCollectors.toOptional) QueryId(com.facebook.presto.spi.QueryId) QueryMemoryInfo(com.facebook.presto.memory.LowMemoryKiller.QueryMemoryInfo) PrestoException(com.facebook.presto.spi.PrestoException) QueryExecution(com.facebook.presto.execution.QueryExecution)

Aggregations

HttpClient (com.facebook.airlift.http.client.HttpClient)1 Codec (com.facebook.airlift.json.Codec)1 JsonCodec (com.facebook.airlift.json.JsonCodec)1 SmileCodec (com.facebook.airlift.json.smile.SmileCodec)1 Logger (com.facebook.airlift.log.Logger)1 ExceededMemoryLimitException.exceededGlobalTotalLimit (com.facebook.presto.ExceededMemoryLimitException.exceededGlobalTotalLimit)1 ExceededMemoryLimitException.exceededGlobalUserLimit (com.facebook.presto.ExceededMemoryLimitException.exceededGlobalUserLimit)1 RESOURCE_OVERCOMMIT (com.facebook.presto.SystemSessionProperties.RESOURCE_OVERCOMMIT)1 SystemSessionProperties.getQueryMaxMemory (com.facebook.presto.SystemSessionProperties.getQueryMaxMemory)1 SystemSessionProperties.getQueryMaxTotalMemory (com.facebook.presto.SystemSessionProperties.getQueryMaxTotalMemory)1 SystemSessionProperties.resourceOvercommit (com.facebook.presto.SystemSessionProperties.resourceOvercommit)1 LocationFactory (com.facebook.presto.execution.LocationFactory)1 QueryExecution (com.facebook.presto.execution.QueryExecution)1 QueryIdGenerator (com.facebook.presto.execution.QueryIdGenerator)1 QueryLimit (com.facebook.presto.execution.QueryLimit)1 QUERY (com.facebook.presto.execution.QueryLimit.Source.QUERY)1 RESOURCE_GROUP (com.facebook.presto.execution.QueryLimit.Source.RESOURCE_GROUP)1 SYSTEM (com.facebook.presto.execution.QueryLimit.Source.SYSTEM)1 QueryLimit.createDataSizeLimit (com.facebook.presto.execution.QueryLimit.createDataSizeLimit)1 QueryLimit.getMinimum (com.facebook.presto.execution.QueryLimit.getMinimum)1