Search in sources :

Example 6 with QueryExecution

use of com.facebook.presto.execution.QueryExecution in project presto by prestodb.

the class TestLocalDispatchQuery method testQueryQueuedExceptionBeforeDispatch.

@Test
public void testQueryQueuedExceptionBeforeDispatch() {
    QueryStateMachine stateMachine = createStateMachine();
    CountingEventListener eventListener = new CountingEventListener();
    SettableFuture<QueryExecution> queryExecutionFuture = SettableFuture.create();
    LocalDispatchQuery query = new LocalDispatchQuery(stateMachine, createQueryMonitor(eventListener), queryExecutionFuture, createClusterSizeMonitor(0), directExecutor(), dispatchQuery -> {
        throw new QueryQueueFullException(new ResourceGroupId("global"));
    }, execution -> {
    }, false, QUERY_PREREQUISITES);
    query.startWaitingForPrerequisites();
    queryExecutionFuture.setException(new IllegalStateException("abc"));
    assertEquals(query.getBasicQueryInfo().getState(), FAILED);
    assertEquals(query.getBasicQueryInfo().getErrorCode(), QUERY_QUEUE_FULL.toErrorCode());
    assertTrue(eventListener.getQueryCompletedEvent().isPresent());
    assertTrue(eventListener.getQueryCompletedEvent().get().getFailureInfo().isPresent());
    assertEquals(eventListener.getQueryCompletedEvent().get().getFailureInfo().get().getErrorCode(), QUERY_QUEUE_FULL.toErrorCode());
}
Also used : TaskTestUtils.createQueryStateMachine(com.facebook.presto.execution.TaskTestUtils.createQueryStateMachine) QueryStateMachine(com.facebook.presto.execution.QueryStateMachine) ResourceGroupId(com.facebook.presto.spi.resourceGroups.ResourceGroupId) QueryQueueFullException(com.facebook.presto.execution.resourceGroups.QueryQueueFullException) QueryExecution(com.facebook.presto.execution.QueryExecution) MockQueryExecution(com.facebook.presto.execution.MockQueryExecution) Test(org.testng.annotations.Test)

Example 7 with QueryExecution

use of com.facebook.presto.execution.QueryExecution in project presto by prestodb.

the class ClusterMemoryManager method callOomKiller.

private synchronized void callOomKiller(Iterable<QueryExecution> runningQueries) {
    List<QueryMemoryInfo> queryMemoryInfoList = Streams.stream(runningQueries).map(this::createQueryMemoryInfo).collect(toImmutableList());
    List<MemoryInfo> nodeMemoryInfos = nodes.values().stream().map(RemoteNodeMemory::getInfo).filter(Optional::isPresent).map(Optional::get).collect(toImmutableList());
    Optional<QueryId> chosenQueryId = lowMemoryKiller.chooseQueryToKill(queryMemoryInfoList, nodeMemoryInfos);
    if (chosenQueryId.isPresent()) {
        log.debug("Low memory killer chose %s", chosenQueryId.get());
        Optional<QueryExecution> chosenQuery = Streams.stream(runningQueries).filter(query -> chosenQueryId.get().equals(query.getQueryId())).collect(toOptional());
        if (chosenQuery.isPresent()) {
            // See comments in  isLastKilledQueryGone for why chosenQuery might be absent.
            chosenQuery.get().fail(new PrestoException(CLUSTER_OUT_OF_MEMORY, "Query killed because the cluster is out of memory. Please try again in a few minutes."));
            queriesKilledDueToOutOfMemory.incrementAndGet();
            lastKilledQuery = chosenQueryId.get();
            logQueryKill(chosenQueryId.get(), nodeMemoryInfos);
        }
    }
}
Also used : QueryMemoryInfo(com.facebook.presto.memory.LowMemoryKiller.QueryMemoryInfo) JsonCodec(com.facebook.airlift.json.JsonCodec) SYSTEM(com.facebook.presto.execution.QueryLimit.Source.SYSTEM) QUERY(com.facebook.presto.execution.QueryLimit.Source.QUERY) Duration(io.airlift.units.Duration) ACTIVE(com.facebook.presto.spi.NodeState.ACTIVE) RESERVED_POOL(com.facebook.presto.memory.LocalMemoryManager.RESERVED_POOL) PreDestroy(javax.annotation.PreDestroy) Sets.difference(com.google.common.collect.Sets.difference) InternalNodeManager(com.facebook.presto.metadata.InternalNodeManager) QueryMemoryInfo(com.facebook.presto.memory.LowMemoryKiller.QueryMemoryInfo) QueryLimit.getMinimum(com.facebook.presto.execution.QueryLimit.getMinimum) Duration.nanosSince(io.airlift.units.Duration.nanosSince) Map(java.util.Map) GENERAL_POOL(com.facebook.presto.memory.LocalMemoryManager.GENERAL_POOL) BasicQueryInfo(com.facebook.presto.server.BasicQueryInfo) ServerConfig(com.facebook.presto.server.ServerConfig) QueryLimit(com.facebook.presto.execution.QueryLimit) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) ExceededMemoryLimitException.exceededGlobalUserLimit(com.facebook.presto.ExceededMemoryLimitException.exceededGlobalUserLimit) Math.min(java.lang.Math.min) Streams(com.google.common.collect.Streams) GuardedBy(javax.annotation.concurrent.GuardedBy) Codec(com.facebook.airlift.json.Codec) Executors(java.util.concurrent.Executors) String.format(java.lang.String.format) QueryIdGenerator(com.facebook.presto.execution.QueryIdGenerator) QueryExecution(com.facebook.presto.execution.QueryExecution) DataSize(io.airlift.units.DataSize) List(java.util.List) Stream(java.util.stream.Stream) Comparator.comparingLong(java.util.Comparator.comparingLong) Entry(java.util.Map.Entry) Optional(java.util.Optional) JmxException(org.weakref.jmx.JmxException) RESOURCE_GROUP(com.facebook.presto.execution.QueryLimit.Source.RESOURCE_GROUP) SystemSessionProperties.resourceOvercommit(com.facebook.presto.SystemSessionProperties.resourceOvercommit) Joiner(com.google.common.base.Joiner) QueryLimit.createDataSizeLimit(com.facebook.presto.execution.QueryLimit.createDataSizeLimit) NodeSchedulerConfig(com.facebook.presto.execution.scheduler.NodeSchedulerConfig) RESOURCE_OVERCOMMIT(com.facebook.presto.SystemSessionProperties.RESOURCE_OVERCOMMIT) Logger(com.facebook.airlift.log.Logger) DataSize.succinctBytes(io.airlift.units.DataSize.succinctBytes) HashMap(java.util.HashMap) PrestoException(com.facebook.presto.spi.PrestoException) CLUSTER_OUT_OF_MEMORY(com.facebook.presto.spi.StandardErrorCode.CLUSTER_OUT_OF_MEMORY) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) Inject(javax.inject.Inject) HashSet(java.util.HashSet) ResourceGroupQueryLimits(com.facebook.presto.spi.resourceGroups.ResourceGroupQueryLimits) SystemSessionProperties.getQueryMaxTotalMemory(com.facebook.presto.SystemSessionProperties.getQueryMaxTotalMemory) ImmutableList(com.google.common.collect.ImmutableList) Managed(org.weakref.jmx.Managed) Closer(com.google.common.io.Closer) Verify.verify(com.google.common.base.Verify.verify) SmileCodec(com.facebook.airlift.json.smile.SmileCodec) Objects.requireNonNull(java.util.Objects.requireNonNull) MemoryPoolInfo(com.facebook.presto.spi.memory.MemoryPoolInfo) SystemSessionProperties.getQueryMaxMemory(com.facebook.presto.SystemSessionProperties.getQueryMaxMemory) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) SimpleEntry(java.util.AbstractMap.SimpleEntry) ExecutorService(java.util.concurrent.ExecutorService) MoreCollectors.toOptional(com.google.common.collect.MoreCollectors.toOptional) ClusterMemoryPoolManager(com.facebook.presto.spi.memory.ClusterMemoryPoolManager) ObjectNames.generatedNameOf(org.weakref.jmx.ObjectNames.generatedNameOf) SHUTTING_DOWN(com.facebook.presto.spi.NodeState.SHUTTING_DOWN) InternalCommunicationConfig(com.facebook.presto.server.InternalCommunicationConfig) IOException(java.io.IOException) LocationFactory(com.facebook.presto.execution.LocationFactory) HttpClient(com.facebook.airlift.http.client.HttpClient) ExceededMemoryLimitException.exceededGlobalTotalLimit(com.facebook.presto.ExceededMemoryLimitException.exceededGlobalTotalLimit) InternalNode(com.facebook.presto.metadata.InternalNode) Consumer(java.util.function.Consumer) MemoryPoolId(com.facebook.presto.spi.memory.MemoryPoolId) AtomicLong(java.util.concurrent.atomic.AtomicLong) ClusterMemoryPoolInfo(com.facebook.presto.spi.memory.ClusterMemoryPoolInfo) QueryId(com.facebook.presto.spi.QueryId) VisibleForTesting(com.google.common.annotations.VisibleForTesting) MBeanExporter(org.weakref.jmx.MBeanExporter) Comparator(java.util.Comparator) ClusterMemoryManagerService(com.facebook.presto.resourcemanager.ClusterMemoryManagerService) Optional(java.util.Optional) MoreCollectors.toOptional(com.google.common.collect.MoreCollectors.toOptional) QueryId(com.facebook.presto.spi.QueryId) QueryMemoryInfo(com.facebook.presto.memory.LowMemoryKiller.QueryMemoryInfo) PrestoException(com.facebook.presto.spi.PrestoException) QueryExecution(com.facebook.presto.execution.QueryExecution)

Aggregations

QueryExecution (com.facebook.presto.execution.QueryExecution)7 PrestoException (com.facebook.presto.spi.PrestoException)5 MemoryPoolId (com.facebook.presto.spi.memory.MemoryPoolId)4 ClusterMemoryPoolInfo (com.facebook.presto.spi.memory.ClusterMemoryPoolInfo)3 DataSize (io.airlift.units.DataSize)3 HashMap (java.util.HashMap)3 HttpClient (com.facebook.airlift.http.client.HttpClient)2 Codec (com.facebook.airlift.json.Codec)2 JsonCodec (com.facebook.airlift.json.JsonCodec)2 SmileCodec (com.facebook.airlift.json.smile.SmileCodec)2 Logger (com.facebook.airlift.log.Logger)2 ExceededMemoryLimitException.exceededGlobalTotalLimit (com.facebook.presto.ExceededMemoryLimitException.exceededGlobalTotalLimit)2 ExceededMemoryLimitException.exceededGlobalUserLimit (com.facebook.presto.ExceededMemoryLimitException.exceededGlobalUserLimit)2 RESOURCE_OVERCOMMIT (com.facebook.presto.SystemSessionProperties.RESOURCE_OVERCOMMIT)2 SystemSessionProperties.getQueryMaxMemory (com.facebook.presto.SystemSessionProperties.getQueryMaxMemory)2 SystemSessionProperties.getQueryMaxTotalMemory (com.facebook.presto.SystemSessionProperties.getQueryMaxTotalMemory)2 SystemSessionProperties.resourceOvercommit (com.facebook.presto.SystemSessionProperties.resourceOvercommit)2 LocationFactory (com.facebook.presto.execution.LocationFactory)2 QueryIdGenerator (com.facebook.presto.execution.QueryIdGenerator)2 QueryLimit (com.facebook.presto.execution.QueryLimit)2