Examples with MemoryPoolId io.prestosql.spi.memory.MemoryPoolId used on opensource projects

Example 6 with MemoryPoolId

use of io.prestosql.spi.memory.MemoryPoolId in project hetu-core by openlookeng.

the class ClusterMemoryManager method process.

public synchronized void process(Iterable<QueryExecution> runningQueries, Supplier<List<BasicQueryInfo>> allQueryInfoSupplier) {
    if (!enabled) {
        return;
    }
    // make sure state store is loaded when multiple coordinator is enabled
    if (!hetuConfig.isMultipleCoordinatorEnabled() || (hetuConfig.isMultipleCoordinatorEnabled() && stateStoreProvider.getStateStore() != null)) {
        Lock lock = null;
        boolean locked = false;
        try {
            if (hetuConfig.isMultipleCoordinatorEnabled()) {
                lock = stateStoreProvider.getStateStore().getLock(StateStoreConstants.HANDLE_OOM_QUERY_LOCK_NAME);
            } else {
                lock = new ReentrantLock();
            }
            locked = lock.tryLock(StateStoreConstants.DEFAULT_ACQUIRED_LOCK_TIME_MS, TimeUnit.MILLISECONDS);
            if (locked) {
                // TODO revocable memory reservations can also leak and may need to be detected in the future
                // We are only concerned about the leaks in general pool.
                memoryLeakDetector.checkForMemoryLeaks(allQueryInfoSupplier, pools.get(GENERAL_POOL).getQueryMemoryReservations());
                boolean outOfMemory = isClusterOutOfMemory();
                if (!outOfMemory) {
                    lastTimeNotOutOfMemory = System.nanoTime();
                }
                boolean queryKilled = false;
                long totalUserMemoryBytes = 0L;
                long totalMemoryBytes = 0L;
                for (QueryExecution query : runningQueries) {
                    boolean resourceOvercommit = resourceOvercommit(query.getSession());
                    long userMemoryReservation = query.getUserMemoryReservation().toBytes();
                    long totalMemoryReservation = query.getTotalMemoryReservation().toBytes();
                    if (resourceOvercommit && outOfMemory) {
                        // If a query has requested resource overcommit, only kill it if the cluster has run out of memory
                        DataSize memory = succinctBytes(getQueryMemoryReservation(query));
                        query.fail(new PrestoException(CLUSTER_OUT_OF_MEMORY, format("The cluster is out of memory and %s=true, so this query was killed. It was using %s of memory", RESOURCE_OVERCOMMIT, memory)));
                        queryKilled = true;
                    }
                    if (!resourceOvercommit) {
                        long userMemoryLimit = min(maxQueryMemory.toBytes(), getQueryMaxMemory(query.getSession()).toBytes());
                        if (userMemoryReservation > userMemoryLimit) {
                            query.fail(exceededGlobalUserLimit(succinctBytes(userMemoryLimit)));
                            queryKilled = true;
                        }
                        long totalMemoryLimit = min(maxQueryTotalMemory.toBytes(), getQueryMaxTotalMemory(query.getSession()).toBytes());
                        if (totalMemoryReservation > totalMemoryLimit) {
                            query.fail(exceededGlobalTotalLimit(succinctBytes(totalMemoryLimit)));
                            queryKilled = true;
                        }
                    }
                    totalUserMemoryBytes += userMemoryReservation;
                    totalMemoryBytes += totalMemoryReservation;
                }
                clusterUserMemoryReservation.set(totalUserMemoryBytes);
                clusterTotalMemoryReservation.set(totalMemoryBytes);
                if (!(lowMemoryKiller instanceof NoneLowMemoryKiller) && outOfMemory && !queryKilled && nanosSince(lastTimeNotOutOfMemory).compareTo(killOnOutOfMemoryDelay) > 0) {
                    if (isLastKilledQueryGone()) {
                        callOomKiller(runningQueries);
                    } else {
                        log.debug("Last killed query is still not gone: %s", lastKilledQuery);
                    }
                }
            }
        } catch (Exception e) {
            log.error("Error handleOOMQuery: " + e.getMessage());
        } finally {
            if (lock != null && locked) {
                lock.unlock();
            }
        }
    }
    Map<MemoryPoolId, Integer> countByPool = new HashMap<>();
    for (QueryExecution query : runningQueries) {
        MemoryPoolId id = query.getMemoryPool().getId();
        countByPool.put(id, countByPool.getOrDefault(id, 0) + 1);
    }
    updatePools(countByPool);
    MemoryPoolAssignmentsRequest assignmentsRequest;
    if (pools.containsKey(RESERVED_POOL)) {
        assignmentsRequest = updateAssignments(runningQueries);
    } else {
        // If reserved pool is not enabled, we don't create a MemoryPoolAssignmentsRequest that puts all the queries
        // in the general pool (as they already are). In this case we create an effectively NOOP MemoryPoolAssignmentsRequest.
        // Once the reserved pool is removed we should get rid of the logic of putting queries into reserved pool including
        // this piece of code.
        assignmentsRequest = new MemoryPoolAssignmentsRequest(coordinatorId, Long.MIN_VALUE, ImmutableList.of());
    }
    updateNodes(assignmentsRequest);
}

Example 7 with MemoryPoolId

use of io.prestosql.spi.memory.MemoryPoolId in project hetu-core by openlookeng.

the class ClusterMemoryManager method createClusterMemoryPools.

private Map<MemoryPoolId, ClusterMemoryPool> createClusterMemoryPools(boolean reservedPoolEnabled) {
    Set<MemoryPoolId> memoryPools = new HashSet<>();
    memoryPools.add(GENERAL_POOL);
    if (reservedPoolEnabled) {
        memoryPools.add(RESERVED_POOL);
    }
    ImmutableMap.Builder<MemoryPoolId, ClusterMemoryPool> builder = ImmutableMap.builder();
    for (MemoryPoolId poolId : memoryPools) {
        ClusterMemoryPool pool = new ClusterMemoryPool(poolId);
        builder.put(poolId, pool);
        try {
            exporter.exportWithGeneratedName(pool, ClusterMemoryPool.class, poolId.toString());
        } catch (JmxException e) {
            log.error(e, "Error exporting memory pool %s", poolId);
        }
    }
    return builder.build();
}

Example 8 with MemoryPoolId

use of io.prestosql.spi.memory.MemoryPoolId in project hetu-core by openlookeng.

the class LocalMemoryManager method configureMemoryPools.

private void configureMemoryPools(NodeMemoryConfig config, long availableMemory) {
    validateHeapHeadroom(config, availableMemory);
    maxMemory = new DataSize(availableMemory - config.getHeapHeadroom().toBytes(), BYTE);
    checkArgument(config.getMaxQueryMemoryPerNode().toBytes() <= config.getMaxQueryTotalMemoryPerNode().toBytes(), "Max query memory per node (%s) cannot be greater than the max query total memory per node (%s).", QUERY_MAX_MEMORY_PER_NODE_CONFIG, QUERY_MAX_TOTAL_MEMORY_PER_NODE_CONFIG);
    ImmutableMap.Builder<MemoryPoolId, MemoryPool> builder = ImmutableMap.builder();
    long generalPoolSize = maxMemory.toBytes();
    if (config.isReservedPoolEnabled()) {
        builder.put(RESERVED_POOL, new MemoryPool(RESERVED_POOL, config.getMaxQueryTotalMemoryPerNode()));
        generalPoolSize -= config.getMaxQueryTotalMemoryPerNode().toBytes();
    }
    verify(generalPoolSize > 0, "general memory pool size is 0");
    builder.put(GENERAL_POOL, new MemoryPool(GENERAL_POOL, new DataSize(generalPoolSize, BYTE)));
    this.pools = builder.build();
}

Example 9 with MemoryPoolId

use of io.prestosql.spi.memory.MemoryPoolId in project hetu-core by openlookeng.

the class TestSqlTask method createInitialTask.

private SqlTask createInitialTask() {
    TaskId taskId = new TaskId("query", 0, nextTaskId.incrementAndGet());
    String instanceId = "0-query_test_instance_id";
    URI location = URI.create("fake://task/" + taskId);
    QueryContext queryContext = new QueryContext(new QueryId("query"), new DataSize(1, MEGABYTE), new DataSize(2, MEGABYTE), new MemoryPool(new MemoryPoolId("test"), new DataSize(1, GIGABYTE)), new TestingGcMonitor(), taskNotificationExecutor, driverYieldExecutor, new DataSize(1, MEGABYTE), new SpillSpaceTracker(new DataSize(1, GIGABYTE)), NOOP_SNAPSHOT_UTILS);
    queryContext.addTaskContext(new TaskStateMachine(taskId, taskNotificationExecutor), testSessionBuilder().build(), false, false, OptionalInt.empty(), Optional.empty(), TESTING_SERDE_FACTORY);
    return createSqlTask(taskId, instanceId, location, "fake", queryContext, sqlTaskExecutionFactory, taskNotificationExecutor, Functions.identity(), new DataSize(32, MEGABYTE), new CounterStat(), createTestMetadataManager());
}

Example 10 with MemoryPoolId

use of io.prestosql.spi.memory.MemoryPoolId in project hetu-core by openlookeng.

the class BufferTestUtils method newTestingTaskContext.

static TaskContext newTestingTaskContext() {
    ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
    QueryContext queryContext = new QueryContext(new QueryId("query_id"), new DataSize(1, MEGABYTE), new DataSize(2, MEGABYTE), new MemoryPool(new MemoryPoolId("test"), new DataSize(1, GIGABYTE)), new TestingGcMonitor(), executor, executor, new DataSize(1, MEGABYTE), new SpillSpaceTracker(new DataSize(1, GIGABYTE)), NOOP_SNAPSHOT_UTILS);
    TaskId taskId = TaskId.valueOf("query_id.1.2");
    TaskContext taskContext = queryContext.addTaskContext(new TaskStateMachine(taskId, executor), TEST_SNAPSHOT_SESSION, false, false, OptionalInt.empty(), Optional.empty(), TESTING_SERDE_FACTORY);
    taskContext.getSnapshotManager().setTotalComponents(100);
    return taskContext;
}