Examples with MemoryPoolInfo io.trino.spi.memory.MemoryPoolInfo used on opensource projects

Example 1 with MemoryPoolInfo

use of io.trino.spi.memory.MemoryPoolInfo in project trino by trinodb.

the class TotalReservationOnBlockedNodesLowMemoryKiller method chooseWholeQueryToKill.

private Optional<KillTarget> chooseWholeQueryToKill(List<MemoryInfo> nodes) {
    Map<QueryId, Long> memoryReservationOnBlockedNodes = new HashMap<>();
    for (MemoryInfo node : nodes) {
        MemoryPoolInfo memoryPool = node.getPool();
        if (memoryPool == null) {
            continue;
        }
        if (memoryPool.getFreeBytes() + memoryPool.getReservedRevocableBytes() > 0) {
            continue;
        }
        Map<QueryId, Long> queryMemoryReservations = memoryPool.getQueryMemoryReservations();
        queryMemoryReservations.forEach((queryId, memoryReservation) -> {
            memoryReservationOnBlockedNodes.compute(queryId, (id, oldValue) -> oldValue == null ? memoryReservation : oldValue + memoryReservation);
        });
    }
    return memoryReservationOnBlockedNodes.entrySet().stream().max(comparingLong(Map.Entry::getValue)).map(Map.Entry::getKey).map(KillTarget::wholeQuery);
}

Example 2 with MemoryPoolInfo

use of io.trino.spi.memory.MemoryPoolInfo in project trino by trinodb.

the class TotalReservationOnBlockedNodesLowMemoryKiller method chooseTasksToKill.

private Optional<KillTarget> chooseTasksToKill(List<MemoryInfo> nodes) {
    ImmutableSet.Builder<TaskId> tasksToKillBuilder = ImmutableSet.builder();
    for (MemoryInfo node : nodes) {
        MemoryPoolInfo memoryPool = node.getPool();
        if (memoryPool == null) {
            continue;
        }
        if (memoryPool.getFreeBytes() + memoryPool.getReservedRevocableBytes() > 0) {
            continue;
        }
        node.getTasksMemoryInfo().values().stream().max(comparing(TaskMemoryInfo::getMemoryReservation)).map(TaskMemoryInfo::getTaskId).ifPresent(tasksToKillBuilder::add);
    }
    Set<TaskId> tasksToKill = tasksToKillBuilder.build();
    if (tasksToKill.isEmpty()) {
        return Optional.empty();
    }
    return Optional.of(KillTarget.selectedTasks(tasksToKill));
}

Example 3 with MemoryPoolInfo

use of io.trino.spi.memory.MemoryPoolInfo in project trino by trinodb.

the class ClusterMemoryManager method updateMemoryPool.

private synchronized void updateMemoryPool(int queryCount) {
    // Update view of cluster memory and pools
    List<MemoryInfo> nodeMemoryInfos = nodes.values().stream().map(RemoteNodeMemory::getInfo).filter(Optional::isPresent).map(Optional::get).collect(toImmutableList());
    long totalProcessors = nodeMemoryInfos.stream().mapToLong(MemoryInfo::getAvailableProcessors).sum();
    totalAvailableProcessors.set(totalProcessors);
    long totalClusterMemory = nodeMemoryInfos.stream().mapToLong(memoryInfo -> memoryInfo.getPool().getMaxBytes()).sum();
    clusterMemoryBytes.set(totalClusterMemory);
    pool.update(nodeMemoryInfos, queryCount);
    if (!changeListeners.isEmpty()) {
        MemoryPoolInfo info = pool.getInfo();
        for (Consumer<MemoryPoolInfo> listener : changeListeners) {
            listenerExecutor.execute(() -> listener.accept(info));
        }
    }
}

Example 4 with MemoryPoolInfo

use of io.trino.spi.memory.MemoryPoolInfo in project trino by trinodb.

the class ClusterMemoryPool method update.

public synchronized void update(List<MemoryInfo> memoryInfos, int assignedQueries) {
    nodes = 0;
    blockedNodes = 0;
    totalDistributedBytes = 0;
    reservedDistributedBytes = 0;
    reservedRevocableDistributedBytes = 0;
    this.assignedQueries = assignedQueries;
    this.queryMemoryReservations.clear();
    this.queryMemoryAllocations.clear();
    this.queryMemoryRevocableReservations.clear();
    for (MemoryInfo info : memoryInfos) {
        MemoryPoolInfo poolInfo = info.getPool();
        nodes++;
        if (poolInfo.getFreeBytes() + poolInfo.getReservedRevocableBytes() <= 0) {
            blockedNodes++;
        }
        totalDistributedBytes += poolInfo.getMaxBytes();
        reservedDistributedBytes += poolInfo.getReservedBytes();
        reservedRevocableDistributedBytes += poolInfo.getReservedRevocableBytes();
        for (Map.Entry<QueryId, Long> entry : poolInfo.getQueryMemoryReservations().entrySet()) {
            queryMemoryReservations.merge(entry.getKey(), entry.getValue(), Long::sum);
        }
        for (Map.Entry<QueryId, List<MemoryAllocation>> entry : poolInfo.getQueryMemoryAllocations().entrySet()) {
            queryMemoryAllocations.merge(entry.getKey(), entry.getValue(), this::mergeQueryAllocations);
        }
        for (Map.Entry<QueryId, Long> entry : poolInfo.getQueryMemoryRevocableReservations().entrySet()) {
            queryMemoryRevocableReservations.merge(entry.getKey(), entry.getValue(), Long::sum);
        }
    }
}

Example 5 with MemoryPoolInfo

use of io.trino.spi.memory.MemoryPoolInfo in project trino by trinodb.

the class TestFullNodeCapableNodeAllocator method setupNodeAllocatorService.

private void setupNodeAllocatorService(InMemoryNodeManager nodeManager, int maxFullNodesPerQuery) {
    // just in case
    shutdownNodeAllocatorService();
    MemoryInfo memoryInfo = new MemoryInfo(4, new MemoryPoolInfo(DataSize.of(64, GIGABYTE).toBytes(), 0, 0, ImmutableMap.of(), ImmutableMap.of(), ImmutableMap.of()));
    Map<String, Optional<MemoryInfo>> workerMemoryInfos = ImmutableMap.of(NODE_1.getNodeIdentifier(), Optional.of(memoryInfo), NODE_2.getNodeIdentifier(), Optional.of(memoryInfo), NODE_3.getNodeIdentifier(), Optional.of(memoryInfo), NODE_4.getNodeIdentifier(), Optional.of(memoryInfo));
    nodeAllocatorService = new FullNodeCapableNodeAllocatorService(nodeManager, () -> workerMemoryInfos, maxFullNodesPerQuery, 1.0, false);
    nodeAllocatorService.start();
}