Examples with MemoryPoolInfo com.facebook.presto.spi.memory.MemoryPoolInfo used on opensource projects

Example 1 with MemoryPoolInfo

use of com.facebook.presto.spi.memory.MemoryPoolInfo in project presto by prestodb.

the class TotalReservationOnBlockedNodesLowMemoryKiller method chooseQueryToKill.

@Override
public Optional<QueryId> chooseQueryToKill(List<QueryMemoryInfo> runningQueries, List<MemoryInfo> nodes) {
    Map<QueryId, Long> memoryReservationOnBlockedNodes = new HashMap<>();
    for (MemoryInfo node : nodes) {
        MemoryPoolInfo generalPool = node.getPools().get(GENERAL_POOL);
        if (generalPool == null) {
            continue;
        }
        if (generalPool.getFreeBytes() + generalPool.getReservedRevocableBytes() > 0) {
            continue;
        }
        Map<QueryId, Long> queryMemoryReservations = generalPool.getQueryMemoryReservations();
        queryMemoryReservations.forEach((queryId, memoryReservation) -> {
            memoryReservationOnBlockedNodes.compute(queryId, (id, oldValue) -> oldValue == null ? memoryReservation : oldValue + memoryReservation);
        });
    }
    return memoryReservationOnBlockedNodes.entrySet().stream().max(comparingLong(Map.Entry::getValue)).map(Map.Entry::getKey);
}

Example 2 with MemoryPoolInfo

use of com.facebook.presto.spi.memory.MemoryPoolInfo in project presto by prestodb.

the class ClusterMemoryManager method updatePools.

private synchronized void updatePools(Map<MemoryPoolId, Integer> queryCounts) {
    // Update view of cluster memory and pools
    List<MemoryInfo> nodeMemoryInfos = nodes.values().stream().map(RemoteNodeMemory::getInfo).filter(Optional::isPresent).map(Optional::get).collect(toImmutableList());
    long totalClusterMemory = nodeMemoryInfos.stream().map(MemoryInfo::getTotalNodeMemory).mapToLong(DataSize::toBytes).sum();
    clusterMemoryBytes.set(totalClusterMemory);
    for (ClusterMemoryPool pool : pools.values()) {
        pool.update(nodeMemoryInfos, queryCounts.getOrDefault(pool.getId(), 0));
        if (changeListeners.containsKey(pool.getId())) {
            MemoryPoolInfo info = getClusterInfo(pool.getId()).getMemoryPoolInfo();
            for (Consumer<MemoryPoolInfo> listener : changeListeners.get(pool.getId())) {
                listenerExecutor.execute(() -> listener.accept(info));
            }
        }
    }
}

Example 3 with MemoryPoolInfo

use of com.facebook.presto.spi.memory.MemoryPoolInfo in project presto by prestodb.

the class ClusterMemoryPool method update.

public synchronized void update(List<MemoryInfo> memoryInfos, int assignedQueries) {
    nodes = 0;
    blockedNodes = 0;
    totalDistributedBytes = 0;
    reservedDistributedBytes = 0;
    reservedRevocableDistributedBytes = 0;
    this.assignedQueries = assignedQueries;
    this.queryMemoryReservations.clear();
    this.queryMemoryAllocations.clear();
    this.queryMemoryRevocableReservations.clear();
    for (MemoryInfo info : memoryInfos) {
        MemoryPoolInfo poolInfo = info.getPools().get(id);
        if (poolInfo != null) {
            nodes++;
            if (poolInfo.getFreeBytes() + poolInfo.getReservedRevocableBytes() <= 0) {
                blockedNodes++;
            }
            totalDistributedBytes += poolInfo.getMaxBytes();
            reservedDistributedBytes += poolInfo.getReservedBytes();
            reservedRevocableDistributedBytes += poolInfo.getReservedRevocableBytes();
            for (Map.Entry<QueryId, Long> entry : poolInfo.getQueryMemoryReservations().entrySet()) {
                queryMemoryReservations.merge(entry.getKey(), entry.getValue(), Long::sum);
            }
            for (Map.Entry<QueryId, List<MemoryAllocation>> entry : poolInfo.getQueryMemoryAllocations().entrySet()) {
                queryMemoryAllocations.merge(entry.getKey(), entry.getValue(), this::mergeQueryAllocations);
            }
            for (Map.Entry<QueryId, Long> entry : poolInfo.getQueryMemoryRevocableReservations().entrySet()) {
                queryMemoryRevocableReservations.merge(entry.getKey(), entry.getValue(), Long::sum);
            }
        }
    }
}

Example 4 with MemoryPoolInfo

use of com.facebook.presto.spi.memory.MemoryPoolInfo in project presto by prestodb.

the class LowMemoryKillerTestingUtils method toNodeMemoryInfoList.

static List<MemoryInfo> toNodeMemoryInfoList(long maxReservedPoolBytes, long maxGeneralPoolBytes, String reservedQuery, Map<String, Map<String, Long>> queries) {
    Map<InternalNode, NodeReservation> nodeReservations = new HashMap<>();
    for (Map.Entry<String, Map<String, Long>> entry : queries.entrySet()) {
        QueryId queryId = new QueryId(entry.getKey());
        Map<String, Long> reservationByNode = entry.getValue();
        for (Map.Entry<String, Long> nodeEntry : reservationByNode.entrySet()) {
            InternalNode node = new InternalNode(nodeEntry.getKey(), URI.create("http://localhost"), new NodeVersion("version"), false);
            long bytes = nodeEntry.getValue();
            if (bytes == 0) {
                continue;
            }
            if (reservedQuery.equals(entry.getKey())) {
                nodeReservations.computeIfAbsent(node, ignored -> new NodeReservation()).getReserved().add(queryId, bytes);
            } else {
                nodeReservations.computeIfAbsent(node, ignored -> new NodeReservation()).getGeneral().add(queryId, bytes);
            }
        }
    }
    ImmutableList.Builder<MemoryInfo> result = ImmutableList.builder();
    for (Map.Entry<InternalNode, NodeReservation> entry : nodeReservations.entrySet()) {
        NodeReservation nodeReservation = entry.getValue();
        ImmutableMap.Builder<MemoryPoolId, MemoryPoolInfo> pools = ImmutableMap.builder();
        if (nodeReservation.getGeneral().getTotalReservedBytes() > 0) {
            pools.put(GENERAL_POOL, new MemoryPoolInfo(maxGeneralPoolBytes, nodeReservation.getGeneral().getTotalReservedBytes(), 0, nodeReservation.getGeneral().getReservationByQuery(), ImmutableMap.of(), ImmutableMap.of()));
        }
        if (nodeReservation.getReserved().getTotalReservedBytes() > 0) {
            pools.put(RESERVED_POOL, new MemoryPoolInfo(maxReservedPoolBytes, nodeReservation.getReserved().getTotalReservedBytes(), 0, nodeReservation.getReserved().getReservationByQuery(), ImmutableMap.of(), ImmutableMap.of()));
        }
        result.add(new MemoryInfo(new DataSize(maxReservedPoolBytes + maxGeneralPoolBytes, BYTE), pools.build()));
    }
    return result.build();
}