Examples with ClusterMemoryPoolInfo com.facebook.presto.spi.memory.ClusterMemoryPoolInfo used on opensource projects

Example 1 with ClusterMemoryPoolInfo

use of com.facebook.presto.spi.memory.ClusterMemoryPoolInfo in project presto by prestodb.

the class ClusterMemoryManager method findLargestMemoryQuery.

private QueryExecution findLargestMemoryQuery(ClusterMemoryPoolInfo generalPool, Iterable<QueryExecution> queries) {
    QueryExecution biggestQuery = null;
    long maxMemory = -1;
    Optional<QueryId> largestMemoryQuery = generalPool.getLargestMemoryQuery();
    // If present, this means the resource manager is determining the largest query, so do not make this determination locally
    if (memoryManagerService.isPresent()) {
        return largestMemoryQuery.flatMap(largestMemoryQueryId -> Streams.stream(queries).filter(query -> query.getQueryId().equals(largestMemoryQueryId)).findFirst()).orElse(null);
    }
    for (QueryExecution queryExecution : queries) {
        if (resourceOvercommit(queryExecution.getSession())) {
            // since their memory usage is unbounded.
            continue;
        }
        long bytesUsed = getQueryMemoryReservation(queryExecution);
        if (bytesUsed > maxMemory) {
            biggestQuery = queryExecution;
            maxMemory = bytesUsed;
        }
    }
    return biggestQuery;
}

Example 2 with ClusterMemoryPoolInfo

use of com.facebook.presto.spi.memory.ClusterMemoryPoolInfo in project presto by prestodb.

the class ClusterMemoryManager method updateAssignments.

// TODO once the reserved pool is removed we can remove this method. We can also update
// RemoteNodeMemory as we don't need to POST anything.
private synchronized MemoryPoolAssignmentsRequest updateAssignments(Iterable<QueryExecution> queries) {
    ClusterMemoryPoolInfo reservedPool = getClusterInfo(RESERVED_POOL);
    ClusterMemoryPoolInfo generalPool = getClusterInfo(GENERAL_POOL);
    verify(generalPool != null, "generalPool is null");
    verify(reservedPool != null, "reservedPool is null");
    long version = memoryPoolAssignmentsVersion.incrementAndGet();
    // and is more of a safety check than a guarantee
    if (allAssignmentsHavePropagated(queries)) {
        if (reservedPool.getAssignedQueries() == 0 && generalPool.getBlockedNodes() > 0) {
            QueryExecution biggestQuery = findLargestMemoryQuery(generalPool, queries);
            if (biggestQuery != null) {
                log.info("Moving query %s to the reserved pool", biggestQuery.getQueryId());
                biggestQuery.setMemoryPool(new VersionedMemoryPoolId(RESERVED_POOL, version));
            }
        }
    }
    ImmutableList.Builder<MemoryPoolAssignment> assignments = ImmutableList.builder();
    for (QueryExecution queryExecution : queries) {
        assignments.add(new MemoryPoolAssignment(queryExecution.getQueryId(), queryExecution.getMemoryPool().getId()));
    }
    return new MemoryPoolAssignmentsRequest(coordinatorId, version, assignments.build());
}

Example 3 with ClusterMemoryPoolInfo

use of com.facebook.presto.spi.memory.ClusterMemoryPoolInfo in project presto by prestodb.

the class TestResourceManagerClusterStateProvider method assertMemoryPoolMap.

private void assertMemoryPoolMap(ResourceManagerClusterStateProvider provider, int memoryPoolSize, MemoryPoolId memoryPoolId, int assignedQueries, int blockedNodes, int maxBytes, int reservedBytes, int reservedRevocableBytes, Optional<String> largestMemoryQuery) {
    Map<MemoryPoolId, ClusterMemoryPoolInfo> memoryPoolMap = provider.getClusterMemoryPoolInfo();
    assertNotNull(memoryPoolMap);
    assertEquals(memoryPoolMap.size(), memoryPoolSize);
    ClusterMemoryPoolInfo clusterMemoryPoolInfo = memoryPoolMap.get(memoryPoolId);
    assertNotNull(clusterMemoryPoolInfo);
    assertEquals(clusterMemoryPoolInfo.getAssignedQueries(), assignedQueries);
    assertEquals(clusterMemoryPoolInfo.getBlockedNodes(), blockedNodes);
    assertEquals(clusterMemoryPoolInfo.getMemoryPoolInfo().getMaxBytes(), maxBytes);
    assertEquals(clusterMemoryPoolInfo.getMemoryPoolInfo().getReservedBytes(), reservedBytes);
    assertEquals(clusterMemoryPoolInfo.getMemoryPoolInfo().getReservedRevocableBytes(), reservedRevocableBytes);
    assertEquals(clusterMemoryPoolInfo.getLargestMemoryQuery().map(QueryId::getId), largestMemoryQuery);
}

Example 4 with ClusterMemoryPoolInfo

use of com.facebook.presto.spi.memory.ClusterMemoryPoolInfo in project presto by prestodb.

the class ResourceManagerClusterStateProvider method getClusterMemoryPoolInfoInternal.

private Map<MemoryPoolId, ClusterMemoryPoolInfo> getClusterMemoryPoolInfoInternal() {
    List<MemoryInfo> memoryInfos = nodeStatuses.values().stream().map(nodeStatus -> nodeStatus.getNodeStatus().getMemoryInfo()).collect(toImmutableList());
    int queriesAssignedToGeneralPool = 0;
    int queriesAssignedToReservedPool = 0;
    Query largestGeneralPoolQuery = null;
    for (CoordinatorQueriesState nodeQueryState : nodeQueryStates.values()) {
        for (Query query : nodeQueryState.getActiveQueries()) {
            MemoryPoolId memoryPool = query.getBasicQueryInfo().getMemoryPool();
            if (GENERAL_POOL.equals(memoryPool)) {
                queriesAssignedToGeneralPool = Math.incrementExact(queriesAssignedToGeneralPool);
                if (!resourceOvercommit(query.getBasicQueryInfo().getSession().toSession(sessionPropertyManager))) {
                    largestGeneralPoolQuery = getLargestMemoryQuery(Optional.ofNullable(largestGeneralPoolQuery), query);
                }
            } else if (RESERVED_POOL.equals(memoryPool)) {
                queriesAssignedToReservedPool = Math.incrementExact(queriesAssignedToReservedPool);
            } else {
                throw new IllegalArgumentException("Unrecognized memory pool: " + memoryPool);
            }
        }
    }
    ImmutableMap.Builder<MemoryPoolId, ClusterMemoryPoolInfo> memoryPoolInfos = ImmutableMap.builder();
    ClusterMemoryPool pool = new ClusterMemoryPool(GENERAL_POOL);
    pool.update(memoryInfos, queriesAssignedToGeneralPool);
    ClusterMemoryPoolInfo clusterInfo = pool.getClusterInfo(Optional.ofNullable(largestGeneralPoolQuery).map(Query::getQueryId));
    memoryPoolInfos.put(GENERAL_POOL, clusterInfo);
    if (isReservedPoolEnabled) {
        pool = new ClusterMemoryPool(RESERVED_POOL);
        pool.update(memoryInfos, queriesAssignedToReservedPool);
        memoryPoolInfos.put(RESERVED_POOL, pool.getClusterInfo());
    }
    return memoryPoolInfos.build();
}

Example 5 with ClusterMemoryPoolInfo

use of com.facebook.presto.spi.memory.ClusterMemoryPoolInfo in project presto by prestodb.

the class ClusterMemoryManager method isClusterOutOfMemory.

private synchronized boolean isClusterOutOfMemory() {
    ClusterMemoryPoolInfo reservedPool = getClusterInfo(RESERVED_POOL);
    ClusterMemoryPoolInfo generalPool = getClusterInfo(GENERAL_POOL);
    if (reservedPool == null) {
        return generalPool.getBlockedNodes() > 0;
    }
    return reservedPool.getAssignedQueries() > 0 && generalPool.getBlockedNodes() > 0;
}