Examples with ClusterMemoryPoolInfo com.facebook.presto.spi.memory.ClusterMemoryPoolInfo used on opensource projects

Example 6 with ClusterMemoryPoolInfo

use of com.facebook.presto.spi.memory.ClusterMemoryPoolInfo in project urban-eureka by errir503.

the class TestResourceManagerClusterStateProvider method assertMemoryPoolMap.

private void assertMemoryPoolMap(ResourceManagerClusterStateProvider provider, int memoryPoolSize, MemoryPoolId memoryPoolId, int assignedQueries, int blockedNodes, int maxBytes, int reservedBytes, int reservedRevocableBytes, Optional<String> largestMemoryQuery) {
    Map<MemoryPoolId, ClusterMemoryPoolInfo> memoryPoolMap = provider.getClusterMemoryPoolInfo();
    assertNotNull(memoryPoolMap);
    assertEquals(memoryPoolMap.size(), memoryPoolSize);
    ClusterMemoryPoolInfo clusterMemoryPoolInfo = memoryPoolMap.get(memoryPoolId);
    assertNotNull(clusterMemoryPoolInfo);
    assertEquals(clusterMemoryPoolInfo.getAssignedQueries(), assignedQueries);
    assertEquals(clusterMemoryPoolInfo.getBlockedNodes(), blockedNodes);
    assertEquals(clusterMemoryPoolInfo.getMemoryPoolInfo().getMaxBytes(), maxBytes);
    assertEquals(clusterMemoryPoolInfo.getMemoryPoolInfo().getReservedBytes(), reservedBytes);
    assertEquals(clusterMemoryPoolInfo.getMemoryPoolInfo().getReservedRevocableBytes(), reservedRevocableBytes);
    assertEquals(clusterMemoryPoolInfo.getLargestMemoryQuery().map(QueryId::getId), largestMemoryQuery);
}

Example 7 with ClusterMemoryPoolInfo

use of com.facebook.presto.spi.memory.ClusterMemoryPoolInfo in project presto by prestodb.

the class ResourceManagerClusterStateProvider method getClusterMemoryPoolInfoInternal.

private Map<MemoryPoolId, ClusterMemoryPoolInfo> getClusterMemoryPoolInfoInternal() {
    List<MemoryInfo> memoryInfos = nodeStatuses.values().stream().map(nodeStatus -> nodeStatus.getNodeStatus().getMemoryInfo()).collect(toImmutableList());
    int queriesAssignedToGeneralPool = 0;
    int queriesAssignedToReservedPool = 0;
    Query largestGeneralPoolQuery = null;
    for (CoordinatorQueriesState nodeQueryState : nodeQueryStates.values()) {
        for (Query query : nodeQueryState.getActiveQueries()) {
            MemoryPoolId memoryPool = query.getBasicQueryInfo().getMemoryPool();
            if (GENERAL_POOL.equals(memoryPool)) {
                queriesAssignedToGeneralPool = Math.incrementExact(queriesAssignedToGeneralPool);
                if (!resourceOvercommit(query.getBasicQueryInfo().getSession().toSession(sessionPropertyManager))) {
                    largestGeneralPoolQuery = getLargestMemoryQuery(Optional.ofNullable(largestGeneralPoolQuery), query);
                }
            } else if (RESERVED_POOL.equals(memoryPool)) {
                queriesAssignedToReservedPool = Math.incrementExact(queriesAssignedToReservedPool);
            } else {
                throw new IllegalArgumentException("Unrecognized memory pool: " + memoryPool);
            }
        }
    }
    ImmutableMap.Builder<MemoryPoolId, ClusterMemoryPoolInfo> memoryPoolInfos = ImmutableMap.builder();
    ClusterMemoryPool pool = new ClusterMemoryPool(GENERAL_POOL);
    pool.update(memoryInfos, queriesAssignedToGeneralPool);
    ClusterMemoryPoolInfo clusterInfo = pool.getClusterInfo(Optional.ofNullable(largestGeneralPoolQuery).map(Query::getQueryId));
    memoryPoolInfos.put(GENERAL_POOL, clusterInfo);
    if (isReservedPoolEnabled) {
        pool = new ClusterMemoryPool(RESERVED_POOL);
        pool.update(memoryInfos, queriesAssignedToReservedPool);
        memoryPoolInfos.put(RESERVED_POOL, pool.getClusterInfo());
    }
    return memoryPoolInfos.build();
}

Example 8 with ClusterMemoryPoolInfo

use of com.facebook.presto.spi.memory.ClusterMemoryPoolInfo in project presto by prestodb.

the class ClusterMemoryManager method isClusterOutOfMemory.

private synchronized boolean isClusterOutOfMemory() {
    ClusterMemoryPoolInfo reservedPool = getClusterInfo(RESERVED_POOL);
    ClusterMemoryPoolInfo generalPool = getClusterInfo(GENERAL_POOL);
    if (reservedPool == null) {
        return generalPool.getBlockedNodes() > 0;
    }
    return reservedPool.getAssignedQueries() > 0 && generalPool.getBlockedNodes() > 0;
}

Example 9 with ClusterMemoryPoolInfo

use of com.facebook.presto.spi.memory.ClusterMemoryPoolInfo in project presto by prestodb.

the class TestMemoryManager method testClusterPoolsMultiCoordinator.

@Test(timeOut = 60_000)
public void testClusterPoolsMultiCoordinator() throws Exception {
    Map<String, String> properties = ImmutableMap.<String, String>builder().put("task.verbose-stats", "true").put("resource-manager.memory-pool-fetch-interval", "10ms").put("resource-manager.query-heartbeat-interval", "10ms").put("resource-manager.node-status-timeout", "5s").build();
    try (DistributedQueryRunner queryRunner = createQueryRunner(properties, ImmutableMap.of(), properties, 2)) {
        // Reserve all the memory
        QueryId fakeQueryId = new QueryId("fake");
        for (TestingPrestoServer server : queryRunner.getCoordinatorWorkers()) {
            for (MemoryPool pool : server.getLocalMemoryManager().getPools()) {
                assertTrue(pool.tryReserve(fakeQueryId, "test", pool.getMaxBytes()));
            }
        }
        List<Future<?>> queryFutures = new ArrayList<>();
        for (int i = 0; i < 2; i++) {
            int coordinator = i;
            queryFutures.add(executor.submit(() -> queryRunner.execute(coordinator, "SELECT COUNT(*), clerk FROM orders GROUP BY clerk")));
        }
        ClusterMemoryManager memoryManager = queryRunner.getCoordinator(0).getClusterMemoryManager();
        ClusterMemoryPoolInfo reservedPool;
        while ((reservedPool = memoryManager.getClusterInfo(RESERVED_POOL)) == null) {
            MILLISECONDS.sleep(10);
        }
        ClusterMemoryPoolInfo generalPool = memoryManager.getClusterInfo(GENERAL_POOL);
        assertNotNull(generalPool);
        // Wait for the queries to start running and get assigned to the expected pools
        while (generalPool.getAssignedQueries() != 1 || reservedPool.getAssignedQueries() != 1 || generalPool.getBlockedNodes() != 3 || reservedPool.getBlockedNodes() != 3) {
            generalPool = memoryManager.getClusterInfo(GENERAL_POOL);
            reservedPool = memoryManager.getClusterInfo(RESERVED_POOL);
            MILLISECONDS.sleep(10);
        }
        // Make sure the queries are blocked
        List<BasicQueryInfo> currentQueryInfos;
        do {
            currentQueryInfos = queryRunner.getCoordinators().stream().map(TestingPrestoServer::getQueryManager).map(QueryManager::getQueries).flatMap(Collection::stream).collect(toImmutableList());
            MILLISECONDS.sleep(10);
        } while (currentQueryInfos.size() != 2);
        for (BasicQueryInfo info : currentQueryInfos) {
            assertFalse(info.getState().isDone());
        }
        // Check that the pool information propagated to the query objects
        assertNotEquals(currentQueryInfos.get(0).getMemoryPool(), currentQueryInfos.get(1).getMemoryPool());
        while (!currentQueryInfos.stream().allMatch(TestMemoryManager::isBlockedWaitingForMemory)) {
            MILLISECONDS.sleep(10);
            currentQueryInfos = queryRunner.getCoordinators().stream().map(TestingPrestoServer::getQueryManager).map(QueryManager::getQueries).flatMap(Collection::stream).collect(toImmutableList());
            for (BasicQueryInfo info : currentQueryInfos) {
                assertFalse(info.getState().isDone());
            }
        }
        // Release the memory in the reserved pool
        for (TestingPrestoServer server : queryRunner.getCoordinatorWorkers()) {
            Optional<MemoryPool> reserved = server.getLocalMemoryManager().getReservedPool();
            assertTrue(reserved.isPresent());
            // Free up the entire pool
            reserved.get().free(fakeQueryId, "test", reserved.get().getMaxBytes());
            assertTrue(reserved.get().getFreeBytes() > 0);
        }
        // This also checks that the query in the general pool is successfully moved to the reserved pool.
        for (Future<?> query : queryFutures) {
            query.get();
        }
        queryRunner.getCoordinators().stream().map(TestingPrestoServer::getQueryManager).map(QueryManager::getQueries).flatMap(Collection::stream).forEach(info -> assertEquals(info.getState(), FINISHED));
        // Make sure we didn't leak any memory on the workers
        for (TestingPrestoServer worker : queryRunner.getCoordinatorWorkers()) {
            Optional<MemoryPool> reserved = worker.getLocalMemoryManager().getReservedPool();
            assertTrue(reserved.isPresent());
            assertEquals(reserved.get().getMaxBytes(), reserved.get().getFreeBytes());
            MemoryPool general = worker.getLocalMemoryManager().getGeneralPool();
            // Free up the memory we reserved earlier
            general.free(fakeQueryId, "test", general.getMaxBytes());
            assertEquals(general.getMaxBytes(), general.getFreeBytes());
        }
    }
}

Example 10 with ClusterMemoryPoolInfo

use of com.facebook.presto.spi.memory.ClusterMemoryPoolInfo in project urban-eureka by errir503.

the class ResourceManagerClusterStateProvider method getClusterMemoryPoolInfoInternal.

private Map<MemoryPoolId, ClusterMemoryPoolInfo> getClusterMemoryPoolInfoInternal() {
    List<MemoryInfo> memoryInfos = nodeStatuses.values().stream().map(nodeStatus -> nodeStatus.getNodeStatus().getMemoryInfo()).collect(toImmutableList());
    int queriesAssignedToGeneralPool = 0;
    int queriesAssignedToReservedPool = 0;
    Query largestGeneralPoolQuery = null;
    for (CoordinatorQueriesState nodeQueryState : nodeQueryStates.values()) {
        for (Query query : nodeQueryState.getActiveQueries()) {
            MemoryPoolId memoryPool = query.getBasicQueryInfo().getMemoryPool();
            if (GENERAL_POOL.equals(memoryPool)) {
                queriesAssignedToGeneralPool = Math.incrementExact(queriesAssignedToGeneralPool);
                if (!resourceOvercommit(query.getBasicQueryInfo().getSession().toSession(sessionPropertyManager))) {
                    largestGeneralPoolQuery = getLargestMemoryQuery(Optional.ofNullable(largestGeneralPoolQuery), query);
                }
            } else if (RESERVED_POOL.equals(memoryPool)) {
                queriesAssignedToReservedPool = Math.incrementExact(queriesAssignedToReservedPool);
            } else {
                throw new IllegalArgumentException("Unrecognized memory pool: " + memoryPool);
            }
        }
    }
    ImmutableMap.Builder<MemoryPoolId, ClusterMemoryPoolInfo> memoryPoolInfos = ImmutableMap.builder();
    ClusterMemoryPool pool = new ClusterMemoryPool(GENERAL_POOL);
    pool.update(memoryInfos, queriesAssignedToGeneralPool);
    ClusterMemoryPoolInfo clusterInfo = pool.getClusterInfo(Optional.ofNullable(largestGeneralPoolQuery).map(Query::getQueryId));
    memoryPoolInfos.put(GENERAL_POOL, clusterInfo);
    if (isReservedPoolEnabled) {
        pool = new ClusterMemoryPool(RESERVED_POOL);
        pool.update(memoryInfos, queriesAssignedToReservedPool);
        memoryPoolInfos.put(RESERVED_POOL, pool.getClusterInfo());
    }
    return memoryPoolInfos.build();
}