Examples with QueryId io.prestosql.spi.QueryId used on opensource projects

Example 6 with QueryId

use of io.prestosql.spi.QueryId in project hetu-core by openlookeng.

the class TestQueuesDb method testTooManyQueries.

@Test(timeOut = 90_000)
public void testTooManyQueries() throws Exception {
    QueryId firstDashboardQuery = createQuery(queryRunner, dashboardSession(), LONG_LASTING_QUERY);
    waitForQueryState(queryRunner, firstDashboardQuery, RUNNING);
    QueryId secondDashboardQuery = createQuery(queryRunner, dashboardSession(), LONG_LASTING_QUERY);
    waitForQueryState(queryRunner, secondDashboardQuery, QUEUED);
    QueryId thirdDashboardQuery = createQuery(queryRunner, dashboardSession(), LONG_LASTING_QUERY);
    waitForQueryState(queryRunner, thirdDashboardQuery, FAILED);
    // Allow one more query to run and resubmit third query
    // Hetu: add parameters softReservedMemory and hardReservedConcurrency
    dao.updateResourceGroup(3, "user-${USER}", "1MB", "10%", 3, 4, 4, 4, null, null, null, null, null, "RECENT_QUERIES", 1L, TEST_ENVIRONMENT);
    dao.updateResourceGroup(5, "dashboard-${USER}", "1MB", "10%", 1, 2, 2, 2, null, null, null, null, null, "RECENT_QUERIES", 3L, TEST_ENVIRONMENT);
    InternalResourceGroupManager<?> manager = queryRunner.getCoordinator().getResourceGroupManager().get();
    DbResourceGroupConfigurationManager dbConfigurationManager = (DbResourceGroupConfigurationManager) manager.getConfigurationManager();
    // Trigger reload to make the test more deterministic
    dbConfigurationManager.load();
    waitForQueryState(queryRunner, secondDashboardQuery, RUNNING);
    thirdDashboardQuery = createQuery(queryRunner, dashboardSession(), LONG_LASTING_QUERY);
    waitForQueryState(queryRunner, thirdDashboardQuery, QUEUED);
    // Lower running queries in dashboard resource groups and reload the config
    dao.updateResourceGroup(5, "dashboard-${USER}", "1MB", "10%", 1, 1, 1, 1, null, null, null, null, null, "RECENT_QUERIES", 3L, TEST_ENVIRONMENT);
    dbConfigurationManager.load();
    // Cancel query and verify that third query is still queued
    cancelQuery(queryRunner, firstDashboardQuery);
    waitForQueryState(queryRunner, firstDashboardQuery, FAILED);
    MILLISECONDS.sleep(2000);
    waitForQueryState(queryRunner, thirdDashboardQuery, QUEUED);
}

Example 7 with QueryId

use of io.prestosql.spi.QueryId in project hetu-core by openlookeng.

the class TestClusterMemoryLeakDetector method testLeakDetector.

@Test
public void testLeakDetector() {
    QueryId testQuery = new QueryId("test");
    ClusterMemoryLeakDetector leakDetector = new ClusterMemoryLeakDetector();
    leakDetector.checkForMemoryLeaks(ImmutableList::of, ImmutableMap.of());
    assertEquals(leakDetector.getNumberOfLeakedQueries(), 0);
    // the leak detector should report no leaked queries as the query is still running
    leakDetector.checkForMemoryLeaks(() -> ImmutableList.of(createQueryInfo(testQuery.getId(), RUNNING)), ImmutableMap.of(testQuery, 1L));
    assertEquals(leakDetector.getNumberOfLeakedQueries(), 0);
    // the leak detector should report exactly one leaked query since the query is finished, and its end time is way in the past
    leakDetector.checkForMemoryLeaks(() -> ImmutableList.of(createQueryInfo(testQuery.getId(), FINISHED)), ImmutableMap.of(testQuery, 1L));
    assertEquals(leakDetector.getNumberOfLeakedQueries(), 1);
    // the leak detector should report no leaked queries as the query doesn't have any memory reservation
    leakDetector.checkForMemoryLeaks(() -> ImmutableList.of(createQueryInfo(testQuery.getId(), FINISHED)), ImmutableMap.of(testQuery, 0L));
    assertEquals(leakDetector.getNumberOfLeakedQueries(), 0);
    // the leak detector should report exactly one leaked query since the coordinator doesn't know of any query
    leakDetector.checkForMemoryLeaks(ImmutableList::of, ImmutableMap.of(testQuery, 1L));
    assertEquals(leakDetector.getNumberOfLeakedQueries(), 1);
}

Example 8 with QueryId

use of io.prestosql.spi.QueryId in project hetu-core by openlookeng.

the class TestMemoryManager method testClusterPools.

@Test(timeOut = 240_000)
public void testClusterPools() throws Exception {
    Map<String, String> properties = ImmutableMap.<String, String>builder().put("task.verbose-stats", "true").put("experimental.reserved-pool-enabled", "true").build();
    try (DistributedQueryRunner queryRunner = createQueryRunner(TINY_SESSION, properties)) {
        // Reserve all the memory
        QueryId fakeQueryId = new QueryId("fake");
        for (TestingPrestoServer server : queryRunner.getServers()) {
            for (MemoryPool pool : server.getLocalMemoryManager().getPools()) {
                assertTrue(pool.tryReserve(fakeQueryId, "test", pool.getMaxBytes()));
            }
        }
        List<Future<?>> queryFutures = new ArrayList<>();
        for (int i = 0; i < 2; i++) {
            queryFutures.add(executor.submit(() -> queryRunner.execute("SELECT COUNT(*), clerk FROM orders GROUP BY clerk")));
        }
        ClusterMemoryManager memoryManager = queryRunner.getCoordinator().getClusterMemoryManager();
        ClusterMemoryPool reservedPool;
        while ((reservedPool = memoryManager.getPools().get(RESERVED_POOL)) == null) {
            MILLISECONDS.sleep(10);
        }
        ClusterMemoryPool generalPool = memoryManager.getPools().get(GENERAL_POOL);
        assertNotNull(generalPool);
        // Wait for the queries to start running and get assigned to the expected pools
        while (generalPool.getAssignedQueries() != 1 || reservedPool.getAssignedQueries() != 1 || generalPool.getBlockedNodes() != 2 || reservedPool.getBlockedNodes() != 2) {
            MILLISECONDS.sleep(10);
        }
        // Make sure the queries are blocked
        List<BasicQueryInfo> currentQueryInfos = queryRunner.getCoordinator().getQueryManager().getQueries();
        for (BasicQueryInfo info : currentQueryInfos) {
            assertFalse(info.getState().isDone());
        }
        assertEquals(currentQueryInfos.size(), 2);
        // Check that the pool information propagated to the query objects
        assertNotEquals(currentQueryInfos.get(0).getMemoryPool(), currentQueryInfos.get(1).getMemoryPool());
        while (!currentQueryInfos.stream().allMatch(TestMemoryManager::isBlockedWaitingForMemory)) {
            MILLISECONDS.sleep(10);
            currentQueryInfos = queryRunner.getCoordinator().getQueryManager().getQueries();
            for (BasicQueryInfo info : currentQueryInfos) {
                assertFalse(info.getState().isDone());
            }
        }
        // Release the memory in the reserved pool
        for (TestingPrestoServer server : queryRunner.getServers()) {
            Optional<MemoryPool> reserved = server.getLocalMemoryManager().getReservedPool();
            assertTrue(reserved.isPresent());
            // Free up the entire pool
            reserved.get().free(fakeQueryId, "test", reserved.get().getMaxBytes());
            assertTrue(reserved.get().getFreeBytes() > 0);
        }
        // This also checks that the query in the general pool is successfully moved to the reserved pool.
        for (Future<?> query : queryFutures) {
            query.get();
        }
        for (BasicQueryInfo info : queryRunner.getCoordinator().getQueryManager().getQueries()) {
            assertEquals(info.getState(), FINISHED);
        }
        // Make sure we didn't leak any memory on the workers
        for (TestingPrestoServer worker : queryRunner.getServers()) {
            Optional<MemoryPool> reserved = worker.getLocalMemoryManager().getReservedPool();
            assertTrue(reserved.isPresent());
            assertEquals(reserved.get().getMaxBytes(), reserved.get().getFreeBytes());
            MemoryPool general = worker.getLocalMemoryManager().getGeneralPool();
            // Free up the memory we reserved earlier
            general.free(fakeQueryId, "test", general.getMaxBytes());
            assertEquals(general.getMaxBytes(), general.getFreeBytes());
        }
    }
}

Example 9 with QueryId

use of io.prestosql.spi.QueryId in project hetu-core by openlookeng.

the class TestMemoryManager method testOutOfMemoryKiller.

@Test(timeOut = 240_000, expectedExceptions = ExecutionException.class, expectedExceptionsMessageRegExp = ".*Query killed because the cluster is out of memory. Please try again in a few minutes.")
public void testOutOfMemoryKiller() throws Exception {
    Map<String, String> properties = ImmutableMap.<String, String>builder().put("task.verbose-stats", "true").put("query.low-memory-killer.delay", "5s").put("query.low-memory-killer.policy", "total-reservation").put("experimental.reserved-pool-enabled", "true").build();
    try (DistributedQueryRunner queryRunner = createQueryRunner(TINY_SESSION, properties)) {
        // Reserve all the memory
        QueryId fakeQueryId = new QueryId("fake");
        for (TestingPrestoServer server : queryRunner.getServers()) {
            for (MemoryPool pool : server.getLocalMemoryManager().getPools()) {
                assertTrue(pool.tryReserve(fakeQueryId, "test", pool.getMaxBytes()));
            }
        }
        List<Future<?>> queryFutures = new ArrayList<>();
        for (int i = 0; i < 2; i++) {
            queryFutures.add(executor.submit(() -> queryRunner.execute("SELECT COUNT(*), clerk FROM orders GROUP BY clerk")));
        }
        // Wait for one of the queries to die
        waitForQueryToBeKilled(queryRunner);
        // Release the memory in the reserved pool
        for (TestingPrestoServer server : queryRunner.getServers()) {
            Optional<MemoryPool> reserved = server.getLocalMemoryManager().getReservedPool();
            assertTrue(reserved.isPresent());
            // Free up the entire pool
            reserved.get().free(fakeQueryId, "test", reserved.get().getMaxBytes());
            assertTrue(reserved.get().getFreeBytes() > 0);
        }
        for (Future<?> query : queryFutures) {
            query.get();
        }
    }
}

Example 10 with QueryId

use of io.prestosql.spi.QueryId in project hetu-core by openlookeng.

the class TestMemoryManager method testReservedPoolDisabled.

@Test(timeOut = 240_000, expectedExceptions = ExecutionException.class, expectedExceptionsMessageRegExp = ".*Query killed because the cluster is out of memory. Please try again in a few minutes.")
public void testReservedPoolDisabled() throws Exception {
    Map<String, String> properties = ImmutableMap.<String, String>builder().put("experimental.reserved-pool-enabled", "false").put("query.low-memory-killer.delay", "5s").put("query.low-memory-killer.policy", "total-reservation").build();
    try (DistributedQueryRunner queryRunner = createQueryRunner(TINY_SESSION, properties)) {
        // Reserve all the memory
        QueryId fakeQueryId = new QueryId("fake");
        for (TestingPrestoServer server : queryRunner.getServers()) {
            List<MemoryPool> memoryPools = server.getLocalMemoryManager().getPools();
            assertEquals(memoryPools.size(), 1, "Only general pool should exist");
            assertTrue(memoryPools.get(0).tryReserve(fakeQueryId, "test", memoryPools.get(0).getMaxBytes()));
        }
        List<Future<?>> queryFutures = new ArrayList<>();
        for (int i = 0; i < 2; i++) {
            queryFutures.add(executor.submit(() -> queryRunner.execute("SELECT COUNT(*), clerk FROM orders GROUP BY clerk")));
        }
        // Wait for one of the queries to die
        waitForQueryToBeKilled(queryRunner);
        // Reserved pool shouldn't exist on the workers and allocation should have been done in the general pool
        for (TestingPrestoServer server : queryRunner.getServers()) {
            Optional<MemoryPool> reserved = server.getLocalMemoryManager().getReservedPool();
            MemoryPool general = server.getLocalMemoryManager().getGeneralPool();
            assertFalse(reserved.isPresent());
            assertTrue(general.getReservedBytes() > 0);
            // Free up the entire pool
            general.free(fakeQueryId, "test", general.getMaxBytes());
            assertTrue(general.getFreeBytes() > 0);
        }
        for (Future<?> query : queryFutures) {
            query.get();
        }
    }
}