Examples with BasicQueryInfo io.prestosql.server.BasicQueryInfo used on opensource projects

Example 1 with BasicQueryInfo

use of io.prestosql.server.BasicQueryInfo in project hetu-core by openlookeng.

the class TestMemoryManager method testClusterPools.

@Test(timeOut = 240_000)
public void testClusterPools() throws Exception {
    Map<String, String> properties = ImmutableMap.<String, String>builder().put("task.verbose-stats", "true").put("experimental.reserved-pool-enabled", "true").build();
    try (DistributedQueryRunner queryRunner = createQueryRunner(TINY_SESSION, properties)) {
        // Reserve all the memory
        QueryId fakeQueryId = new QueryId("fake");
        for (TestingPrestoServer server : queryRunner.getServers()) {
            for (MemoryPool pool : server.getLocalMemoryManager().getPools()) {
                assertTrue(pool.tryReserve(fakeQueryId, "test", pool.getMaxBytes()));
            }
        }
        List<Future<?>> queryFutures = new ArrayList<>();
        for (int i = 0; i < 2; i++) {
            queryFutures.add(executor.submit(() -> queryRunner.execute("SELECT COUNT(*), clerk FROM orders GROUP BY clerk")));
        }
        ClusterMemoryManager memoryManager = queryRunner.getCoordinator().getClusterMemoryManager();
        ClusterMemoryPool reservedPool;
        while ((reservedPool = memoryManager.getPools().get(RESERVED_POOL)) == null) {
            MILLISECONDS.sleep(10);
        }
        ClusterMemoryPool generalPool = memoryManager.getPools().get(GENERAL_POOL);
        assertNotNull(generalPool);
        // Wait for the queries to start running and get assigned to the expected pools
        while (generalPool.getAssignedQueries() != 1 || reservedPool.getAssignedQueries() != 1 || generalPool.getBlockedNodes() != 2 || reservedPool.getBlockedNodes() != 2) {
            MILLISECONDS.sleep(10);
        }
        // Make sure the queries are blocked
        List<BasicQueryInfo> currentQueryInfos = queryRunner.getCoordinator().getQueryManager().getQueries();
        for (BasicQueryInfo info : currentQueryInfos) {
            assertFalse(info.getState().isDone());
        }
        assertEquals(currentQueryInfos.size(), 2);
        // Check that the pool information propagated to the query objects
        assertNotEquals(currentQueryInfos.get(0).getMemoryPool(), currentQueryInfos.get(1).getMemoryPool());
        while (!currentQueryInfos.stream().allMatch(TestMemoryManager::isBlockedWaitingForMemory)) {
            MILLISECONDS.sleep(10);
            currentQueryInfos = queryRunner.getCoordinator().getQueryManager().getQueries();
            for (BasicQueryInfo info : currentQueryInfos) {
                assertFalse(info.getState().isDone());
            }
        }
        // Release the memory in the reserved pool
        for (TestingPrestoServer server : queryRunner.getServers()) {
            Optional<MemoryPool> reserved = server.getLocalMemoryManager().getReservedPool();
            assertTrue(reserved.isPresent());
            // Free up the entire pool
            reserved.get().free(fakeQueryId, "test", reserved.get().getMaxBytes());
            assertTrue(reserved.get().getFreeBytes() > 0);
        }
        // This also checks that the query in the general pool is successfully moved to the reserved pool.
        for (Future<?> query : queryFutures) {
            query.get();
        }
        for (BasicQueryInfo info : queryRunner.getCoordinator().getQueryManager().getQueries()) {
            assertEquals(info.getState(), FINISHED);
        }
        // Make sure we didn't leak any memory on the workers
        for (TestingPrestoServer worker : queryRunner.getServers()) {
            Optional<MemoryPool> reserved = worker.getLocalMemoryManager().getReservedPool();
            assertTrue(reserved.isPresent());
            assertEquals(reserved.get().getMaxBytes(), reserved.get().getFreeBytes());
            MemoryPool general = worker.getLocalMemoryManager().getGeneralPool();
            // Free up the memory we reserved earlier
            general.free(fakeQueryId, "test", general.getMaxBytes());
            assertEquals(general.getMaxBytes(), general.getFreeBytes());
        }
    }
}

Example 2 with BasicQueryInfo

use of io.prestosql.server.BasicQueryInfo in project hetu-core by openlookeng.

the class TestMemoryManager method testNoLeak.

private void testNoLeak(@Language("SQL") String query) throws Exception {
    Map<String, String> properties = ImmutableMap.<String, String>builder().put("task.verbose-stats", "true").put("experimental.reserved-pool-enabled", "true").build();
    try (DistributedQueryRunner queryRunner = createQueryRunner(TINY_SESSION, properties)) {
        executor.submit(() -> queryRunner.execute(query)).get();
        for (BasicQueryInfo info : queryRunner.getCoordinator().getQueryManager().getQueries()) {
            assertEquals(info.getState(), FINISHED);
        }
        // Make sure we didn't leak any memory on the workers
        for (TestingPrestoServer worker : queryRunner.getServers()) {
            Optional<MemoryPool> reserved = worker.getLocalMemoryManager().getReservedPool();
            assertTrue(reserved.isPresent());
            assertEquals(reserved.get().getMaxBytes(), reserved.get().getFreeBytes());
            MemoryPool general = worker.getLocalMemoryManager().getGeneralPool();
            assertEquals(general.getMaxBytes(), general.getFreeBytes());
        }
    }
}

Example 3 with BasicQueryInfo

use of io.prestosql.server.BasicQueryInfo in project hetu-core by openlookeng.

the class TestQueryManager method testQueryCpuLimit.

@Test(timeOut = 60_000L)
public void testQueryCpuLimit() throws Exception {
    try (DistributedQueryRunner distributedQueryRunner = TpchQueryRunnerBuilder.builder().setSingleExtraProperty("query.max-cpu-time", "1ms").build()) {
        QueryId queryId = createQuery(distributedQueryRunner, TEST_SESSION, "SELECT COUNT(*) FROM lineitem");
        waitForQueryState(distributedQueryRunner, queryId, FAILED);
        QueryManager queryManager = distributedQueryRunner.getCoordinator().getQueryManager();
        BasicQueryInfo queryInfo = queryManager.getQueryInfo(queryId);
        assertEquals(queryInfo.getState(), FAILED);
        assertEquals(queryInfo.getErrorCode(), EXCEEDED_CPU_LIMIT.toErrorCode());
    }
}

Example 4 with BasicQueryInfo

use of io.prestosql.server.BasicQueryInfo in project hetu-core by openlookeng.

the class TestNodeStateChange method testCoordinatorShutdown.

@Test
public void testCoordinatorShutdown() throws Exception {
    try (DistributedQueryRunner queryRunner = createQueryRunner(TINY_SESSION, properties)) {
        List<ListenableFuture<?>> queryFutures = new ArrayList<>();
        for (int i = 0; i < 5; i++) {
            queryFutures.add(executor.submit(() -> queryRunner.execute("SELECT COUNT(*), clerk FROM orders GROUP BY clerk")));
        }
        TestingPrestoServer coordinator = queryRunner.getServers().stream().filter(TestingPrestoServer::isCoordinator).findFirst().get();
        QueryManager queryManager = coordinator.getQueryManager();
        // wait until queries show up on the coordinator
        while (queryManager.getQueries().isEmpty()) {
            MILLISECONDS.sleep(500);
        }
        coordinator.getNodeStateChangeHandler().doStateTransition(NodeState.SHUTTING_DOWN);
        Futures.allAsList(queryFutures).get();
        List<BasicQueryInfo> queryInfos = queryRunner.getCoordinator().getQueryManager().getQueries();
        for (BasicQueryInfo info : queryInfos) {
            assertEquals(info.getState(), FINISHED);
        }
        TestingPrestoServer.TestShutdownAction shutdownAction = (TestingPrestoServer.TestShutdownAction) coordinator.getShutdownAction();
        shutdownAction.waitForShutdownComplete(SHUTDOWN_TIMEOUT_MILLIS2);
        assertTrue(shutdownAction.isShutdown());
    }
}

Example 5 with BasicQueryInfo

use of io.prestosql.server.BasicQueryInfo in project hetu-core by openlookeng.

the class AbstractTestDistributedQueries method testQueryLoggingCount.

@Test
public void testQueryLoggingCount() {
    QueryManager queryManager = ((DistributedQueryRunner) getQueryRunner()).getCoordinator().getQueryManager();
    executeExclusively(() -> {
        assertUntilTimeout(() -> assertEquals(queryManager.getQueries().stream().map(BasicQueryInfo::getQueryId).map(queryManager::getFullQueryInfo).filter(info -> !info.isFinalQueryInfo()).collect(toList()), ImmutableList.of()), new Duration(1, MINUTES));
        // We cannot simply get the number of completed queries as soon as all the queries are completed, because this counter may not be up-to-date at that point.
        // The completed queries counter is updated in a final query info listener, which is called eventually.
        // Therefore, here we wait until the value of this counter gets stable.
        DispatchManager dispatchManager = ((DistributedQueryRunner) getQueryRunner()).getCoordinator().getDispatchManager();
        long beforeCompletedQueriesCount = waitUntilStable(() -> dispatchManager.getStats().getCompletedQueries().getTotalCount(), new Duration(5, SECONDS));
        long beforeSubmittedQueriesCount = dispatchManager.getStats().getSubmittedQueries().getTotalCount();
        assertUpdate("CREATE TABLE test_query_logging_count AS SELECT 1 foo_1, 2 foo_2_4", 1);
        assertQuery("SELECT foo_1, foo_2_4 FROM test_query_logging_count", "SELECT 1, 2");
        assertUpdate("DROP TABLE test_query_logging_count");
        assertQueryFails("SELECT * FROM test_query_logging_count", ".*Table .* does not exist");
        // TODO: Figure out a better way of synchronization
        assertUntilTimeout(() -> assertEquals(dispatchManager.getStats().getCompletedQueries().getTotalCount() - beforeCompletedQueriesCount, 4), new Duration(1, MINUTES));
        assertEquals(dispatchManager.getStats().getSubmittedQueries().getTotalCount() - beforeSubmittedQueriesCount, 4);
    });
}