Examples with QueryId io.trino.spi.QueryId used on opensource projects

Example 81 with QueryId

use of io.trino.spi.QueryId in project trino by trinodb.

the class TestEventListenerBasic method testOutputStats.

@Test
public void testOutputStats() throws Exception {
    MaterializedResult result = runQueryAndWaitForEvents("SELECT 1 FROM lineitem", 2);
    QueryCreatedEvent queryCreatedEvent = getOnlyElement(generatedEvents.getQueryCreatedEvents());
    QueryCompletedEvent queryCompletedEvent = getOnlyElement(generatedEvents.getQueryCompletedEvents());
    QueryStats queryStats = getDistributedQueryRunner().getCoordinator().getQueryManager().getFullQueryInfo(new QueryId(queryCreatedEvent.getMetadata().getQueryId())).getQueryStats();
    assertTrue(queryStats.getOutputDataSize().toBytes() > 0L);
    assertTrue(queryCompletedEvent.getStatistics().getOutputBytes() > 0L);
    assertEquals(result.getRowCount(), queryStats.getOutputPositions());
    assertEquals(result.getRowCount(), queryCompletedEvent.getStatistics().getOutputRows());
    runQueryAndWaitForEvents("SELECT COUNT(1) FROM lineitem", 2);
    queryCreatedEvent = getOnlyElement(generatedEvents.getQueryCreatedEvents());
    queryCompletedEvent = getOnlyElement(generatedEvents.getQueryCompletedEvents());
    queryStats = getDistributedQueryRunner().getCoordinator().getQueryManager().getFullQueryInfo(new QueryId(queryCreatedEvent.getMetadata().getQueryId())).getQueryStats();
    assertTrue(queryStats.getOutputDataSize().toBytes() > 0L);
    assertTrue(queryCompletedEvent.getStatistics().getOutputBytes() > 0L);
    assertEquals(1L, queryStats.getOutputPositions());
    assertEquals(1L, queryCompletedEvent.getStatistics().getOutputRows());
    // Ensure the proper conversion in QueryMonitor#createQueryStatistics
    QueryStatistics statistics = queryCompletedEvent.getStatistics();
    assertEquals(statistics.getCpuTime().toMillis(), queryStats.getTotalCpuTime().toMillis());
    assertEquals(statistics.getWallTime().toMillis(), queryStats.getElapsedTime().toMillis());
    assertEquals(statistics.getQueuedTime().toMillis(), queryStats.getQueuedTime().toMillis());
    assertEquals(statistics.getScheduledTime().get().toMillis(), queryStats.getTotalScheduledTime().toMillis());
    assertEquals(statistics.getResourceWaitingTime().get().toMillis(), queryStats.getResourceWaitingTime().toMillis());
    assertEquals(statistics.getAnalysisTime().get().toMillis(), queryStats.getAnalysisTime().toMillis());
    assertEquals(statistics.getPlanningTime().get().toMillis(), queryStats.getPlanningTime().toMillis());
    assertEquals(statistics.getExecutionTime().get().toMillis(), queryStats.getExecutionTime().toMillis());
    assertEquals(statistics.getPeakUserMemoryBytes(), queryStats.getPeakUserMemoryReservation().toBytes());
    assertEquals(statistics.getPeakTaskUserMemory(), queryStats.getPeakTaskUserMemory().toBytes());
    assertEquals(statistics.getPeakTaskTotalMemory(), queryStats.getPeakTaskTotalMemory().toBytes());
    assertEquals(statistics.getPhysicalInputBytes(), queryStats.getPhysicalInputDataSize().toBytes());
    assertEquals(statistics.getPhysicalInputRows(), queryStats.getPhysicalInputPositions());
    assertEquals(statistics.getInternalNetworkBytes(), queryStats.getInternalNetworkInputDataSize().toBytes());
    assertEquals(statistics.getInternalNetworkRows(), queryStats.getInternalNetworkInputPositions());
    assertEquals(statistics.getTotalBytes(), queryStats.getRawInputDataSize().toBytes());
    assertEquals(statistics.getTotalRows(), queryStats.getRawInputPositions());
    assertEquals(statistics.getOutputBytes(), queryStats.getOutputDataSize().toBytes());
    assertEquals(statistics.getOutputRows(), queryStats.getOutputPositions());
    assertEquals(statistics.getWrittenBytes(), queryStats.getLogicalWrittenDataSize().toBytes());
    assertEquals(statistics.getWrittenRows(), queryStats.getWrittenPositions());
    assertEquals(statistics.getCumulativeMemory(), queryStats.getCumulativeUserMemory());
    assertEquals(statistics.getStageGcStatistics(), queryStats.getStageGcStatistics());
    assertEquals(statistics.getCompletedSplits(), queryStats.getCompletedDrivers());
}

Example 82 with QueryId

use of io.trino.spi.QueryId in project trino by trinodb.

the class TestClusterMemoryLeakDetector method testLeakDetector.

@Test
public void testLeakDetector() {
    QueryId testQuery = new QueryId("test");
    ClusterMemoryLeakDetector leakDetector = new ClusterMemoryLeakDetector();
    leakDetector.checkForMemoryLeaks(ImmutableList::of, ImmutableMap.of());
    assertEquals(leakDetector.getNumberOfLeakedQueries(), 0);
    // the leak detector should report no leaked queries as the query is still running
    leakDetector.checkForMemoryLeaks(() -> ImmutableList.of(createQueryInfo(testQuery.getId(), RUNNING)), ImmutableMap.of(testQuery, 1L));
    assertEquals(leakDetector.getNumberOfLeakedQueries(), 0);
    // the leak detector should report exactly one leaked query since the query is finished, and its end time is way in the past
    leakDetector.checkForMemoryLeaks(() -> ImmutableList.of(createQueryInfo(testQuery.getId(), FINISHED)), ImmutableMap.of(testQuery, 1L));
    assertEquals(leakDetector.getNumberOfLeakedQueries(), 1);
    // the leak detector should report no leaked queries as the query doesn't have any memory reservation
    leakDetector.checkForMemoryLeaks(() -> ImmutableList.of(createQueryInfo(testQuery.getId(), FINISHED)), ImmutableMap.of(testQuery, 0L));
    assertEquals(leakDetector.getNumberOfLeakedQueries(), 0);
    // the leak detector should report exactly one leaked query since the coordinator doesn't know of any query
    leakDetector.checkForMemoryLeaks(ImmutableList::of, ImmutableMap.of(testQuery, 1L));
    assertEquals(leakDetector.getNumberOfLeakedQueries(), 1);
}

Example 83 with QueryId

use of io.trino.spi.QueryId in project trino by trinodb.

the class TestMemoryManager method testClusterPools.

@Test(timeOut = 240_000)
public void testClusterPools() throws Exception {
    Map<String, String> properties = ImmutableMap.<String, String>builder().put("task.verbose-stats", "true").buildOrThrow();
    try (DistributedQueryRunner queryRunner = createQueryRunner(TINY_SESSION, properties)) {
        // Reserve all the memory
        QueryId fakeQueryId = new QueryId("fake");
        for (TestingTrinoServer server : queryRunner.getServers()) {
            MemoryPool pool = server.getLocalMemoryManager().getMemoryPool();
            assertTrue(pool.tryReserve(fakeQueryId, "test", pool.getMaxBytes()));
        }
        List<Future<?>> queryFutures = new ArrayList<>();
        for (int i = 0; i < 2; i++) {
            queryFutures.add(executor.submit(() -> queryRunner.execute("SELECT COUNT(*), clerk FROM orders GROUP BY clerk")));
        }
        ClusterMemoryManager memoryManager = queryRunner.getCoordinator().getClusterMemoryManager();
        ClusterMemoryPool clusterPool = memoryManager.getPool();
        assertNotNull(clusterPool);
        // Wait for the pools to become blocked
        while (clusterPool.getBlockedNodes() != 2) {
            MILLISECONDS.sleep(10);
        }
        // Ger query infos for both queries
        List<BasicQueryInfo> currentQueryInfos = queryRunner.getCoordinator().getQueryManager().getQueries();
        while (currentQueryInfos.size() != 2) {
            MILLISECONDS.sleep(10);
            currentQueryInfos = queryRunner.getCoordinator().getQueryManager().getQueries();
        }
        // Make sure the queries are blocked
        for (BasicQueryInfo info : currentQueryInfos) {
            assertFalse(info.getState().isDone());
        }
        while (!currentQueryInfos.stream().allMatch(TestMemoryManager::isBlockedWaitingForMemory)) {
            MILLISECONDS.sleep(10);
            currentQueryInfos = queryRunner.getCoordinator().getQueryManager().getQueries();
            for (BasicQueryInfo info : currentQueryInfos) {
                assertFalse(info.getState().isDone());
            }
        }
        // Release the memory in the memory pool
        for (TestingTrinoServer server : queryRunner.getServers()) {
            MemoryPool pool = server.getLocalMemoryManager().getMemoryPool();
            // Free up the entire pool
            pool.free(fakeQueryId, "test", pool.getMaxBytes());
            assertTrue(pool.getFreeBytes() > 0);
        }
        // Make sure both queries finish now that there's memory free in the memory pool.
        for (Future<?> query : queryFutures) {
            query.get();
        }
        for (BasicQueryInfo info : queryRunner.getCoordinator().getQueryManager().getQueries()) {
            assertEquals(info.getState(), FINISHED);
        }
        // Make sure we didn't leak any memory on the workers
        for (TestingTrinoServer worker : queryRunner.getServers()) {
            MemoryPool pool = worker.getLocalMemoryManager().getMemoryPool();
            assertEquals(pool.getMaxBytes(), pool.getFreeBytes());
        }
    }
}

Example 84 with QueryId

use of io.trino.spi.QueryId in project trino by trinodb.

the class TestExecutionJmxMetrics method testQueryStats.

@Test(timeOut = 30_000)
public void testQueryStats() throws Exception {
    try (DistributedQueryRunner queryRunner = TpchQueryRunnerBuilder.builder().setAdditionalModule(new PrefixObjectNameGeneratorModule("io.trino")).build()) {
        queryRunner.installPlugin(new ResourceGroupManagerPlugin());
        InternalResourceGroupManager<?> resourceGroupManager = queryRunner.getCoordinator().getResourceGroupManager().orElseThrow(() -> new IllegalStateException("Resource manager not configured"));
        resourceGroupManager.setConfigurationManager("file", ImmutableMap.of("resource-groups.config-file", getClass().getClassLoader().getResource("resource_groups_single_query.json").getPath()));
        MBeanServer mbeanServer = queryRunner.getCoordinator().getMbeanServer();
        QueryId firstDashboardQuery = createQuery(queryRunner, dashboardSession(), LONG_RUNNING_QUERY);
        waitForQueryState(queryRunner, firstDashboardQuery, RUNNING);
        assertEquals(getMbeanAttribute(mbeanServer, "RunningQueries"), 1);
        assertEquals(getMbeanAttribute(mbeanServer, "QueuedQueries"), 0);
        // the second "dashboard" query can't run right away because the resource group has a hardConcurrencyLimit of 1
        QueryId secondDashboardQuery = createQuery(queryRunner, dashboardSession(), LONG_RUNNING_QUERY);
        waitForQueryState(queryRunner, secondDashboardQuery, QUEUED);
        assertEquals(getMbeanAttribute(mbeanServer, "RunningQueries"), 1);
        assertEquals(getMbeanAttribute(mbeanServer, "QueuedQueries"), 1);
        cancelQuery(queryRunner, secondDashboardQuery);
        waitForQueryState(queryRunner, secondDashboardQuery, FAILED);
        assertEquals(getMbeanAttribute(mbeanServer, "RunningQueries"), 1);
        assertEquals(getMbeanAttribute(mbeanServer, "QueuedQueries"), 0);
        // cancel the running query to avoid polluting the logs with meaningless stack traces
        try {
            cancelQuery(queryRunner, firstDashboardQuery);
            waitForQueryState(queryRunner, firstDashboardQuery, FAILED);
        } catch (Exception ignore) {
        }
    }
}

Example 85 with QueryId

use of io.trino.spi.QueryId in project trino by trinodb.

the class TestFinalQueryInfo method startQuery.

private static QueryId startQuery(String sql, DistributedQueryRunner queryRunner) {
    OkHttpClient httpClient = new OkHttpClient();
    try {
        ClientSession clientSession = new ClientSession(queryRunner.getCoordinator().getBaseUrl(), "user", Optional.empty(), "source", Optional.empty(), ImmutableSet.of(), null, null, null, null, ZoneId.of("America/Los_Angeles"), Locale.ENGLISH, ImmutableMap.of(), ImmutableMap.of(), ImmutableMap.of(), ImmutableMap.of(), ImmutableMap.of(), null, new Duration(2, MINUTES), true);
        // start query
        StatementClient client = newStatementClient(httpClient, clientSession, sql);
        // wait for query to be fully scheduled
        while (client.isRunning() && !client.currentStatusInfo().getStats().isScheduled()) {
            client.advance();
        }
        return new QueryId(client.currentStatusInfo().getId());
    } finally {
        // close the client since, query is not managed by the client protocol
        httpClient.dispatcher().executorService().shutdown();
        httpClient.connectionPool().evictAll();
    }
}