Examples with QueryInfo com.facebook.presto.execution.QueryInfo used on opensource projects

Example 1 with QueryInfo

use of com.facebook.presto.execution.QueryInfo in project presto by prestodb.

the class TestMemoryManager method testClusterPools.

@Test(timeOut = 240_000)
public void testClusterPools() throws Exception {
    Map<String, String> properties = ImmutableMap.<String, String>builder().put("task.verbose-stats", "true").build();
    try (DistributedQueryRunner queryRunner = createQueryRunner(TINY_SESSION, properties)) {
        // Reserve all the memory
        QueryId fakeQueryId = new QueryId("fake");
        for (TestingPrestoServer server : queryRunner.getServers()) {
            for (MemoryPool pool : server.getLocalMemoryManager().getPools()) {
                assertTrue(pool.tryReserve(fakeQueryId, pool.getMaxBytes()));
            }
        }
        List<Future<?>> queryFutures = new ArrayList<>();
        for (int i = 0; i < 2; i++) {
            queryFutures.add(executor.submit(() -> queryRunner.execute("SELECT COUNT(*), clerk FROM orders GROUP BY clerk")));
        }
        ClusterMemoryManager memoryManager = queryRunner.getCoordinator().getClusterMemoryManager();
        ClusterMemoryPool reservedPool;
        while ((reservedPool = memoryManager.getPools().get(RESERVED_POOL)) == null) {
            MILLISECONDS.sleep(10);
        }
        ClusterMemoryPool generalPool = memoryManager.getPools().get(GENERAL_POOL);
        assertNotNull(generalPool);
        // Wait for the queries to start running and get assigned to the expected pools
        while (generalPool.getAssignedQueries() != 1 || reservedPool.getAssignedQueries() != 1 || generalPool.getBlockedNodes() != 2 || reservedPool.getBlockedNodes() != 2) {
            MILLISECONDS.sleep(10);
        }
        // Make sure the queries are blocked
        List<QueryInfo> currentQueryInfos = queryRunner.getCoordinator().getQueryManager().getAllQueryInfo();
        for (QueryInfo info : currentQueryInfos) {
            assertFalse(info.getState().isDone());
        }
        assertEquals(currentQueryInfos.size(), 2);
        // Check that the pool information propagated to the query objects
        assertNotEquals(currentQueryInfos.get(0).getMemoryPool(), currentQueryInfos.get(1).getMemoryPool());
        while (!allQueriesBlocked(currentQueryInfos)) {
            MILLISECONDS.sleep(10);
            currentQueryInfos = queryRunner.getCoordinator().getQueryManager().getAllQueryInfo();
            for (QueryInfo info : currentQueryInfos) {
                assertFalse(info.getState().isDone());
            }
        }
        // Release the memory in the reserved pool and the system pool
        for (TestingPrestoServer server : queryRunner.getServers()) {
            MemoryPool reserved = server.getLocalMemoryManager().getPool(RESERVED_POOL);
            // Free up the entire pool
            reserved.free(fakeQueryId, reserved.getMaxBytes());
            assertTrue(reserved.getFreeBytes() > 0);
            MemoryPool system = server.getLocalMemoryManager().getPool(SYSTEM_POOL);
            // Free up the entire pool
            system.free(fakeQueryId, system.getMaxBytes());
            assertTrue(system.getFreeBytes() > 0);
        }
        // This also checks that the query in the general pool is successfully moved to the reserved pool.
        for (Future<?> query : queryFutures) {
            query.get();
        }
        List<QueryInfo> queryInfos = queryRunner.getCoordinator().getQueryManager().getAllQueryInfo();
        for (QueryInfo info : queryInfos) {
            assertEquals(info.getState(), FINISHED);
        }
        // Make sure we didn't leak any memory on the workers
        for (TestingPrestoServer worker : queryRunner.getServers()) {
            MemoryPool reserved = worker.getLocalMemoryManager().getPool(RESERVED_POOL);
            assertEquals(reserved.getMaxBytes(), reserved.getFreeBytes());
            MemoryPool general = worker.getLocalMemoryManager().getPool(GENERAL_POOL);
            // Free up the memory we reserved earlier
            general.free(fakeQueryId, general.getMaxBytes());
            assertEquals(general.getMaxBytes(), general.getFreeBytes());
            MemoryPool system = worker.getLocalMemoryManager().getPool(SYSTEM_POOL);
            assertEquals(system.getMaxBytes(), system.getFreeBytes());
        }
    }
}

Example 2 with QueryInfo

use of com.facebook.presto.execution.QueryInfo in project presto by prestodb.

the class TestMemoryManager method testNoLeak.

private void testNoLeak(@Language("SQL") String query) throws Exception {
    Map<String, String> properties = ImmutableMap.<String, String>builder().put("task.verbose-stats", "true").build();
    try (DistributedQueryRunner queryRunner = createQueryRunner(TINY_SESSION, properties)) {
        executor.submit(() -> queryRunner.execute(query)).get();
        List<QueryInfo> queryInfos = queryRunner.getCoordinator().getQueryManager().getAllQueryInfo();
        for (QueryInfo info : queryInfos) {
            assertEquals(info.getState(), FINISHED);
        }
        // Make sure we didn't leak any memory on the workers
        for (TestingPrestoServer worker : queryRunner.getServers()) {
            MemoryPool reserved = worker.getLocalMemoryManager().getPool(RESERVED_POOL);
            assertEquals(reserved.getMaxBytes(), reserved.getFreeBytes());
            MemoryPool general = worker.getLocalMemoryManager().getPool(GENERAL_POOL);
            assertEquals(general.getMaxBytes(), general.getFreeBytes());
            MemoryPool system = worker.getLocalMemoryManager().getPool(SYSTEM_POOL);
            assertEquals(system.getMaxBytes(), system.getFreeBytes());
        }
    }
}

Example 3 with QueryInfo

use of com.facebook.presto.execution.QueryInfo in project presto by prestodb.

the class QueryMonitor method queryCompletedEvent.

public void queryCompletedEvent(QueryInfo queryInfo) {
    try {
        Optional<QueryFailureInfo> queryFailureInfo = Optional.empty();
        if (queryInfo.getFailureInfo() != null) {
            FailureInfo failureInfo = queryInfo.getFailureInfo();
            Optional<TaskInfo> failedTask = queryInfo.getOutputStage().flatMap(QueryMonitor::findFailedTask);
            queryFailureInfo = Optional.of(new QueryFailureInfo(queryInfo.getErrorCode(), Optional.ofNullable(failureInfo.getType()), Optional.ofNullable(failureInfo.getMessage()), failedTask.map(task -> task.getTaskStatus().getTaskId().toString()), failedTask.map(task -> task.getTaskStatus().getSelf().getHost()), objectMapper.writeValueAsString(queryInfo.getFailureInfo())));
        }
        ImmutableList.Builder<QueryInputMetadata> inputs = ImmutableList.builder();
        for (Input input : queryInfo.getInputs()) {
            inputs.add(new QueryInputMetadata(input.getConnectorId().getCatalogName(), input.getSchema(), input.getTable(), input.getColumns().stream().map(Column::toString).collect(Collectors.toList()), input.getConnectorInfo()));
        }
        QueryStats queryStats = queryInfo.getQueryStats();
        Optional<QueryOutputMetadata> output = Optional.empty();
        if (queryInfo.getOutput().isPresent()) {
            Optional<TableFinishInfo> tableFinishInfo = queryStats.getOperatorSummaries().stream().map(OperatorStats::getInfo).filter(TableFinishInfo.class::isInstance).map(TableFinishInfo.class::cast).findFirst();
            output = Optional.of(new QueryOutputMetadata(queryInfo.getOutput().get().getConnectorId().getCatalogName(), queryInfo.getOutput().get().getSchema(), queryInfo.getOutput().get().getTable(), tableFinishInfo.map(TableFinishInfo::getConnectorOutputMetadata), tableFinishInfo.map(TableFinishInfo::isJsonLengthLimitExceeded)));
        }
        eventListenerManager.queryCompleted(new QueryCompletedEvent(new QueryMetadata(queryInfo.getQueryId().toString(), queryInfo.getSession().getTransactionId().map(TransactionId::toString), queryInfo.getQuery(), queryInfo.getState().toString(), queryInfo.getSelf(), queryInfo.getOutputStage().flatMap(stage -> stageInfoCodec.toJsonWithLengthLimit(stage, toIntExact(config.getMaxOutputStageJsonSize().toBytes())))), new QueryStatistics(ofMillis(queryStats.getTotalCpuTime().toMillis()), ofMillis(queryStats.getTotalScheduledTime().toMillis()), ofMillis(queryStats.getQueuedTime().toMillis()), Optional.ofNullable(queryStats.getAnalysisTime()).map(duration -> ofMillis(duration.toMillis())), Optional.ofNullable(queryStats.getDistributedPlanningTime()).map(duration -> ofMillis(duration.toMillis())), queryStats.getPeakMemoryReservation().toBytes(), queryStats.getRawInputDataSize().toBytes(), queryStats.getRawInputPositions(), queryStats.getCompletedDrivers(), queryInfo.isCompleteInfo(), objectMapper.writeValueAsString(queryInfo.getQueryStats().getOperatorSummaries())), new QueryContext(queryInfo.getSession().getUser(), queryInfo.getSession().getPrincipal(), queryInfo.getSession().getRemoteUserAddress(), queryInfo.getSession().getUserAgent(), queryInfo.getSession().getClientInfo(), queryInfo.getSession().getSource(), queryInfo.getSession().getCatalog(), queryInfo.getSession().getSchema(), mergeSessionAndCatalogProperties(queryInfo), serverAddress, serverVersion, environment), new QueryIOMetadata(inputs.build(), output), queryFailureInfo, ofEpochMilli(queryStats.getCreateTime().getMillis()), ofEpochMilli(queryStats.getExecutionStartTime().getMillis()), ofEpochMilli(queryStats.getEndTime().getMillis())));
        logQueryTimeline(queryInfo);
    } catch (JsonProcessingException e) {
        throw Throwables.propagate(e);
    }
}

Example 4 with QueryInfo

use of com.facebook.presto.execution.QueryInfo in project presto by prestodb.

the class QuerySystemTable method cursor.

@Override
public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, ConnectorSession session, TupleDomain<Integer> constraint) {
    Builder table = InMemoryRecordSet.builder(QUERY_TABLE);
    for (QueryInfo queryInfo : queryManager.getAllQueryInfo()) {
        QueryStats queryStats = queryInfo.getQueryStats();
        table.addRow(nodeId, queryInfo.getQueryId().toString(), queryInfo.getState().toString(), queryInfo.getSession().getUser(), queryInfo.getSession().getSource().orElse(null), queryInfo.getQuery(), toMillis(queryStats.getQueuedTime()), toMillis(queryStats.getAnalysisTime()), toMillis(queryStats.getDistributedPlanningTime()), toTimeStamp(queryStats.getCreateTime()), toTimeStamp(queryStats.getExecutionStartTime()), toTimeStamp(queryStats.getLastHeartbeat()), toTimeStamp(queryStats.getEndTime()));
    }
    return table.build().cursor();
}

Example 5 with QueryInfo

use of com.facebook.presto.execution.QueryInfo in project presto by prestodb.

the class ClusterStatsResource method getClusterStats.

@GET
@Produces(MediaType.APPLICATION_JSON)
public ClusterStats getClusterStats() {
    long runningQueries = 0;
    long blockedQueries = 0;
    long queuedQueries = 0;
    long activeNodes = nodeManager.getNodes(NodeState.ACTIVE).size();
    if (!isIncludeCoordinator) {
        activeNodes -= 1;
    }
    long runningDrivers = 0;
    double memoryReservation = 0;
    double rowInputRate = 0;
    double byteInputRate = 0;
    double cpuTimeRate = 0;
    for (QueryInfo query : queryManager.getAllQueryInfo()) {
        if (query.getState() == QueryState.QUEUED) {
            queuedQueries++;
        } else if (query.getState() == QueryState.RUNNING) {
            if (query.getQueryStats().isFullyBlocked()) {
                blockedQueries++;
            } else {
                runningQueries++;
            }
        }
        if (!query.getState().isDone()) {
            double totalExecutionTimeSeconds = query.getQueryStats().getElapsedTime().getValue(SECONDS);
            if (totalExecutionTimeSeconds != 0) {
                byteInputRate += query.getQueryStats().getProcessedInputDataSize().toBytes() / totalExecutionTimeSeconds;
                rowInputRate += query.getQueryStats().getProcessedInputPositions() / totalExecutionTimeSeconds;
                cpuTimeRate += (query.getQueryStats().getTotalCpuTime().getValue(SECONDS)) / totalExecutionTimeSeconds;
            }
            memoryReservation += query.getQueryStats().getTotalMemoryReservation().toBytes();
            runningDrivers += query.getQueryStats().getRunningDrivers();
        }
    }
    return new ClusterStats(runningQueries, blockedQueries, queuedQueries, activeNodes, runningDrivers, memoryReservation, rowInputRate, byteInputRate, cpuTimeRate);
}