Examples with QueryInfo com.facebook.presto.execution.QueryInfo used on opensource projects

Example 6 with QueryInfo

use of com.facebook.presto.execution.QueryInfo in project presto by prestodb.

the class TestServer method testQuery.

@Test
public void testQuery() throws Exception {
    // start query
    Request request = preparePost().setUri(uriFor("/v1/statement")).setBodyGenerator(createStaticBodyGenerator("show catalogs", UTF_8)).setHeader(PRESTO_USER, "user").setHeader(PRESTO_SOURCE, "source").setHeader(PRESTO_CATALOG, "catalog").setHeader(PRESTO_SCHEMA, "schema").setHeader(PRESTO_CLIENT_INFO, "{\"clientVersion\":\"testVersion\"}").addHeader(PRESTO_SESSION, QUERY_MAX_MEMORY + "=1GB").addHeader(PRESTO_SESSION, DISTRIBUTED_JOIN + "=true," + HASH_PARTITION_COUNT + " = 43").addHeader(PRESTO_PREPARED_STATEMENT, "foo=select * from bar").build();
    QueryResults queryResults = client.execute(request, createJsonResponseHandler(jsonCodec(QueryResults.class)));
    // get the query info
    QueryInfo queryInfo = server.getQueryManager().getQueryInfo(new QueryId(queryResults.getId()));
    // verify session properties
    assertEquals(queryInfo.getSession().getSystemProperties(), ImmutableMap.builder().put(QUERY_MAX_MEMORY, "1GB").put(DISTRIBUTED_JOIN, "true").put(HASH_PARTITION_COUNT, "43").build());
    // verify client info in session
    assertEquals(queryInfo.getSession().getClientInfo().get(), "{\"clientVersion\":\"testVersion\"}");
    // verify prepared statements
    assertEquals(queryInfo.getSession().getPreparedStatements(), ImmutableMap.builder().put("foo", "select * from bar").build());
    ImmutableList.Builder<List<Object>> data = ImmutableList.builder();
    if (queryResults.getData() != null) {
        data.addAll(queryResults.getData());
    }
    while (queryResults.getNextUri() != null) {
        queryResults = client.execute(prepareGet().setUri(queryResults.getNextUri()).build(), createJsonResponseHandler(jsonCodec(QueryResults.class)));
        if (queryResults.getData() != null) {
            data.addAll(queryResults.getData());
        }
    }
    assertNull(queryResults.getError());
    // only the system catalog exists by default
    List<List<Object>> rows = data.build();
    assertEquals(rows, ImmutableList.of(ImmutableList.of("system")));
}

Example 7 with QueryInfo

use of com.facebook.presto.execution.QueryInfo in project presto by prestodb.

the class ExplainAnalyzeOperator method getOutput.

@Override
public Page getOutput() {
    if (!finishing) {
        return null;
    }
    QueryInfo queryInfo = queryPerformanceFetcher.getQueryInfo(operatorContext.getDriverContext().getTaskId().getQueryId());
    checkState(queryInfo.getOutputStage().isPresent(), "Output stage is missing");
    if (!hasFinalStageInfo(queryInfo.getOutputStage().get())) {
        return null;
    }
    String plan = textDistributedPlan(queryInfo.getOutputStage().get().getSubStages(), metadata, operatorContext.getSession());
    BlockBuilder builder = VARCHAR.createBlockBuilder(new BlockBuilderStatus(), 1);
    VARCHAR.writeString(builder, plan);
    outputConsumed = true;
    return new Page(builder.build());
}

Example 8 with QueryInfo

use of com.facebook.presto.execution.QueryInfo in project presto by prestodb.

the class QueryExecutionResource method getTaskInfo.

@GET
@Path("/v1/query-execution/{queryId}")
@Produces(MediaType.APPLICATION_JSON)
public Response getTaskInfo(@PathParam("queryId") String queryId) {
    QueryInfo query;
    try {
        query = manager.getQueryInfo(QueryId.valueOf(queryId));
    } catch (NoSuchElementException e) {
        return Response.status(Response.Status.NOT_FOUND).build();
    }
    List<StageInfo> stages = collectStages(query.getOutputStage());
    List<Task> tasks = new ArrayList<>();
    List<Flow> flows = new ArrayList<>();
    for (StageInfo stage : stages) {
        for (TaskInfo task : stage.getTasks()) {
            int bufferedPages = 0;
            TaskStatus taskStatus = task.getTaskStatus();
            for (BufferInfo bufferInfo : task.getOutputBuffers().getBuffers()) {
                bufferedPages += bufferInfo.getBufferedPages();
                if (!bufferInfo.getBufferId().equals(OUTPUT_TASK_ID)) {
                    flows.add(new Flow(taskStatus.getTaskId().toString(), bufferInfo.getBufferId().toString(), bufferInfo.getPageBufferInfo().getPagesAdded(), bufferInfo.getBufferedPages(), bufferInfo.isFinished()));
                }
            }
            long last = TimeUnit.NANOSECONDS.toMillis(System.nanoTime());
            if (task.getStats().getEndTime() != null) {
                last = task.getStats().getEndTime().getMillis();
            }
            tasks.add(new Task(taskStatus.getTaskId().toString(), taskStatus.getState().toString(), taskStatus.getSelf().getHost(), last - task.getStats().getCreateTime().getMillis(), task.getStats().getTotalCpuTime().roundTo(TimeUnit.MILLISECONDS), task.getStats().getTotalBlockedTime().roundTo(TimeUnit.MILLISECONDS), task.getStats().getRawInputDataSize().roundTo(DataSize.Unit.BYTE), task.getStats().getRawInputPositions(), task.getStats().getOutputDataSize().roundTo(DataSize.Unit.BYTE), task.getStats().getOutputPositions(), task.getStats().getMemoryReservation().roundTo(DataSize.Unit.BYTE), task.getStats().getQueuedDrivers(), task.getStats().getRunningDrivers(), task.getStats().getCompletedDrivers(), bufferedPages));
        }
    }
    Map<String, Object> result = ImmutableMap.<String, Object>builder().put("tasks", tasks).put("flows", flows).build();
    return Response.ok(result).build();
}

Example 9 with QueryInfo

use of com.facebook.presto.execution.QueryInfo in project presto by prestodb.

the class TestBasicQueryInfo method testConstructor.

@Test
public void testConstructor() {
    BasicQueryInfo basicInfo = new BasicQueryInfo(new QueryInfo(new QueryId("0"), TEST_SESSION.toSessionRepresentation(), RUNNING, new MemoryPoolId("reserved"), false, URI.create("1"), ImmutableList.of("2", "3"), "SELECT 4", new QueryStats(DateTime.parse("1991-09-06T05:00-05:30"), DateTime.parse("1991-09-06T05:01-05:30"), DateTime.parse("1991-09-06T05:02-05:30"), DateTime.parse("1991-09-06T06:00-05:30"), Duration.valueOf("8m"), Duration.valueOf("7m"), Duration.valueOf("9m"), Duration.valueOf("10m"), Duration.valueOf("11m"), Duration.valueOf("12m"), 13, 14, 15, 16, 17, 18, 19, 20.0, DataSize.valueOf("21GB"), DataSize.valueOf("22GB"), Duration.valueOf("23m"), Duration.valueOf("24m"), Duration.valueOf("25m"), Duration.valueOf("26m"), true, ImmutableSet.of(BlockedReason.WAITING_FOR_MEMORY), DataSize.valueOf("27GB"), 28, DataSize.valueOf("29GB"), 30, DataSize.valueOf("31GB"), 32, ImmutableList.of()), ImmutableMap.of(), ImmutableSet.of(), ImmutableMap.of(), ImmutableSet.of(), Optional.empty(), false, "33", Optional.empty(), null, StandardErrorCode.ABANDONED_QUERY.toErrorCode(), ImmutableSet.of(), Optional.empty(), false, Optional.empty()));
    assertEquals(basicInfo.getQueryId().getId(), "0");
    assertEquals(basicInfo.getState(), RUNNING);
    assertEquals(basicInfo.getMemoryPool().getId(), "reserved");
    assertEquals(basicInfo.isScheduled(), false);
    assertEquals(basicInfo.getQuery(), "SELECT 4");
    assertEquals(basicInfo.getQueryStats().getCreateTime(), DateTime.parse("1991-09-06T05:00-05:30"));
    assertEquals(basicInfo.getQueryStats().getEndTime(), DateTime.parse("1991-09-06T06:00-05:30"));
    assertEquals(basicInfo.getQueryStats().getElapsedTime(), Duration.valueOf("8m"));
    assertEquals(basicInfo.getQueryStats().getExecutionTime(), Duration.valueOf("1m"));
    assertEquals(basicInfo.getQueryStats().getTotalDrivers(), 16);
    assertEquals(basicInfo.getQueryStats().getQueuedDrivers(), 17);
    assertEquals(basicInfo.getQueryStats().getRunningDrivers(), 18);
    assertEquals(basicInfo.getQueryStats().getCompletedDrivers(), 19);
    assertEquals(basicInfo.getQueryStats().getCumulativeMemory(), 20.0);
    assertEquals(basicInfo.getQueryStats().getTotalMemoryReservation(), DataSize.valueOf("21GB"));
    assertEquals(basicInfo.getQueryStats().getPeakMemoryReservation(), DataSize.valueOf("22GB"));
    assertEquals(basicInfo.getQueryStats().getTotalCpuTime(), Duration.valueOf("24m"));
    assertEquals(basicInfo.getQueryStats().isFullyBlocked(), true);
    assertEquals(basicInfo.getQueryStats().getBlockedReasons(), ImmutableSet.of(BlockedReason.WAITING_FOR_MEMORY));
    assertEquals(basicInfo.getErrorCode(), StandardErrorCode.ABANDONED_QUERY.toErrorCode());
    assertEquals(basicInfo.getErrorType(), StandardErrorCode.ABANDONED_QUERY.toErrorCode().getType());
}

Example 10 with QueryInfo

use of com.facebook.presto.execution.QueryInfo in project presto by prestodb.

the class TestMemoryManager method testOutOfMemoryKiller.

@Test(timeOut = 240_000, expectedExceptions = ExecutionException.class, expectedExceptionsMessageRegExp = ".*The cluster is out of memory, and your query was killed. Please try again in a few minutes.")
public void testOutOfMemoryKiller() throws Exception {
    Map<String, String> properties = ImmutableMap.<String, String>builder().put("task.verbose-stats", "true").put("query.low-memory-killer.delay", "5s").put("query.low-memory-killer.enabled", "true").build();
    try (DistributedQueryRunner queryRunner = createQueryRunner(TINY_SESSION, properties)) {
        // Reserve all the memory
        QueryId fakeQueryId = new QueryId("fake");
        for (TestingPrestoServer server : queryRunner.getServers()) {
            for (MemoryPool pool : server.getLocalMemoryManager().getPools()) {
                assertTrue(pool.tryReserve(fakeQueryId, pool.getMaxBytes()));
            }
        }
        List<Future<?>> queryFutures = new ArrayList<>();
        for (int i = 0; i < 2; i++) {
            queryFutures.add(executor.submit(() -> queryRunner.execute("SELECT COUNT(*), clerk FROM orders GROUP BY clerk")));
        }
        // Wait for one of the queries to die
        boolean queryDone = false;
        while (!queryDone) {
            for (QueryInfo info : queryRunner.getCoordinator().getQueryManager().getAllQueryInfo()) {
                if (info.getState().isDone()) {
                    assertEquals(info.getErrorCode().getCode(), CLUSTER_OUT_OF_MEMORY.toErrorCode().getCode());
                    queryDone = true;
                    break;
                }
            }
            MILLISECONDS.sleep(10);
        }
        // Release the memory in the reserved pool
        for (TestingPrestoServer server : queryRunner.getServers()) {
            MemoryPool reserved = server.getLocalMemoryManager().getPool(RESERVED_POOL);
            // Free up the entire pool
            reserved.free(fakeQueryId, reserved.getMaxBytes());
            assertTrue(reserved.getFreeBytes() > 0);
        }
        for (Future<?> query : queryFutures) {
            query.get();
        }
    }
}