Examples with QueryId com.facebook.presto.spi.QueryId used on opensource projects

Example 46 with QueryId

use of com.facebook.presto.spi.QueryId in project presto by prestodb.

the class MemoryLocalQueryRunner method execute.

public List<Page> execute(@Language("SQL") String query) {
    MemoryPool memoryPool = new MemoryPool(new MemoryPoolId("test"), new DataSize(2, GIGABYTE));
    SpillSpaceTracker spillSpaceTracker = new SpillSpaceTracker(new DataSize(1, GIGABYTE));
    QueryContext queryContext = new QueryContext(new QueryId("test"), new DataSize(1, GIGABYTE), new DataSize(2, GIGABYTE), new DataSize(1, GIGABYTE), new DataSize(2, GIGABYTE), memoryPool, new TestingGcMonitor(), localQueryRunner.getExecutor(), localQueryRunner.getScheduler(), new DataSize(4, GIGABYTE), spillSpaceTracker, listJsonCodec(TaskMemoryReservationSummary.class));
    TaskContext taskContext = queryContext.addTaskContext(new TaskStateMachine(new TaskId("query", 0, 0, 0), localQueryRunner.getExecutor()), localQueryRunner.getDefaultSession(), Optional.empty(), false, false, false, false, false);
    // Use NullOutputFactory to avoid coping out results to avoid affecting benchmark results
    ImmutableList.Builder<Page> output = ImmutableList.builder();
    List<Driver> drivers = localQueryRunner.createDrivers(query, new PageConsumerOperator.PageConsumerOutputFactory(types -> output::add), taskContext);
    boolean done = false;
    while (!done) {
        boolean processed = false;
        for (Driver driver : drivers) {
            if (!driver.isFinished()) {
                driver.process();
                processed = true;
            }
        }
        done = !processed;
    }
    return output.build();
}

Example 47 with QueryId

use of com.facebook.presto.spi.QueryId in project presto by prestodb.

the class TestHiveDistributedJoinQueriesWithDynamicFiltering method searchScanFilterAndProjectOperatorStats.

private OperatorStats searchScanFilterAndProjectOperatorStats(QueryId queryId, String tableName) {
    DistributedQueryRunner runner = (DistributedQueryRunner) getQueryRunner();
    Plan plan = runner.getQueryPlan(queryId);
    PlanNodeId nodeId = PlanNodeSearcher.searchFrom(plan.getRoot()).where(node -> {
        if (!(node instanceof ProjectNode)) {
            return false;
        }
        ProjectNode projectNode = (ProjectNode) node;
        FilterNode filterNode = (FilterNode) projectNode.getSource();
        TableScanNode tableScanNode = (TableScanNode) filterNode.getSource();
        return tableName.equals(((HiveTableHandle) (tableScanNode.getTable().getConnectorHandle())).getTableName());
    }).findOnlyElement().getId();
    return runner.getCoordinator().getQueryManager().getFullQueryInfo(queryId).getQueryStats().getOperatorSummaries().stream().filter(summary -> nodeId.equals(summary.getPlanNodeId())).collect(MoreCollectors.onlyElement());
}

Example 48 with QueryId

use of com.facebook.presto.spi.QueryId in project presto by prestodb.

the class TestEventListenerWithExchangeMaterialization method testRuntimeOptimizedStagesCorrectness.

@Test
public void testRuntimeOptimizedStagesCorrectness() throws Exception {
    // We expect one runtime optimized stage: 1.
    int expectedEvents = 2;
    QueryId queryId = runQueryAndWaitForEvents("SELECT phone, regionkey FROM nation INNER JOIN supplier ON supplier.nationkey=nation.nationkey", expectedEvents);
    QueryCreatedEvent queryCreatedEvent = getOnlyElement(generatedEvents.getQueryCreatedEvents());
    QueryCompletedEvent queryCompletedEvent = getOnlyElement(generatedEvents.getQueryCompletedEvents());
    QueryMetadata queryMetadata = queryCompletedEvent.getMetadata();
    Optional<List<StageId>> runtimeOptimizedStages = queryRunner.getCoordinator().getQueryManager().getFullQueryInfo(new QueryId(queryCreatedEvent.getMetadata().getQueryId())).getRuntimeOptimizedStages();
    assertEquals(queryMetadata.getQueryId(), queryId.toString());
    assertEquals(queryMetadata.getRuntimeOptimizedStages().size(), 1);
    assertEquals(queryMetadata.getRuntimeOptimizedStages().get(0), "1");
    assertTrue(runtimeOptimizedStages.isPresent());
    assertEquals(runtimeOptimizedStages.get().size(), 1);
    assertEquals(queryMetadata.getRuntimeOptimizedStages(), runtimeOptimizedStages.get().stream().map(stageId -> String.valueOf(stageId.getId())).collect(toImmutableList()));
    // Now, the following query should not trigger runtime optimizations, so should have empty list of runtime optimized stages.
    runQueryAndWaitForEvents("SELECT phone, regionkey FROM supplier INNER JOIN nation ON supplier.nationkey=nation.nationkey", expectedEvents);
    queryCreatedEvent = getOnlyElement(generatedEvents.getQueryCreatedEvents());
    queryCompletedEvent = getOnlyElement(generatedEvents.getQueryCompletedEvents());
    runtimeOptimizedStages = queryRunner.getCoordinator().getQueryManager().getFullQueryInfo(new QueryId(queryCreatedEvent.getMetadata().getQueryId())).getRuntimeOptimizedStages();
    assertTrue(queryCompletedEvent.getMetadata().getRuntimeOptimizedStages().isEmpty());
    assertFalse(runtimeOptimizedStages.isPresent());
    // Now, the following query should have two optimized joins in a single stage (both on the same nationkey), therefore expect only one optimized stage: 1.
    runQueryAndWaitForEvents("SELECT supplier.phone, regionkey, custkey FROM nation INNER JOIN supplier ON supplier.nationkey=nation.nationkey INNER JOIN customer ON nation.nationkey=customer.nationkey", expectedEvents);
    queryCreatedEvent = getOnlyElement(generatedEvents.getQueryCreatedEvents());
    queryCompletedEvent = getOnlyElement(generatedEvents.getQueryCompletedEvents());
    queryMetadata = queryCompletedEvent.getMetadata();
    runtimeOptimizedStages = queryRunner.getCoordinator().getQueryManager().getFullQueryInfo(new QueryId(queryCreatedEvent.getMetadata().getQueryId())).getRuntimeOptimizedStages();
    assertEquals(queryMetadata.getRuntimeOptimizedStages().size(), 1);
    assertEquals(queryMetadata.getRuntimeOptimizedStages().get(0), "1");
    assertTrue(runtimeOptimizedStages.isPresent());
    assertEquals(runtimeOptimizedStages.get().size(), 1);
    assertEquals(queryMetadata.getRuntimeOptimizedStages(), runtimeOptimizedStages.get().stream().map(stageId -> String.valueOf(stageId.getId())).collect(toImmutableList()));
    // Now, the following query should have two runtime optimized stages: 1 and 4, corresponding to the two join operations (on regionkey and nationkey respectively).
    runQueryAndWaitForEvents("WITH natreg AS (SELECT nation.regionkey, nationkey, region.name FROM region INNER JOIN nation ON nation.regionkey=region.regionkey) SELECT phone, regionkey FROM natreg INNER JOIN supplier ON supplier.nationkey=natreg.nationkey", expectedEvents);
    queryCreatedEvent = getOnlyElement(generatedEvents.getQueryCreatedEvents());
    queryCompletedEvent = getOnlyElement(generatedEvents.getQueryCompletedEvents());
    queryMetadata = queryCompletedEvent.getMetadata();
    runtimeOptimizedStages = queryRunner.getCoordinator().getQueryManager().getFullQueryInfo(new QueryId(queryCreatedEvent.getMetadata().getQueryId())).getRuntimeOptimizedStages();
    assertEquals(queryMetadata.getRuntimeOptimizedStages().size(), 2);
    assertEquals(ImmutableSet.copyOf(queryMetadata.getRuntimeOptimizedStages()), ImmutableSet.of("1", "4"));
    assertTrue(runtimeOptimizedStages.isPresent());
    assertEquals(runtimeOptimizedStages.get().size(), 2);
    assertEquals(queryMetadata.getRuntimeOptimizedStages(), runtimeOptimizedStages.get().stream().map(stageId -> String.valueOf(stageId.getId())).collect(toImmutableList()));
}

Example 49 with QueryId

use of com.facebook.presto.spi.QueryId in project presto by prestodb.

the class TestEventListenerWithExchangeMaterialization method runQueryAndWaitForEvents.

private QueryId runQueryAndWaitForEvents(@Language("SQL") String sql, int numEventsExpected) throws Exception {
    generatedEvents.initialize(numEventsExpected);
    QueryId resultId = queryRunner.executeWithQueryId(session, sql).getQueryId();
    generatedEvents.waitForEvents(600);
    return resultId;
}

Example 50 with QueryId

use of com.facebook.presto.spi.QueryId in project presto by prestodb.

the class ClusterMemoryManager method findLargestMemoryQuery.

private QueryExecution findLargestMemoryQuery(ClusterMemoryPoolInfo generalPool, Iterable<QueryExecution> queries) {
    QueryExecution biggestQuery = null;
    long maxMemory = -1;
    Optional<QueryId> largestMemoryQuery = generalPool.getLargestMemoryQuery();
    // If present, this means the resource manager is determining the largest query, so do not make this determination locally
    if (memoryManagerService.isPresent()) {
        return largestMemoryQuery.flatMap(largestMemoryQueryId -> Streams.stream(queries).filter(query -> query.getQueryId().equals(largestMemoryQueryId)).findFirst()).orElse(null);
    }
    for (QueryExecution queryExecution : queries) {
        if (resourceOvercommit(queryExecution.getSession())) {
            // since their memory usage is unbounded.
            continue;
        }
        long bytesUsed = getQueryMemoryReservation(queryExecution);
        if (bytesUsed > maxMemory) {
            biggestQuery = queryExecution;
            maxMemory = bytesUsed;
        }
    }
    return biggestQuery;
}