Examples with QueryId com.facebook.presto.spi.QueryId used on opensource projects

Example 71 with QueryId

use of com.facebook.presto.spi.QueryId in project presto by prestodb.

the class AbstractOperatorBenchmark method runOnce.

@Override
protected Map<String, Long> runOnce() {
    Session session = testSessionBuilder().setSystemProperty("optimizer.optimize-hash-generation", "true").build();
    MemoryPool memoryPool = new MemoryPool(new MemoryPoolId("test"), new DataSize(1, GIGABYTE));
    SpillSpaceTracker spillSpaceTracker = new SpillSpaceTracker(new DataSize(1, GIGABYTE));
    TaskContext taskContext = new QueryContext(new QueryId("test"), new DataSize(256, MEGABYTE), new DataSize(512, MEGABYTE), new DataSize(256, MEGABYTE), new DataSize(1, GIGABYTE), memoryPool, new TestingGcMonitor(), localQueryRunner.getExecutor(), localQueryRunner.getScheduler(), new DataSize(256, MEGABYTE), spillSpaceTracker, listJsonCodec(TaskMemoryReservationSummary.class)).addTaskContext(new TaskStateMachine(new TaskId("query", 0, 0, 0), localQueryRunner.getExecutor()), session, Optional.empty(), false, false, false, false, false);
    CpuTimer cpuTimer = new CpuTimer();
    Map<String, Long> executionStats = execute(taskContext);
    CpuDuration executionTime = cpuTimer.elapsedTime();
    TaskStats taskStats = taskContext.getTaskStats();
    long inputRows = taskStats.getRawInputPositions();
    long inputBytes = taskStats.getRawInputDataSizeInBytes();
    long outputRows = taskStats.getOutputPositions();
    long outputBytes = taskStats.getOutputDataSizeInBytes();
    double inputMegaBytes = new DataSize(inputBytes, BYTE).getValue(MEGABYTE);
    return ImmutableMap.<String, Long>builder().putAll(executionStats).put("elapsed_millis", executionTime.getWall().toMillis()).put("input_rows_per_second", (long) (inputRows / executionTime.getWall().getValue(SECONDS))).put("output_rows_per_second", (long) (outputRows / executionTime.getWall().getValue(SECONDS))).put("input_megabytes", (long) inputMegaBytes).put("input_megabytes_per_second", (long) (inputMegaBytes / executionTime.getWall().getValue(SECONDS))).put("wall_nanos", executionTime.getWall().roundTo(NANOSECONDS)).put("cpu_nanos", executionTime.getCpu().roundTo(NANOSECONDS)).put("user_nanos", executionTime.getUser().roundTo(NANOSECONDS)).put("input_rows", inputRows).put("input_bytes", inputBytes).put("output_rows", outputRows).put("output_bytes", outputBytes).build();
}

Example 72 with QueryId

use of com.facebook.presto.spi.QueryId in project presto by prestodb.

the class ResourceManagerClusterStatusSender method registerQuery.

@Override
public void registerQuery(ManagedQueryExecution queryExecution) {
    QueryId queryId = queryExecution.getBasicQueryInfo().getQueryId();
    queries.computeIfAbsent(queryId, unused -> {
        AtomicLong sequenceId = new AtomicLong();
        PeriodicTaskExecutor taskExecutor = new PeriodicTaskExecutor(queryHeartbeatInterval.toMillis(), executor, () -> sendQueryHeartbeat(queryExecution, sequenceId.incrementAndGet()));
        taskExecutor.start();
        return taskExecutor;
    });
    queryExecution.addStateChangeListener(newState -> {
        if (newState.isDone()) {
            queries.computeIfPresent(queryId, (unused, queryHeartbeatSender) -> {
                queryHeartbeatSender.forceRun();
                queryHeartbeatSender.stop();
                return null;
            });
        }
    });
}

Example 73 with QueryId

use of com.facebook.presto.spi.QueryId in project presto by prestodb.

the class MemoryRevokingScheduler method revokeQueryMemory.

private void revokeQueryMemory(QueryContext queryContext, long maxTotalMemory) {
    QueryId queryId = queryContext.getQueryId();
    MemoryPool memoryPool = queryContext.getMemoryPool();
    // get a fresh value for queryTotalMemory in case it's changed (e.g. by a previous revocation request)
    long queryTotalMemory = getTotalQueryMemoryReservation(queryId, memoryPool);
    // order tasks by decreasing revocableMemory so that we don't spill more tasks than needed
    SortedMap<Long, TaskContext> queryTaskContextsMap = new TreeMap<>(Comparator.reverseOrder());
    queryContext.getAllTaskContexts().forEach(taskContext -> queryTaskContextsMap.put(taskContext.getTaskMemoryContext().getRevocableMemory(), taskContext));
    AtomicLong remainingBytesToRevoke = new AtomicLong(queryTotalMemory - maxTotalMemory);
    Collection<TaskContext> queryTaskContexts = queryTaskContextsMap.values();
    remainingBytesToRevoke.addAndGet(-MemoryRevokingSchedulerUtils.getMemoryAlreadyBeingRevoked(queryTaskContexts, remainingBytesToRevoke.get()));
    for (TaskContext taskContext : queryTaskContexts) {
        if (remainingBytesToRevoke.get() <= 0) {
            break;
        }
        taskContext.accept(new VoidTraversingQueryContextVisitor<AtomicLong>() {

            @Override
            public Void visitOperatorContext(OperatorContext operatorContext, AtomicLong remainingBytesToRevoke) {
                if (remainingBytesToRevoke.get() > 0) {
                    long revokedBytes = operatorContext.requestMemoryRevoking();
                    if (revokedBytes > 0) {
                        remainingBytesToRevoke.addAndGet(-revokedBytes);
                        log.debug("taskId=%s: requested revoking %s; remaining %s", taskContext.getTaskId(), revokedBytes, remainingBytesToRevoke);
                    }
                }
                return null;
            }
        }, remainingBytesToRevoke);
    }
}

Example 74 with QueryId

use of com.facebook.presto.spi.QueryId in project presto by prestodb.

the class TestMemoryRevokingScheduler method testQueryMemoryNotRevokedWhenNotEnabled.

@Test
public void testQueryMemoryNotRevokedWhenNotEnabled() throws Exception {
    // The various tasks created here use a small amount of system memory independent of what's set explicitly
    // in this test. Triggering spilling based on differences of thousands of bytes rather than hundreds
    // makes the test resilient to any noise that creates.
    QueryId queryId = new QueryId("query");
    // use a larger memory pool so that we don't trigger spilling due to filling the memory pool
    MemoryPool queryLimitMemoryPool = new MemoryPool(new MemoryPoolId("test"), new DataSize(100, GIGABYTE));
    SqlTask sqlTask1 = newSqlTask(queryId, queryLimitMemoryPool);
    TestOperatorContext operatorContext11 = createTestingOperatorContexts(sqlTask1, "operator11");
    allOperatorContexts = ImmutableSet.of(operatorContext11);
    List<SqlTask> tasks = ImmutableList.of(sqlTask1);
    MemoryRevokingScheduler scheduler = new MemoryRevokingScheduler(singletonList(queryLimitMemoryPool), () -> tasks, queryContexts::get, 1.0, 1.0, ORDER_BY_REVOCABLE_BYTES, false);
    try {
        scheduler.start();
        assertMemoryRevokingNotRequested();
        // exceed the query memory limit of 500KB
        operatorContext11.localRevocableMemoryContext().setBytes(600_000);
        scheduler.awaitAsynchronousCallbacksRun();
        assertMemoryRevokingNotRequested();
    } finally {
        scheduler.stop();
    }
}

Example 75 with QueryId

use of com.facebook.presto.spi.QueryId in project presto by prestodb.

the class TestLocalDispatchQuery method testPrerequisitesQueryFinishedCalled.

@Test
public void testPrerequisitesQueryFinishedCalled() {
    QueryStateMachine stateMachine = createStateMachine();
    CountingEventListener eventListener = new CountingEventListener();
    CompletableFuture<?> prequisitesFuture = new CompletableFuture<>();
    AtomicBoolean queryFinishedCalled = new AtomicBoolean();
    LocalDispatchQuery query = new LocalDispatchQuery(stateMachine, createQueryMonitor(eventListener), immediateFuture(null), createClusterSizeMonitor(0), directExecutor(), dispatchQuery -> {
    }, execution -> {
    }, false, new QueryPrerequisites() {

        @Override
        public CompletableFuture<?> waitForPrerequisites(QueryId queryId, QueryPrerequisitesContext context, WarningCollector warningCollector) {
            return prequisitesFuture;
        }

        @Override
        public void queryFinished(QueryId queryId) {
            queryFinishedCalled.set(true);
        }
    });
    assertEquals(query.getBasicQueryInfo().getState(), WAITING_FOR_PREREQUISITES);
    assertFalse(eventListener.getQueryCompletedEvent().isPresent());
    query.startWaitingForPrerequisites();
    prequisitesFuture.complete(null);
    query.fail(new PrestoException(ABANDONED_QUERY, "foo"));
    assertTrue(queryFinishedCalled.get());
}