Examples with QueryId com.facebook.presto.spi.QueryId used on opensource projects

Example 91 with QueryId

use of com.facebook.presto.spi.QueryId in project presto by prestodb.

the class LowMemoryKillerTestingUtils method toNodeMemoryInfoList.

static List<MemoryInfo> toNodeMemoryInfoList(long maxReservedPoolBytes, long maxGeneralPoolBytes, String reservedQuery, Map<String, Map<String, Long>> queries) {
    Map<InternalNode, NodeReservation> nodeReservations = new HashMap<>();
    for (Map.Entry<String, Map<String, Long>> entry : queries.entrySet()) {
        QueryId queryId = new QueryId(entry.getKey());
        Map<String, Long> reservationByNode = entry.getValue();
        for (Map.Entry<String, Long> nodeEntry : reservationByNode.entrySet()) {
            InternalNode node = new InternalNode(nodeEntry.getKey(), URI.create("http://localhost"), new NodeVersion("version"), false);
            long bytes = nodeEntry.getValue();
            if (bytes == 0) {
                continue;
            }
            if (reservedQuery.equals(entry.getKey())) {
                nodeReservations.computeIfAbsent(node, ignored -> new NodeReservation()).getReserved().add(queryId, bytes);
            } else {
                nodeReservations.computeIfAbsent(node, ignored -> new NodeReservation()).getGeneral().add(queryId, bytes);
            }
        }
    }
    ImmutableList.Builder<MemoryInfo> result = ImmutableList.builder();
    for (Map.Entry<InternalNode, NodeReservation> entry : nodeReservations.entrySet()) {
        NodeReservation nodeReservation = entry.getValue();
        ImmutableMap.Builder<MemoryPoolId, MemoryPoolInfo> pools = ImmutableMap.builder();
        if (nodeReservation.getGeneral().getTotalReservedBytes() > 0) {
            pools.put(GENERAL_POOL, new MemoryPoolInfo(maxGeneralPoolBytes, nodeReservation.getGeneral().getTotalReservedBytes(), 0, nodeReservation.getGeneral().getReservationByQuery(), ImmutableMap.of(), ImmutableMap.of()));
        }
        if (nodeReservation.getReserved().getTotalReservedBytes() > 0) {
            pools.put(RESERVED_POOL, new MemoryPoolInfo(maxReservedPoolBytes, nodeReservation.getReserved().getTotalReservedBytes(), 0, nodeReservation.getReserved().getReservationByQuery(), ImmutableMap.of(), ImmutableMap.of()));
        }
        result.add(new MemoryInfo(new DataSize(maxReservedPoolBytes + maxGeneralPoolBytes, BYTE), pools.build()));
    }
    return result.build();
}

Example 92 with QueryId

use of com.facebook.presto.spi.QueryId in project presto by prestodb.

the class TestMemoryTracking method testTrySetZeroBytesFullPool.

@Test
public void testTrySetZeroBytesFullPool() {
    LocalMemoryContext localMemoryContext = operatorContext.localUserMemoryContext();
    // fill up the pool
    memoryPool.reserve(new QueryId("test_query"), "test", memoryPool.getFreeBytes());
    // try to reserve 0 bytes in the full pool
    assertTrue(localMemoryContext.trySetBytes(localMemoryContext.getBytes()));
}

Example 93 with QueryId

use of com.facebook.presto.spi.QueryId in project presto by prestodb.

the class GcStatusMonitor method logTaskStats.

private static void logTaskStats(List<QueryId> queryIds, ListMultimap<QueryId, SqlTask> tasksByQueryId) {
    List<List<String>> rows = queryIds.stream().flatMap(queryId -> {
        List<SqlTask> sqlTasks = tasksByQueryId.get(queryId);
        Comparator<SqlTask> comparator = (first, second) -> {
            TaskStats aTaskStats = first.getTaskInfo().getStats();
            TaskStats bTaskStats = second.getTaskInfo().getStats();
            return Long.compare(aTaskStats.getUserMemoryReservationInBytes() + aTaskStats.getSystemMemoryReservationInBytes(), bTaskStats.getUserMemoryReservationInBytes() + bTaskStats.getSystemMemoryReservationInBytes());
        };
        return sqlTasks.stream().sorted(comparator.reversed()).map(task -> {
            TaskInfo taskInfo = task.getTaskInfo();
            SqlTaskIoStats taskIOStats = task.getIoStats();
            TaskStatus taskStatus = taskInfo.getTaskStatus();
            TaskStats taskStats = taskInfo.getStats();
            return ImmutableList.of(task.getQueryContext().getQueryId().toString(), task.getTaskId().toString(), taskStatus.getState().toString(), taskStats.getCreateTime().toString(), Long.toString(taskStats.getUserMemoryReservationInBytes()), Long.toString(taskStats.getSystemMemoryReservationInBytes()), Long.toString(taskIOStats.getInputDataSize().getTotalCount()), Long.toString(taskIOStats.getOutputDataSize().getTotalCount()), Long.toString(taskIOStats.getInputPositions().getTotalCount()), Long.toString(taskIOStats.getOutputPositions().getTotalCount()));
        });
    }).collect(toImmutableList());
    if (!rows.isEmpty()) {
        logInfoTable(ImmutableList.<List<String>>builder().add(ImmutableList.of("Query ID", "Task ID", "State", "Created Ts", "User Memory", "System Memory", "Input Bytes", "Output Bytes", "Input Row Count", "Output Row Count")).addAll(rows).build());
    }
}

Example 94 with QueryId

use of com.facebook.presto.spi.QueryId in project presto by prestodb.

the class TestBasicQueryInfo method testConstructor.

@Test
public void testConstructor() {
    BasicQueryInfo basicInfo = new BasicQueryInfo(new QueryInfo(new QueryId("0"), TEST_SESSION.toSessionRepresentation(), RUNNING, new MemoryPoolId("reserved"), false, URI.create("1"), ImmutableList.of("2", "3"), "SELECT 4", Optional.empty(), Optional.empty(), new QueryStats(DateTime.parse("1991-09-06T05:00-05:30"), DateTime.parse("1991-09-06T05:01-05:30"), DateTime.parse("1991-09-06T05:02-05:30"), DateTime.parse("1991-09-06T06:00-05:30"), Duration.valueOf("8m"), Duration.valueOf("5m"), Duration.valueOf("7m"), Duration.valueOf("34m"), Duration.valueOf("5m"), Duration.valueOf("6m"), Duration.valueOf("35m"), Duration.valueOf("44m"), Duration.valueOf("9m"), Duration.valueOf("10m"), Duration.valueOf("11m"), 13, 14, 21, 15, 16, 17, 18, 34, 19, 20.0, 43.0, DataSize.valueOf("21GB"), DataSize.valueOf("22GB"), DataSize.valueOf("23GB"), DataSize.valueOf("24GB"), DataSize.valueOf("25GB"), DataSize.valueOf("26GB"), DataSize.valueOf("42GB"), true, Duration.valueOf("23m"), Duration.valueOf("24m"), Duration.valueOf("0m"), Duration.valueOf("26m"), true, ImmutableSet.of(BlockedReason.WAITING_FOR_MEMORY), DataSize.valueOf("123MB"), DataSize.valueOf("27GB"), 28, DataSize.valueOf("29GB"), 30, DataSize.valueOf("31GB"), 32, 33, DataSize.valueOf("34GB"), DataSize.valueOf("35GB"), DataSize.valueOf("36GB"), ImmutableList.of(new StageGcStatistics(101, 1002, 102, 103, 104, 105, 106, 107)), ImmutableList.of(), new RuntimeStats()), Optional.empty(), Optional.empty(), ImmutableMap.of(), ImmutableSet.of(), ImmutableMap.of(), ImmutableMap.of(), ImmutableSet.of(), Optional.empty(), false, "33", Optional.empty(), null, StandardErrorCode.ABANDONED_QUERY.toErrorCode(), ImmutableList.of(), ImmutableSet.of(), Optional.empty(), false, Optional.empty(), Optional.of(QueryType.INSERT), Optional.empty(), Optional.empty(), ImmutableMap.of(), ImmutableSet.of()));
    assertEquals(basicInfo.getQueryId().getId(), "0");
    assertEquals(basicInfo.getState(), RUNNING);
    assertEquals(basicInfo.getMemoryPool().getId(), "reserved");
    assertEquals(basicInfo.isScheduled(), false);
    assertEquals(basicInfo.getQuery(), "SELECT 4");
    assertEquals(basicInfo.getQueryType(), Optional.of(QueryType.INSERT));
    assertEquals(basicInfo.getQueryStats().getCreateTime(), DateTime.parse("1991-09-06T05:00-05:30"));
    assertEquals(basicInfo.getQueryStats().getEndTime(), DateTime.parse("1991-09-06T06:00-05:30"));
    assertEquals(basicInfo.getQueryStats().getElapsedTime(), Duration.valueOf("8m"));
    assertEquals(basicInfo.getQueryStats().getExecutionTime(), Duration.valueOf("44m"));
    assertEquals(basicInfo.getQueryStats().getWaitingForPrerequisitesTime(), Duration.valueOf("5m"));
    assertEquals(basicInfo.getQueryStats().getTotalDrivers(), 16);
    assertEquals(basicInfo.getQueryStats().getQueuedDrivers(), 17);
    assertEquals(basicInfo.getQueryStats().getRunningDrivers(), 18);
    assertEquals(basicInfo.getQueryStats().getCompletedDrivers(), 19);
    assertEquals(basicInfo.getQueryStats().getCumulativeUserMemory(), 20.0);
    assertEquals(basicInfo.getQueryStats().getUserMemoryReservation(), DataSize.valueOf("21GB"));
    assertEquals(basicInfo.getQueryStats().getPeakUserMemoryReservation(), DataSize.valueOf("23GB"));
    assertEquals(basicInfo.getQueryStats().getPeakTotalMemoryReservation(), DataSize.valueOf("24GB"));
    assertEquals(basicInfo.getQueryStats().getPeakTaskTotalMemoryReservation(), DataSize.valueOf("26GB"));
    assertEquals(basicInfo.getQueryStats().getPeakNodeTotalMemorReservation(), DataSize.valueOf("42GB"));
    assertEquals(basicInfo.getQueryStats().getTotalCpuTime(), Duration.valueOf("24m"));
    assertEquals(basicInfo.getQueryStats().isFullyBlocked(), true);
    assertEquals(basicInfo.getQueryStats().getBlockedReasons(), ImmutableSet.of(BlockedReason.WAITING_FOR_MEMORY));
    assertEquals(basicInfo.getQueryStats().getTotalAllocation(), DataSize.valueOf("123MB"));
    assertEquals(basicInfo.getQueryStats().getProgressPercentage(), OptionalDouble.of(100));
    assertEquals(basicInfo.getErrorCode(), StandardErrorCode.ABANDONED_QUERY.toErrorCode());
    assertEquals(basicInfo.getErrorType(), StandardErrorCode.ABANDONED_QUERY.toErrorCode().getType());
}

Example 95 with QueryId

use of com.facebook.presto.spi.QueryId in project presto by prestodb.

the class TestAccessControlManager method testInitializing.

@Test(expectedExceptions = PrestoException.class, expectedExceptionsMessageRegExp = "Presto server is still initializing")
public void testInitializing() {
    AccessControlManager accessControlManager = new AccessControlManager(createTestTransactionManager());
    accessControlManager.checkCanSetUser(new Identity(USER_NAME, Optional.of(PRINCIPAL)), new AccessControlContext(new QueryId(QUERY_ID), Optional.empty(), Optional.empty()), Optional.empty(), "foo");
}