Examples with MemoryPoolId com.facebook.presto.spi.memory.MemoryPoolId used on opensource projects

Example 21 with MemoryPoolId

use of com.facebook.presto.spi.memory.MemoryPoolId in project presto by prestodb.

the class TestMemoryPools method testMoveQuery.

@Test
public void testMoveQuery() {
    QueryId testQuery = new QueryId("test_query");
    MemoryPool pool1 = new MemoryPool(new MemoryPoolId("test"), new DataSize(1000, BYTE));
    MemoryPool pool2 = new MemoryPool(new MemoryPoolId("test"), new DataSize(1000, BYTE));
    pool1.reserve(testQuery, "test_tag", 10);
    Map<String, Long> allocations = pool1.getTaggedMemoryAllocations(testQuery);
    assertEquals(allocations, ImmutableMap.of("test_tag", 10L));
    pool1.moveQuery(testQuery, pool2);
    assertNull(pool1.getTaggedMemoryAllocations(testQuery));
    allocations = pool2.getTaggedMemoryAllocations(testQuery);
    assertEquals(allocations, ImmutableMap.of("test_tag", 10L));
    assertEquals(pool1.getFreeBytes(), 1000);
    assertEquals(pool2.getFreeBytes(), 990);
    pool2.free(testQuery, "test", 10);
    assertEquals(pool2.getFreeBytes(), 1000);
}

Example 22 with MemoryPoolId

use of com.facebook.presto.spi.memory.MemoryPoolId in project presto by prestodb.

the class LowMemoryKillerTestingUtils method toNodeMemoryInfoList.

static List<MemoryInfo> toNodeMemoryInfoList(long maxReservedPoolBytes, long maxGeneralPoolBytes, String reservedQuery, Map<String, Map<String, Long>> queries) {
    Map<InternalNode, NodeReservation> nodeReservations = new HashMap<>();
    for (Map.Entry<String, Map<String, Long>> entry : queries.entrySet()) {
        QueryId queryId = new QueryId(entry.getKey());
        Map<String, Long> reservationByNode = entry.getValue();
        for (Map.Entry<String, Long> nodeEntry : reservationByNode.entrySet()) {
            InternalNode node = new InternalNode(nodeEntry.getKey(), URI.create("http://localhost"), new NodeVersion("version"), false);
            long bytes = nodeEntry.getValue();
            if (bytes == 0) {
                continue;
            }
            if (reservedQuery.equals(entry.getKey())) {
                nodeReservations.computeIfAbsent(node, ignored -> new NodeReservation()).getReserved().add(queryId, bytes);
            } else {
                nodeReservations.computeIfAbsent(node, ignored -> new NodeReservation()).getGeneral().add(queryId, bytes);
            }
        }
    }
    ImmutableList.Builder<MemoryInfo> result = ImmutableList.builder();
    for (Map.Entry<InternalNode, NodeReservation> entry : nodeReservations.entrySet()) {
        NodeReservation nodeReservation = entry.getValue();
        ImmutableMap.Builder<MemoryPoolId, MemoryPoolInfo> pools = ImmutableMap.builder();
        if (nodeReservation.getGeneral().getTotalReservedBytes() > 0) {
            pools.put(GENERAL_POOL, new MemoryPoolInfo(maxGeneralPoolBytes, nodeReservation.getGeneral().getTotalReservedBytes(), 0, nodeReservation.getGeneral().getReservationByQuery(), ImmutableMap.of(), ImmutableMap.of()));
        }
        if (nodeReservation.getReserved().getTotalReservedBytes() > 0) {
            pools.put(RESERVED_POOL, new MemoryPoolInfo(maxReservedPoolBytes, nodeReservation.getReserved().getTotalReservedBytes(), 0, nodeReservation.getReserved().getReservationByQuery(), ImmutableMap.of(), ImmutableMap.of()));
        }
        result.add(new MemoryInfo(new DataSize(maxReservedPoolBytes + maxGeneralPoolBytes, BYTE), pools.build()));
    }
    return result.build();
}

Example 23 with MemoryPoolId

use of com.facebook.presto.spi.memory.MemoryPoolId in project presto by prestodb.

the class TestBasicQueryInfo method testConstructor.

@Test
public void testConstructor() {
    BasicQueryInfo basicInfo = new BasicQueryInfo(new QueryInfo(new QueryId("0"), TEST_SESSION.toSessionRepresentation(), RUNNING, new MemoryPoolId("reserved"), false, URI.create("1"), ImmutableList.of("2", "3"), "SELECT 4", Optional.empty(), Optional.empty(), new QueryStats(DateTime.parse("1991-09-06T05:00-05:30"), DateTime.parse("1991-09-06T05:01-05:30"), DateTime.parse("1991-09-06T05:02-05:30"), DateTime.parse("1991-09-06T06:00-05:30"), Duration.valueOf("8m"), Duration.valueOf("5m"), Duration.valueOf("7m"), Duration.valueOf("34m"), Duration.valueOf("5m"), Duration.valueOf("6m"), Duration.valueOf("35m"), Duration.valueOf("44m"), Duration.valueOf("9m"), Duration.valueOf("10m"), Duration.valueOf("11m"), 13, 14, 21, 15, 16, 17, 18, 34, 19, 20.0, 43.0, DataSize.valueOf("21GB"), DataSize.valueOf("22GB"), DataSize.valueOf("23GB"), DataSize.valueOf("24GB"), DataSize.valueOf("25GB"), DataSize.valueOf("26GB"), DataSize.valueOf("42GB"), true, Duration.valueOf("23m"), Duration.valueOf("24m"), Duration.valueOf("0m"), Duration.valueOf("26m"), true, ImmutableSet.of(BlockedReason.WAITING_FOR_MEMORY), DataSize.valueOf("123MB"), DataSize.valueOf("27GB"), 28, DataSize.valueOf("29GB"), 30, DataSize.valueOf("31GB"), 32, 33, DataSize.valueOf("34GB"), DataSize.valueOf("35GB"), DataSize.valueOf("36GB"), ImmutableList.of(new StageGcStatistics(101, 1002, 102, 103, 104, 105, 106, 107)), ImmutableList.of(), new RuntimeStats()), Optional.empty(), Optional.empty(), ImmutableMap.of(), ImmutableSet.of(), ImmutableMap.of(), ImmutableMap.of(), ImmutableSet.of(), Optional.empty(), false, "33", Optional.empty(), null, StandardErrorCode.ABANDONED_QUERY.toErrorCode(), ImmutableList.of(), ImmutableSet.of(), Optional.empty(), false, Optional.empty(), Optional.of(QueryType.INSERT), Optional.empty(), Optional.empty(), ImmutableMap.of(), ImmutableSet.of()));
    assertEquals(basicInfo.getQueryId().getId(), "0");
    assertEquals(basicInfo.getState(), RUNNING);
    assertEquals(basicInfo.getMemoryPool().getId(), "reserved");
    assertEquals(basicInfo.isScheduled(), false);
    assertEquals(basicInfo.getQuery(), "SELECT 4");
    assertEquals(basicInfo.getQueryType(), Optional.of(QueryType.INSERT));
    assertEquals(basicInfo.getQueryStats().getCreateTime(), DateTime.parse("1991-09-06T05:00-05:30"));
    assertEquals(basicInfo.getQueryStats().getEndTime(), DateTime.parse("1991-09-06T06:00-05:30"));
    assertEquals(basicInfo.getQueryStats().getElapsedTime(), Duration.valueOf("8m"));
    assertEquals(basicInfo.getQueryStats().getExecutionTime(), Duration.valueOf("44m"));
    assertEquals(basicInfo.getQueryStats().getWaitingForPrerequisitesTime(), Duration.valueOf("5m"));
    assertEquals(basicInfo.getQueryStats().getTotalDrivers(), 16);
    assertEquals(basicInfo.getQueryStats().getQueuedDrivers(), 17);
    assertEquals(basicInfo.getQueryStats().getRunningDrivers(), 18);
    assertEquals(basicInfo.getQueryStats().getCompletedDrivers(), 19);
    assertEquals(basicInfo.getQueryStats().getCumulativeUserMemory(), 20.0);
    assertEquals(basicInfo.getQueryStats().getUserMemoryReservation(), DataSize.valueOf("21GB"));
    assertEquals(basicInfo.getQueryStats().getPeakUserMemoryReservation(), DataSize.valueOf("23GB"));
    assertEquals(basicInfo.getQueryStats().getPeakTotalMemoryReservation(), DataSize.valueOf("24GB"));
    assertEquals(basicInfo.getQueryStats().getPeakTaskTotalMemoryReservation(), DataSize.valueOf("26GB"));
    assertEquals(basicInfo.getQueryStats().getPeakNodeTotalMemorReservation(), DataSize.valueOf("42GB"));
    assertEquals(basicInfo.getQueryStats().getTotalCpuTime(), Duration.valueOf("24m"));
    assertEquals(basicInfo.getQueryStats().isFullyBlocked(), true);
    assertEquals(basicInfo.getQueryStats().getBlockedReasons(), ImmutableSet.of(BlockedReason.WAITING_FOR_MEMORY));
    assertEquals(basicInfo.getQueryStats().getTotalAllocation(), DataSize.valueOf("123MB"));
    assertEquals(basicInfo.getQueryStats().getProgressPercentage(), OptionalDouble.of(100));
    assertEquals(basicInfo.getErrorCode(), StandardErrorCode.ABANDONED_QUERY.toErrorCode());
    assertEquals(basicInfo.getErrorType(), StandardErrorCode.ABANDONED_QUERY.toErrorCode().getType());
}