Examples with ResourceGroupInfo com.facebook.presto.spi.resourceGroups.ResourceGroupInfo used on opensource projects

Example 1 with ResourceGroupInfo

use of com.facebook.presto.spi.resourceGroups.ResourceGroupInfo in project presto by prestodb.

the class InternalResourceGroup method getInfo.

public ResourceGroupInfo getInfo() {
    synchronized (root) {
        checkState(!subGroups.isEmpty() || (descendantRunningQueries == 0 && descendantQueuedQueries == 0), "Leaf resource group has descendant queries.");
        List<ResourceGroupInfo> infos = subGroups.values().stream().map(InternalResourceGroup::getInfo).collect(toImmutableList());
        ResourceGroupState resourceGroupState;
        if (canRunMore()) {
            resourceGroupState = CAN_RUN;
        } else if (canQueueMore()) {
            resourceGroupState = CAN_QUEUE;
        } else {
            resourceGroupState = FULL;
        }
        return new ResourceGroupInfo(id, new DataSize(softMemoryLimitBytes, BYTE), maxRunningQueries, maxQueuedQueries, resourceGroupState, eligibleSubGroups.size(), new DataSize(cachedMemoryUsageBytes, BYTE), runningQueries.size() + descendantRunningQueries, queuedQueries.size() + descendantQueuedQueries, infos);
    }
}

Example 2 with ResourceGroupInfo

use of com.facebook.presto.spi.resourceGroups.ResourceGroupInfo in project presto by prestodb.

the class QueryStateInfo method createQueryStateInfo.

public static QueryStateInfo createQueryStateInfo(QueryInfo queryInfo, Optional<ResourceGroupId> resourceGroupId, Optional<ResourceGroupInfo> rootResourceGroupInfo) {
    List<ResourceGroupInfo> resourceGroups;
    if (queryInfo.getState() != QUEUED || !resourceGroupId.isPresent() || !rootResourceGroupInfo.isPresent()) {
        resourceGroups = ImmutableList.of();
    } else {
        ImmutableList.Builder<ResourceGroupInfo> builder = ImmutableList.builder();
        ResourceGroupId id = resourceGroupId.get();
        ResourceGroupInfo resourceGroupInfo = rootResourceGroupInfo.get();
        while (true) {
            builder.add(resourceGroupInfo.createSingleNodeInfo());
            if (resourceGroupInfo.getSubGroups().isEmpty()) {
                break;
            }
            Optional<ResourceGroupInfo> subGroupInfo = resourceGroupInfo.getSubGroup(id);
            checkState(subGroupInfo.isPresent(), "No path from root resource group %s to resource group %s", rootResourceGroupInfo.get().getId(), id);
            resourceGroupInfo = subGroupInfo.get();
        }
        resourceGroups = builder.build().reverse();
    }
    return new QueryStateInfo(queryInfo.getQueryId(), queryInfo.getState(), resourceGroupId, queryInfo.getQuery(), queryInfo.getSession().getUser(), queryInfo.getSession().getCatalog(), queryInfo.getSession().getSchema(), resourceGroups);
}

Example 3 with ResourceGroupInfo

use of com.facebook.presto.spi.resourceGroups.ResourceGroupInfo in project presto by prestodb.

the class TestResourceGroups method testGetInfo.

@Test
public void testGetInfo() {
    RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {
    }, directExecutor());
    root.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    root.setMaxQueuedQueries(40);
    // Start with zero capacity, so that nothing starts running until we've added all the queries
    root.setMaxRunningQueries(0);
    root.setSchedulingPolicy(WEIGHTED);
    InternalResourceGroup rootA = root.getOrCreateSubGroup("a");
    rootA.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    rootA.setMaxQueuedQueries(20);
    rootA.setMaxRunningQueries(2);
    InternalResourceGroup rootB = root.getOrCreateSubGroup("b");
    rootB.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    rootB.setMaxQueuedQueries(20);
    rootB.setMaxRunningQueries(2);
    rootB.setSchedulingWeight(2);
    rootB.setSchedulingPolicy(QUERY_PRIORITY);
    InternalResourceGroup rootAX = rootA.getOrCreateSubGroup("x");
    rootAX.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    rootAX.setMaxQueuedQueries(10);
    rootAX.setMaxRunningQueries(10);
    InternalResourceGroup rootAY = rootA.getOrCreateSubGroup("y");
    rootAY.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    rootAY.setMaxQueuedQueries(10);
    rootAY.setMaxRunningQueries(10);
    InternalResourceGroup rootBX = rootB.getOrCreateSubGroup("x");
    rootBX.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    rootBX.setMaxQueuedQueries(10);
    rootBX.setMaxRunningQueries(10);
    InternalResourceGroup rootBY = rootB.getOrCreateSubGroup("y");
    rootBY.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    rootBY.setMaxQueuedQueries(10);
    rootBY.setMaxRunningQueries(10);
    // Queue 40 queries (= maxQueuedQueries (40) + maxRunningQueries (0))
    Set<MockQueryExecution> queries = fillGroupTo(rootAX, ImmutableSet.of(), 10, false);
    queries.addAll(fillGroupTo(rootAY, ImmutableSet.of(), 10, false));
    queries.addAll(fillGroupTo(rootBX, ImmutableSet.of(), 10, true));
    queries.addAll(fillGroupTo(rootBY, ImmutableSet.of(), 10, true));
    ResourceGroupInfo info = root.getInfo();
    assertEquals(info.getNumAggregatedRunningQueries(), 0);
    assertEquals(info.getNumAggregatedQueuedQueries(), 40);
    // root.maxRunningQueries = 4, root.a.maxRunningQueries = 2, root.b.maxRunningQueries = 2. Will have 4 queries running and 36 left queued.
    root.setMaxRunningQueries(4);
    root.processQueuedQueries();
    info = root.getInfo();
    assertEquals(info.getNumAggregatedRunningQueries(), 4);
    assertEquals(info.getNumAggregatedQueuedQueries(), 36);
    // Complete running queries
    Iterator<MockQueryExecution> iterator = queries.iterator();
    while (iterator.hasNext()) {
        MockQueryExecution query = iterator.next();
        if (query.getState() == RUNNING) {
            query.complete();
            iterator.remove();
        }
    }
    // 4 more queries start running, 32 left queued.
    root.processQueuedQueries();
    info = root.getInfo();
    assertEquals(info.getNumAggregatedRunningQueries(), 4);
    assertEquals(info.getNumAggregatedQueuedQueries(), 32);
    // root.maxRunningQueries = 10, root.a.maxRunningQueries = 2, root.b.maxRunningQueries = 2. Still only have 4 running queries and 32 left queued.
    root.setMaxRunningQueries(10);
    root.processQueuedQueries();
    info = root.getInfo();
    assertEquals(info.getNumAggregatedRunningQueries(), 4);
    assertEquals(info.getNumAggregatedQueuedQueries(), 32);
    // root.maxRunningQueries = 10, root.a.maxRunningQueries = 2, root.b.maxRunningQueries = 10. Will have 10 running queries and 26 left queued.
    rootB.setMaxRunningQueries(10);
    root.processQueuedQueries();
    info = root.getInfo();
    assertEquals(info.getNumAggregatedRunningQueries(), 10);
    assertEquals(info.getNumAggregatedQueuedQueries(), 26);
}

Example 4 with ResourceGroupInfo

use of com.facebook.presto.spi.resourceGroups.ResourceGroupInfo in project presto by prestodb.

the class TestQueryStateInfo method testQueryStateInfo.

@Test
public void testQueryStateInfo() {
    ResourceGroupId groupRoot = new ResourceGroupId("root");
    ResourceGroupId groupRootA = new ResourceGroupId(groupRoot, "a");
    ResourceGroupId groupRootAX = new ResourceGroupId(groupRootA, "x");
    ResourceGroupId groupRootAY = new ResourceGroupId(groupRootA, "y");
    ResourceGroupId groupRootB = new ResourceGroupId(groupRoot, "b");
    ResourceGroupInfo rootAXInfo = new ResourceGroupInfo(groupRootAX, new DataSize(6000, BYTE), 1, 10, CAN_QUEUE, 0, new DataSize(4000, BYTE), 1, 1, ImmutableList.of());
    ResourceGroupInfo rootAYInfo = new ResourceGroupInfo(groupRootAY, new DataSize(8000, BYTE), 1, 10, CAN_RUN, 0, new DataSize(0, BYTE), 0, 1, ImmutableList.of());
    ResourceGroupInfo rootAInfo = new ResourceGroupInfo(groupRootA, new DataSize(8000, BYTE), 1, 10, CAN_QUEUE, 1, new DataSize(4000, BYTE), 1, 2, ImmutableList.of(rootAXInfo, rootAYInfo));
    ResourceGroupInfo rootBInfo = new ResourceGroupInfo(groupRootB, new DataSize(8000, BYTE), 1, 10, CAN_QUEUE, 0, new DataSize(4000, BYTE), 1, 1, ImmutableList.of());
    ResourceGroupInfo rootInfo = new ResourceGroupInfo(new ResourceGroupId("root"), new DataSize(10000, BYTE), 2, 20, CAN_QUEUE, 0, new DataSize(6000, BYTE), 2, 3, ImmutableList.of(rootAInfo, rootBInfo));
    // Verify QueryStateInfo for query queued on resource group root.a.y
    QueryStateInfo infoForQueryQueuedOnRootAY = createQueryStateInfo(createQueryInfo("query_root_a_y", QUEUED, "SELECT 1"), Optional.of(groupRootAY), Optional.of(rootInfo));
    assertEquals(infoForQueryQueuedOnRootAY.getQuery(), "SELECT 1");
    assertEquals(infoForQueryQueuedOnRootAY.getQueryId().toString(), "query_root_a_y");
    assertEquals(infoForQueryQueuedOnRootAY.getQueryState(), QUEUED);
    assertEquals(infoForQueryQueuedOnRootAY.getResourceGroupChain().size(), 3);
    List<ResourceGroupInfo> rootAYResourceGroupChainInfo = infoForQueryQueuedOnRootAY.getResourceGroupChain();
    ResourceGroupInfo actualRootAYInfo = rootAYResourceGroupChainInfo.get(0);
    assertEquals(actualRootAYInfo.getId().toString(), groupRootAY.toString());
    assertEquals(actualRootAYInfo.getState(), rootAYInfo.getState());
    assertEquals(actualRootAYInfo.getNumAggregatedRunningQueries(), rootAYInfo.getNumAggregatedRunningQueries());
    assertEquals(actualRootAYInfo.getNumAggregatedQueuedQueries(), rootAYInfo.getNumAggregatedQueuedQueries());
    ResourceGroupInfo actualRootAInfo = rootAYResourceGroupChainInfo.get(1);
    assertEquals(actualRootAInfo.getId().toString(), groupRootA.toString());
    assertEquals(actualRootAInfo.getState(), rootAInfo.getState());
    assertEquals(actualRootAInfo.getNumAggregatedRunningQueries(), rootAInfo.getNumAggregatedRunningQueries());
    assertEquals(actualRootAInfo.getNumAggregatedQueuedQueries(), rootAInfo.getNumAggregatedQueuedQueries());
    ResourceGroupInfo actualRootInfo = rootAYResourceGroupChainInfo.get(2);
    assertEquals(actualRootInfo.getId().toString(), groupRoot.toString());
    assertEquals(actualRootInfo.getState(), rootInfo.getState());
    assertEquals(actualRootInfo.getNumAggregatedRunningQueries(), rootInfo.getNumAggregatedRunningQueries());
    assertEquals(actualRootInfo.getNumAggregatedQueuedQueries(), rootInfo.getNumAggregatedQueuedQueries());
    // Verify QueryStateInfo for query queued on resource group root.b
    QueryStateInfo infoForQueryQueuedOnRootB = createQueryStateInfo(createQueryInfo("query_root_b", QUEUED, "SELECT count(*) FROM t"), Optional.of(groupRootB), Optional.of(rootInfo));
    assertEquals(infoForQueryQueuedOnRootB.getQuery(), "SELECT count(*) FROM t");
    assertEquals(infoForQueryQueuedOnRootB.getQueryId().toString(), "query_root_b");
    assertEquals(infoForQueryQueuedOnRootB.getQueryState(), QUEUED);
    assertEquals(infoForQueryQueuedOnRootB.getResourceGroupChain().size(), 2);
    List<ResourceGroupInfo> rootBResourceGroupChainInfo = infoForQueryQueuedOnRootB.getResourceGroupChain();
    ResourceGroupInfo actualRootBInfo = rootBResourceGroupChainInfo.get(0);
    assertEquals(actualRootBInfo.getId().toString(), groupRootB.toString());
    assertEquals(actualRootBInfo.getState(), rootBInfo.getState());
    assertEquals(actualRootBInfo.getNumAggregatedRunningQueries(), rootBInfo.getNumAggregatedRunningQueries());
    assertEquals(actualRootBInfo.getNumAggregatedQueuedQueries(), rootBInfo.getNumAggregatedQueuedQueries());
    actualRootInfo = rootBResourceGroupChainInfo.get(1);
    assertEquals(actualRootInfo.getId().toString(), groupRoot.toString());
    assertEquals(actualRootInfo.getState(), rootInfo.getState());
    assertEquals(actualRootInfo.getNumAggregatedRunningQueries(), rootInfo.getNumAggregatedRunningQueries());
    assertEquals(actualRootInfo.getNumAggregatedQueuedQueries(), rootInfo.getNumAggregatedQueuedQueries());
}

Example 5 with ResourceGroupInfo

use of com.facebook.presto.spi.resourceGroups.ResourceGroupInfo in project presto by prestodb.

the class TestResourceGroupIntegration method testMemoryFraction.

@Test(timeOut = 60_000)
public void testMemoryFraction() throws Exception {
    try (DistributedQueryRunner queryRunner = getSimpleQueryRunner()) {
        queryRunner.execute("SELECT COUNT(*), clerk FROM orders GROUP BY clerk");
        while (true) {
            TimeUnit.SECONDS.sleep(1);
            ResourceGroupInfo global = queryRunner.getCoordinator().getResourceGroupManager().get().getResourceGroupInfo(new ResourceGroupId("global"));
            if (global.getSoftMemoryLimit().toBytes() > 0) {
                break;
            }
        }
    }
}