Examples with ResourceGroupInfo io.trino.server.ResourceGroupInfo used on opensource projects

Example 1 with ResourceGroupInfo

use of io.trino.server.ResourceGroupInfo in project trino by trinodb.

the class TestResourceGroups method testGetInfo.

@Test
public void testGetInfo() {
    InternalResourceGroup root = new InternalResourceGroup("root", (group, export) -> {
    }, directExecutor()) {

        @Override
        public void triggerProcessQueuedQueries() {
        // No op to allow the test fine-grained control about when to trigger the next query.
        }
    };
    root.setSoftMemoryLimitBytes(DataSize.of(1, MEGABYTE).toBytes());
    root.setMaxQueuedQueries(40);
    // Start with zero capacity, so that nothing starts running until we've added all the queries
    root.setHardConcurrencyLimit(0);
    root.setSchedulingPolicy(WEIGHTED);
    InternalResourceGroup rootA = root.getOrCreateSubGroup("a");
    rootA.setSoftMemoryLimitBytes(DataSize.of(1, MEGABYTE).toBytes());
    rootA.setMaxQueuedQueries(20);
    rootA.setHardConcurrencyLimit(2);
    InternalResourceGroup rootB = root.getOrCreateSubGroup("b");
    rootB.setSoftMemoryLimitBytes(DataSize.of(1, MEGABYTE).toBytes());
    rootB.setMaxQueuedQueries(20);
    rootB.setHardConcurrencyLimit(2);
    rootB.setSchedulingWeight(2);
    rootB.setSchedulingPolicy(QUERY_PRIORITY);
    InternalResourceGroup rootAX = rootA.getOrCreateSubGroup("x");
    rootAX.setSoftMemoryLimitBytes(DataSize.of(1, MEGABYTE).toBytes());
    rootAX.setMaxQueuedQueries(10);
    rootAX.setHardConcurrencyLimit(10);
    InternalResourceGroup rootAY = rootA.getOrCreateSubGroup("y");
    rootAY.setSoftMemoryLimitBytes(DataSize.of(1, MEGABYTE).toBytes());
    rootAY.setMaxQueuedQueries(10);
    rootAY.setHardConcurrencyLimit(10);
    InternalResourceGroup rootBX = rootB.getOrCreateSubGroup("x");
    rootBX.setSoftMemoryLimitBytes(DataSize.of(1, MEGABYTE).toBytes());
    rootBX.setMaxQueuedQueries(10);
    rootBX.setHardConcurrencyLimit(10);
    InternalResourceGroup rootBY = rootB.getOrCreateSubGroup("y");
    rootBY.setSoftMemoryLimitBytes(DataSize.of(1, MEGABYTE).toBytes());
    rootBY.setMaxQueuedQueries(10);
    rootBY.setHardConcurrencyLimit(10);
    // Queue 40 queries (= maxQueuedQueries (40) + maxRunningQueries (0))
    Set<MockManagedQueryExecution> queries = fillGroupTo(rootAX, ImmutableSet.of(), 10, false);
    queries.addAll(fillGroupTo(rootAY, ImmutableSet.of(), 10, false));
    queries.addAll(fillGroupTo(rootBX, ImmutableSet.of(), 10, true));
    queries.addAll(fillGroupTo(rootBY, ImmutableSet.of(), 10, true));
    ResourceGroupInfo info = root.getInfo();
    assertEquals(info.getNumRunningQueries(), 0);
    assertEquals(info.getNumQueuedQueries(), 40);
    // root.maxRunningQueries = 4, root.a.maxRunningQueries = 2, root.b.maxRunningQueries = 2. Will have 4 queries running and 36 left queued.
    root.setHardConcurrencyLimit(4);
    root.updateGroupsAndProcessQueuedQueries();
    info = root.getInfo();
    assertEquals(info.getNumRunningQueries(), 4);
    assertEquals(info.getNumQueuedQueries(), 36);
    // Complete running queries
    Iterator<MockManagedQueryExecution> iterator = queries.iterator();
    while (iterator.hasNext()) {
        MockManagedQueryExecution query = iterator.next();
        if (query.getState() == RUNNING) {
            query.complete();
            iterator.remove();
        }
    }
    // 4 more queries start running, 32 left queued.
    root.updateGroupsAndProcessQueuedQueries();
    info = root.getInfo();
    assertEquals(info.getNumRunningQueries(), 4);
    assertEquals(info.getNumQueuedQueries(), 32);
    // root.maxRunningQueries = 10, root.a.maxRunningQueries = 2, root.b.maxRunningQueries = 2. Still only have 4 running queries and 32 left queued.
    root.setHardConcurrencyLimit(10);
    root.updateGroupsAndProcessQueuedQueries();
    info = root.getInfo();
    assertEquals(info.getNumRunningQueries(), 4);
    assertEquals(info.getNumQueuedQueries(), 32);
    // root.maxRunningQueries = 10, root.a.maxRunningQueries = 2, root.b.maxRunningQueries = 10. Will have 10 running queries and 26 left queued.
    rootB.setHardConcurrencyLimit(10);
    root.updateGroupsAndProcessQueuedQueries();
    info = root.getInfo();
    assertEquals(info.getNumRunningQueries(), 10);
    assertEquals(info.getNumQueuedQueries(), 26);
}

Example 2 with ResourceGroupInfo

use of io.trino.server.ResourceGroupInfo in project trino by trinodb.

the class TestResourceGroups method testGetResourceGroupStateInfo.

@Test
public void testGetResourceGroupStateInfo() {
    InternalResourceGroup root = new InternalResourceGroup("root", (group, export) -> {
    }, directExecutor());
    root.setSoftMemoryLimitBytes(DataSize.of(1, GIGABYTE).toBytes());
    root.setMaxQueuedQueries(40);
    root.setHardConcurrencyLimit(10);
    root.setSchedulingPolicy(WEIGHTED);
    InternalResourceGroup rootA = root.getOrCreateSubGroup("a");
    rootA.setSoftMemoryLimitBytes(DataSize.of(10, MEGABYTE).toBytes());
    rootA.setMaxQueuedQueries(20);
    rootA.setHardConcurrencyLimit(0);
    InternalResourceGroup rootB = root.getOrCreateSubGroup("b");
    rootB.setSoftMemoryLimitBytes(DataSize.of(5, MEGABYTE).toBytes());
    rootB.setMaxQueuedQueries(20);
    rootB.setHardConcurrencyLimit(1);
    rootB.setSchedulingWeight(2);
    rootB.setSchedulingPolicy(QUERY_PRIORITY);
    InternalResourceGroup rootAX = rootA.getOrCreateSubGroup("x");
    rootAX.setSoftMemoryLimitBytes(DataSize.of(1, MEGABYTE).toBytes());
    rootAX.setMaxQueuedQueries(10);
    rootAX.setHardConcurrencyLimit(10);
    InternalResourceGroup rootAY = rootA.getOrCreateSubGroup("y");
    rootAY.setSoftMemoryLimitBytes(DataSize.of(1, MEGABYTE).toBytes());
    rootAY.setMaxQueuedQueries(10);
    rootAY.setHardConcurrencyLimit(10);
    Set<MockManagedQueryExecution> queries = fillGroupTo(rootAX, ImmutableSet.of(), 5, false);
    queries.addAll(fillGroupTo(rootAY, ImmutableSet.of(), 5, false));
    queries.addAll(fillGroupTo(rootB, ImmutableSet.of(), 10, true));
    ResourceGroupInfo rootInfo = root.getFullInfo();
    assertEquals(rootInfo.getId(), root.getId());
    assertEquals(rootInfo.getState(), CAN_RUN);
    assertEquals(rootInfo.getSoftMemoryLimit().toBytes(), root.getSoftMemoryLimitBytes());
    assertEquals(rootInfo.getMemoryUsage(), DataSize.ofBytes(0));
    assertEquals(rootInfo.getCpuUsage().toMillis(), 0);
    List<ResourceGroupInfo> subGroups = rootInfo.getSubGroups().get();
    assertEquals(subGroups.size(), 2);
    assertGroupInfoEquals(subGroups.get(0), rootA.getInfo());
    assertEquals(subGroups.get(0).getId(), rootA.getId());
    assertEquals(subGroups.get(0).getState(), CAN_QUEUE);
    assertEquals(subGroups.get(0).getSoftMemoryLimit().toBytes(), rootA.getSoftMemoryLimitBytes());
    assertEquals(subGroups.get(0).getHardConcurrencyLimit(), rootA.getHardConcurrencyLimit());
    assertEquals(subGroups.get(0).getMaxQueuedQueries(), rootA.getMaxQueuedQueries());
    assertEquals(subGroups.get(0).getNumEligibleSubGroups(), 2);
    assertEquals(subGroups.get(0).getNumRunningQueries(), 0);
    assertEquals(subGroups.get(0).getNumQueuedQueries(), 10);
    assertGroupInfoEquals(subGroups.get(1), rootB.getInfo());
    assertEquals(subGroups.get(1).getId(), rootB.getId());
    assertEquals(subGroups.get(1).getState(), CAN_QUEUE);
    assertEquals(subGroups.get(1).getSoftMemoryLimit().toBytes(), rootB.getSoftMemoryLimitBytes());
    assertEquals(subGroups.get(1).getHardConcurrencyLimit(), rootB.getHardConcurrencyLimit());
    assertEquals(subGroups.get(1).getMaxQueuedQueries(), rootB.getMaxQueuedQueries());
    assertEquals(subGroups.get(1).getNumEligibleSubGroups(), 0);
    assertEquals(subGroups.get(1).getNumRunningQueries(), 1);
    assertEquals(subGroups.get(1).getNumQueuedQueries(), 9);
    assertEquals(rootInfo.getSoftConcurrencyLimit(), root.getSoftConcurrencyLimit());
    assertEquals(rootInfo.getHardConcurrencyLimit(), root.getHardConcurrencyLimit());
    assertEquals(rootInfo.getMaxQueuedQueries(), root.getMaxQueuedQueries());
    assertEquals(rootInfo.getNumQueuedQueries(), 19);
    List<QueryStateInfo> runningQueries = rootInfo.getRunningQueries().get();
    assertEquals(runningQueries.size(), 1);
    QueryStateInfo queryInfo = runningQueries.get(0);
    assertEquals(queryInfo.getResourceGroupId(), Optional.of(rootB.getId()));
}

Example 3 with ResourceGroupInfo

use of io.trino.server.ResourceGroupInfo in project trino by trinodb.

the class TestResourceGroupIntegration method waitForGlobalResourceGroup.

public static void waitForGlobalResourceGroup(DistributedQueryRunner queryRunner) throws InterruptedException {
    long startTime = System.nanoTime();
    while (true) {
        SECONDS.sleep(1);
        ResourceGroupInfo global = getResourceGroupManager(queryRunner).tryGetResourceGroupInfo(new ResourceGroupId("global")).orElseThrow(() -> new IllegalStateException("Resource group not found"));
        if (global.getSoftMemoryLimit().toBytes() > 0) {
            break;
        }
        assertLessThan(nanosSince(startTime).roundTo(SECONDS), 60L);
    }
}

Example 4 with ResourceGroupInfo

use of io.trino.server.ResourceGroupInfo in project trino by trinodb.

the class TestResourceGroupIntegration method testPathToRoot.

@Test
public void testPathToRoot() throws Exception {
    try (DistributedQueryRunner queryRunner = TpchQueryRunnerBuilder.builder().build()) {
        queryRunner.installPlugin(new ResourceGroupManagerPlugin());
        InternalResourceGroupManager<?> manager = getResourceGroupManager(queryRunner);
        manager.setConfigurationManager("file", ImmutableMap.of("resource-groups.config-file", getResourceFilePath("resource_groups_config_dashboard.json")));
        queryRunner.execute(testSessionBuilder().setCatalog("tpch").setSchema("tiny").setSource("dashboard-foo").build(), "SELECT COUNT(*), clerk FROM orders GROUP BY clerk");
        List<ResourceGroupInfo> path = manager.tryGetPathToRoot(new ResourceGroupId(new ResourceGroupId(new ResourceGroupId("global"), "user-user"), "dashboard-user")).orElseThrow(() -> new IllegalStateException("Resource group not found"));
        assertEquals(path.size(), 3);
        assertThat(path.get(1).getSubGroups()).isPresent();
        assertEquals(path.get(2).getId(), new ResourceGroupId("global"));
        assertEquals(path.get(2).getHardConcurrencyLimit(), 100);
        assertThat(path.get(2).getRunningQueries()).isNotPresent();
    }
}

Example 5 with ResourceGroupInfo

use of io.trino.server.ResourceGroupInfo in project trino by trinodb.

the class TestQueuesDb method testRunningQuery.

@Test(timeOut = 60_000)
public void testRunningQuery() throws Exception {
    queryRunner.execute("SELECT COUNT(*), clerk FROM orders GROUP BY clerk");
    while (true) {
        ResourceGroupInfo global = queryRunner.getCoordinator().getResourceGroupManager().get().tryGetResourceGroupInfo(new ResourceGroupId(new ResourceGroupId("global"), "bi-user")).orElseThrow(() -> new IllegalStateException("Resource group not found"));
        if (global.getSoftMemoryLimit().toBytes() > 0) {
            break;
        }
        TimeUnit.SECONDS.sleep(2);
    }
}