Examples with RootInternalResourceGroup com.facebook.presto.execution.resourceGroups.InternalResourceGroup.RootInternalResourceGroup used on opensource projects

Example 6 with RootInternalResourceGroup

use of com.facebook.presto.execution.resourceGroups.InternalResourceGroup.RootInternalResourceGroup in project presto by prestodb.

the class InternalResourceGroupManager method createGroupIfNecessary.

private synchronized void createGroupIfNecessary(ResourceGroupId id, Session session, Executor executor) {
    SelectionContext context = new SelectionContext(session.getIdentity().getPrincipal().isPresent(), session.getUser(), session.getSource(), getQueryPriority(session));
    if (!groups.containsKey(id)) {
        InternalResourceGroup group;
        if (id.getParent().isPresent()) {
            createGroupIfNecessary(id.getParent().get(), session, executor);
            InternalResourceGroup parent = groups.get(id.getParent().get());
            requireNonNull(parent, "parent is null");
            group = parent.getOrCreateSubGroup(id.getLastSegment());
        } else {
            RootInternalResourceGroup root = new RootInternalResourceGroup(id.getSegments().get(0), this::exportGroup, executor);
            group = root;
            rootGroups.add(root);
        }
        configurationManager.get().configure(group, context);
        checkState(groups.put(id, group) == null, "Unexpected existing resource group");
    }
}

Example 7 with RootInternalResourceGroup

use of com.facebook.presto.execution.resourceGroups.InternalResourceGroup.RootInternalResourceGroup in project presto by prestodb.

the class TestResourceGroups method testQueueFull.

@Test(timeOut = 10_000)
public void testQueueFull() {
    RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {
    }, directExecutor());
    root.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    root.setMaxQueuedQueries(1);
    root.setMaxRunningQueries(1);
    MockQueryExecution query1 = new MockQueryExecution(0);
    root.run(query1);
    assertEquals(query1.getState(), RUNNING);
    MockQueryExecution query2 = new MockQueryExecution(0);
    root.run(query2);
    assertEquals(query2.getState(), QUEUED);
    MockQueryExecution query3 = new MockQueryExecution(0);
    root.run(query3);
    assertEquals(query3.getState(), FAILED);
    assertEquals(query3.getFailureCause().getMessage(), "Too many queued queries for \"root\"");
}

Example 8 with RootInternalResourceGroup

use of com.facebook.presto.execution.resourceGroups.InternalResourceGroup.RootInternalResourceGroup in project presto by prestodb.

the class TestResourceGroups method testWeightedScheduling.

@Test(timeOut = 10_000)
public void testWeightedScheduling() {
    RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {
    }, directExecutor());
    root.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    root.setMaxQueuedQueries(4);
    // Start with zero capacity, so that nothing starts running until we've added all the queries
    root.setMaxRunningQueries(0);
    root.setSchedulingPolicy(WEIGHTED);
    InternalResourceGroup group1 = root.getOrCreateSubGroup("1");
    group1.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    group1.setMaxQueuedQueries(2);
    group1.setMaxRunningQueries(2);
    InternalResourceGroup group2 = root.getOrCreateSubGroup("2");
    group2.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    group2.setMaxQueuedQueries(2);
    group2.setMaxRunningQueries(2);
    group2.setSchedulingWeight(2);
    Set<MockQueryExecution> group1Queries = fillGroupTo(group1, ImmutableSet.of(), 2);
    Set<MockQueryExecution> group2Queries = fillGroupTo(group2, ImmutableSet.of(), 2);
    root.setMaxRunningQueries(1);
    int group2Ran = 0;
    for (int i = 0; i < 1000; i++) {
        for (Iterator<MockQueryExecution> iterator = group1Queries.iterator(); iterator.hasNext(); ) {
            MockQueryExecution query = iterator.next();
            if (query.getState() == RUNNING) {
                query.complete();
                iterator.remove();
            }
        }
        for (Iterator<MockQueryExecution> iterator = group2Queries.iterator(); iterator.hasNext(); ) {
            MockQueryExecution query = iterator.next();
            if (query.getState() == RUNNING) {
                query.complete();
                iterator.remove();
                group2Ran++;
            }
        }
        root.processQueuedQueries();
        group1Queries = fillGroupTo(group1, group1Queries, 2);
        group2Queries = fillGroupTo(group2, group2Queries, 2);
    }
    // group1 has a weight of 1 and group2 has a weight of 2, so group2 should account for (2 / (1 + 2)) of the queries.
    // since this is stochastic, we check that the result of 1000 trials are 2/3 with 99.9999% confidence
    BinomialDistribution binomial = new BinomialDistribution(1000, 2.0 / 3.0);
    int lowerBound = binomial.inverseCumulativeProbability(0.000001);
    int upperBound = binomial.inverseCumulativeProbability(0.999999);
    assertLessThan(group2Ran, upperBound);
    assertGreaterThan(group2Ran, lowerBound);
}

Example 9 with RootInternalResourceGroup

use of com.facebook.presto.execution.resourceGroups.InternalResourceGroup.RootInternalResourceGroup in project presto by prestodb.

the class TestResourceGroups method testHardCpuLimit.

@Test(timeOut = 10_000)
public void testHardCpuLimit() {
    RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {
    }, directExecutor());
    root.setSoftMemoryLimit(new DataSize(1, BYTE));
    root.setHardCpuLimit(new Duration(1, SECONDS));
    root.setCpuQuotaGenerationMillisPerSecond(2000);
    root.setMaxQueuedQueries(1);
    root.setMaxRunningQueries(1);
    MockQueryExecution query1 = new MockQueryExecution(1, "query_id", 1, new Duration(2, SECONDS));
    root.run(query1);
    assertEquals(query1.getState(), RUNNING);
    MockQueryExecution query2 = new MockQueryExecution(0);
    root.run(query2);
    assertEquals(query2.getState(), QUEUED);
    query1.complete();
    root.processQueuedQueries();
    assertEquals(query2.getState(), QUEUED);
    root.generateCpuQuota(2);
    root.processQueuedQueries();
    assertEquals(query2.getState(), RUNNING);
}

Example 10 with RootInternalResourceGroup

use of com.facebook.presto.execution.resourceGroups.InternalResourceGroup.RootInternalResourceGroup in project presto by prestodb.

the class TestResourceGroups method testFairEligibility.

@Test(timeOut = 10_000)
public void testFairEligibility() {
    RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {
    }, directExecutor());
    root.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    root.setMaxQueuedQueries(4);
    root.setMaxRunningQueries(1);
    InternalResourceGroup group1 = root.getOrCreateSubGroup("1");
    group1.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    group1.setMaxQueuedQueries(4);
    group1.setMaxRunningQueries(1);
    InternalResourceGroup group2 = root.getOrCreateSubGroup("2");
    group2.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    group2.setMaxQueuedQueries(4);
    group2.setMaxRunningQueries(1);
    InternalResourceGroup group3 = root.getOrCreateSubGroup("3");
    group3.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    group3.setMaxQueuedQueries(4);
    group3.setMaxRunningQueries(1);
    MockQueryExecution query1a = new MockQueryExecution(0);
    group1.run(query1a);
    assertEquals(query1a.getState(), RUNNING);
    MockQueryExecution query1b = new MockQueryExecution(0);
    group1.run(query1b);
    assertEquals(query1b.getState(), QUEUED);
    MockQueryExecution query2a = new MockQueryExecution(0);
    group2.run(query2a);
    assertEquals(query2a.getState(), QUEUED);
    MockQueryExecution query2b = new MockQueryExecution(0);
    group2.run(query2b);
    assertEquals(query2b.getState(), QUEUED);
    MockQueryExecution query3a = new MockQueryExecution(0);
    group3.run(query3a);
    assertEquals(query3a.getState(), QUEUED);
    query1a.complete();
    root.processQueuedQueries();
    // 2a and not 1b should have started, as group1 was not eligible to start a second query
    assertEquals(query1b.getState(), QUEUED);
    assertEquals(query2a.getState(), RUNNING);
    assertEquals(query2b.getState(), QUEUED);
    assertEquals(query3a.getState(), QUEUED);
    query2a.complete();
    root.processQueuedQueries();
    assertEquals(query3a.getState(), RUNNING);
    assertEquals(query2b.getState(), QUEUED);
    assertEquals(query1b.getState(), QUEUED);
    query3a.complete();
    root.processQueuedQueries();
    assertEquals(query1b.getState(), RUNNING);
    assertEquals(query2b.getState(), QUEUED);
}