Examples with RootInternalResourceGroup com.facebook.presto.execution.resourceGroups.InternalResourceGroup.RootInternalResourceGroup used on opensource projects

Example 1 with RootInternalResourceGroup

use of com.facebook.presto.execution.resourceGroups.InternalResourceGroup.RootInternalResourceGroup in project presto by prestodb.

the class InternalResourceGroupManager method refreshAndStartQueries.

private void refreshAndStartQueries() {
    long nanoTime = System.nanoTime();
    long elapsedSeconds = NANOSECONDS.toSeconds(nanoTime - lastCpuQuotaGenerationNanos.get());
    if (elapsedSeconds > 0) {
        // Only advance our clock on second boundaries to avoid calling generateCpuQuota() too frequently, and because it would be a no-op for zero seconds.
        lastCpuQuotaGenerationNanos.addAndGet(elapsedSeconds * 1_000_000_000L);
    } else if (elapsedSeconds < 0) {
        // nano time has overflowed
        lastCpuQuotaGenerationNanos.set(nanoTime);
    }
    for (RootInternalResourceGroup group : rootGroups) {
        try {
            if (elapsedSeconds > 0) {
                group.generateCpuQuota(elapsedSeconds);
            }
        } catch (RuntimeException e) {
            log.error(e, "Exception while generation cpu quota for %s", group);
        }
        try {
            group.processQueuedQueries();
        } catch (RuntimeException e) {
            log.error(e, "Exception while processing queued queries for %s", group);
        }
    }
}

Example 2 with RootInternalResourceGroup

use of com.facebook.presto.execution.resourceGroups.InternalResourceGroup.RootInternalResourceGroup in project presto by prestodb.

the class TestResourceGroups method testSoftCpuLimit.

@Test(timeOut = 10_000)
public void testSoftCpuLimit() {
    RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {
    }, directExecutor());
    root.setSoftMemoryLimit(new DataSize(1, BYTE));
    root.setSoftCpuLimit(new Duration(1, SECONDS));
    root.setHardCpuLimit(new Duration(2, SECONDS));
    root.setCpuQuotaGenerationMillisPerSecond(2000);
    root.setMaxQueuedQueries(1);
    root.setMaxRunningQueries(2);
    MockQueryExecution query1 = new MockQueryExecution(1, "query_id", 1, new Duration(1, SECONDS));
    root.run(query1);
    assertEquals(query1.getState(), RUNNING);
    MockQueryExecution query2 = new MockQueryExecution(0);
    root.run(query2);
    assertEquals(query2.getState(), RUNNING);
    MockQueryExecution query3 = new MockQueryExecution(0);
    root.run(query3);
    assertEquals(query3.getState(), QUEUED);
    query1.complete();
    root.processQueuedQueries();
    assertEquals(query2.getState(), RUNNING);
    assertEquals(query3.getState(), QUEUED);
    root.generateCpuQuota(2);
    root.processQueuedQueries();
    assertEquals(query2.getState(), RUNNING);
    assertEquals(query3.getState(), RUNNING);
}

Example 3 with RootInternalResourceGroup

use of com.facebook.presto.execution.resourceGroups.InternalResourceGroup.RootInternalResourceGroup in project presto by prestodb.

the class TestResourceGroups method testGetInfo.

@Test
public void testGetInfo() {
    RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {
    }, directExecutor());
    root.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    root.setMaxQueuedQueries(40);
    // Start with zero capacity, so that nothing starts running until we've added all the queries
    root.setMaxRunningQueries(0);
    root.setSchedulingPolicy(WEIGHTED);
    InternalResourceGroup rootA = root.getOrCreateSubGroup("a");
    rootA.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    rootA.setMaxQueuedQueries(20);
    rootA.setMaxRunningQueries(2);
    InternalResourceGroup rootB = root.getOrCreateSubGroup("b");
    rootB.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    rootB.setMaxQueuedQueries(20);
    rootB.setMaxRunningQueries(2);
    rootB.setSchedulingWeight(2);
    rootB.setSchedulingPolicy(QUERY_PRIORITY);
    InternalResourceGroup rootAX = rootA.getOrCreateSubGroup("x");
    rootAX.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    rootAX.setMaxQueuedQueries(10);
    rootAX.setMaxRunningQueries(10);
    InternalResourceGroup rootAY = rootA.getOrCreateSubGroup("y");
    rootAY.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    rootAY.setMaxQueuedQueries(10);
    rootAY.setMaxRunningQueries(10);
    InternalResourceGroup rootBX = rootB.getOrCreateSubGroup("x");
    rootBX.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    rootBX.setMaxQueuedQueries(10);
    rootBX.setMaxRunningQueries(10);
    InternalResourceGroup rootBY = rootB.getOrCreateSubGroup("y");
    rootBY.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    rootBY.setMaxQueuedQueries(10);
    rootBY.setMaxRunningQueries(10);
    // Queue 40 queries (= maxQueuedQueries (40) + maxRunningQueries (0))
    Set<MockQueryExecution> queries = fillGroupTo(rootAX, ImmutableSet.of(), 10, false);
    queries.addAll(fillGroupTo(rootAY, ImmutableSet.of(), 10, false));
    queries.addAll(fillGroupTo(rootBX, ImmutableSet.of(), 10, true));
    queries.addAll(fillGroupTo(rootBY, ImmutableSet.of(), 10, true));
    ResourceGroupInfo info = root.getInfo();
    assertEquals(info.getNumAggregatedRunningQueries(), 0);
    assertEquals(info.getNumAggregatedQueuedQueries(), 40);
    // root.maxRunningQueries = 4, root.a.maxRunningQueries = 2, root.b.maxRunningQueries = 2. Will have 4 queries running and 36 left queued.
    root.setMaxRunningQueries(4);
    root.processQueuedQueries();
    info = root.getInfo();
    assertEquals(info.getNumAggregatedRunningQueries(), 4);
    assertEquals(info.getNumAggregatedQueuedQueries(), 36);
    // Complete running queries
    Iterator<MockQueryExecution> iterator = queries.iterator();
    while (iterator.hasNext()) {
        MockQueryExecution query = iterator.next();
        if (query.getState() == RUNNING) {
            query.complete();
            iterator.remove();
        }
    }
    // 4 more queries start running, 32 left queued.
    root.processQueuedQueries();
    info = root.getInfo();
    assertEquals(info.getNumAggregatedRunningQueries(), 4);
    assertEquals(info.getNumAggregatedQueuedQueries(), 32);
    // root.maxRunningQueries = 10, root.a.maxRunningQueries = 2, root.b.maxRunningQueries = 2. Still only have 4 running queries and 32 left queued.
    root.setMaxRunningQueries(10);
    root.processQueuedQueries();
    info = root.getInfo();
    assertEquals(info.getNumAggregatedRunningQueries(), 4);
    assertEquals(info.getNumAggregatedQueuedQueries(), 32);
    // root.maxRunningQueries = 10, root.a.maxRunningQueries = 2, root.b.maxRunningQueries = 10. Will have 10 running queries and 26 left queued.
    rootB.setMaxRunningQueries(10);
    root.processQueuedQueries();
    info = root.getInfo();
    assertEquals(info.getNumAggregatedRunningQueries(), 10);
    assertEquals(info.getNumAggregatedQueuedQueries(), 26);
}

Example 4 with RootInternalResourceGroup

use of com.facebook.presto.execution.resourceGroups.InternalResourceGroup.RootInternalResourceGroup in project presto by prestodb.

the class TestResourceGroups method testFairQueuing.

@Test(timeOut = 10_000)
public void testFairQueuing() {
    RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {
    }, directExecutor());
    root.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    root.setMaxQueuedQueries(4);
    root.setMaxRunningQueries(1);
    InternalResourceGroup group1 = root.getOrCreateSubGroup("1");
    group1.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    group1.setMaxQueuedQueries(4);
    group1.setMaxRunningQueries(2);
    InternalResourceGroup group2 = root.getOrCreateSubGroup("2");
    group2.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    group2.setMaxQueuedQueries(4);
    group2.setMaxRunningQueries(2);
    MockQueryExecution query1a = new MockQueryExecution(0);
    group1.run(query1a);
    assertEquals(query1a.getState(), RUNNING);
    MockQueryExecution query1b = new MockQueryExecution(0);
    group1.run(query1b);
    assertEquals(query1b.getState(), QUEUED);
    MockQueryExecution query1c = new MockQueryExecution(0);
    group1.run(query1c);
    assertEquals(query1c.getState(), QUEUED);
    MockQueryExecution query2a = new MockQueryExecution(0);
    group2.run(query2a);
    assertEquals(query2a.getState(), QUEUED);
    query1a.complete();
    root.processQueuedQueries();
    // 1b and not 2a should have started, as it became queued first and group1 was eligible to run more
    assertEquals(query1b.getState(), RUNNING);
    assertEquals(query1c.getState(), QUEUED);
    assertEquals(query2a.getState(), QUEUED);
    // 2a and not 1c should have started, as all eligible sub groups get fair sharing
    query1b.complete();
    root.processQueuedQueries();
    assertEquals(query2a.getState(), RUNNING);
    assertEquals(query1c.getState(), QUEUED);
}

Example 5 with RootInternalResourceGroup

use of com.facebook.presto.execution.resourceGroups.InternalResourceGroup.RootInternalResourceGroup in project presto by prestodb.

the class TestResourceGroups method testSubgroupMemoryLimit.

@Test
public void testSubgroupMemoryLimit() {
    RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {
    }, directExecutor());
    root.setSoftMemoryLimit(new DataSize(10, BYTE));
    root.setMaxQueuedQueries(4);
    root.setMaxRunningQueries(3);
    InternalResourceGroup subgroup = root.getOrCreateSubGroup("subgroup");
    subgroup.setSoftMemoryLimit(new DataSize(1, BYTE));
    subgroup.setMaxQueuedQueries(4);
    subgroup.setMaxRunningQueries(3);
    MockQueryExecution query1 = new MockQueryExecution(1);
    subgroup.run(query1);
    // Process the group to refresh stats
    root.processQueuedQueries();
    assertEquals(query1.getState(), RUNNING);
    MockQueryExecution query2 = new MockQueryExecution(0);
    subgroup.run(query2);
    assertEquals(query2.getState(), QUEUED);
    MockQueryExecution query3 = new MockQueryExecution(0);
    subgroup.run(query3);
    assertEquals(query3.getState(), QUEUED);
    query1.complete();
    root.processQueuedQueries();
    assertEquals(query2.getState(), RUNNING);
    assertEquals(query3.getState(), RUNNING);
}