Examples with MockManagedQueryExecution com.facebook.presto.execution.MockManagedQueryExecution used on opensource projects

Example 6 with MockManagedQueryExecution

use of com.facebook.presto.execution.MockManagedQueryExecution in project presto by prestodb.

the class TestResourceGroups method testWeightedFairSchedulingNoStarvation.

@Test(timeOut = 10_000)
public void testWeightedFairSchedulingNoStarvation() {
    RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {
    }, directExecutor(), ignored -> Optional.empty(), rg -> false);
    root.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    root.setMaxQueuedQueries(50);
    // Start with zero capacity, so that nothing starts running until we've added all the queries
    root.setHardConcurrencyLimit(0);
    root.setSchedulingPolicy(WEIGHTED_FAIR);
    InternalResourceGroup group1 = root.getOrCreateSubGroup("1", true);
    group1.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    group1.setMaxQueuedQueries(50);
    group1.setHardConcurrencyLimit(2);
    group1.setSoftConcurrencyLimit(2);
    group1.setSchedulingWeight(1);
    InternalResourceGroup group2 = root.getOrCreateSubGroup("2", true);
    group2.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    group2.setMaxQueuedQueries(50);
    group2.setHardConcurrencyLimit(2);
    group2.setSoftConcurrencyLimit(2);
    group2.setSchedulingWeight(2);
    Set<MockManagedQueryExecution> group1Queries = fillGroupTo(group1, ImmutableSet.of(), 4);
    Set<MockManagedQueryExecution> group2Queries = fillGroupTo(group2, ImmutableSet.of(), 4);
    root.setHardConcurrencyLimit(1);
    int group1Ran = 0;
    for (int i = 0; i < 2000; i++) {
        group1Ran += completeGroupQueries(group1Queries);
        completeGroupQueries(group2Queries);
        root.processQueuedQueries();
        group1Queries = fillGroupTo(group1, group1Queries, 4);
        group2Queries = fillGroupTo(group2, group2Queries, 4);
    }
    assertEquals(group1Ran, 1000);
    assertEquals(group1Ran, 1000);
}

Example 7 with MockManagedQueryExecution

use of com.facebook.presto.execution.MockManagedQueryExecution in project presto by prestodb.

the class TestResourceGroups method testSoftCpuLimit.

@Test(timeOut = 10_000)
public void testSoftCpuLimit() {
    RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {
    }, directExecutor(), ignored -> Optional.empty(), rg -> false);
    root.setSoftMemoryLimit(new DataSize(1, BYTE));
    root.setSoftCpuLimit(new Duration(1, SECONDS));
    root.setHardCpuLimit(new Duration(2, SECONDS));
    root.setCpuQuotaGenerationMillisPerSecond(2000);
    root.setMaxQueuedQueries(1);
    root.setHardConcurrencyLimit(2);
    MockManagedQueryExecution query1 = new MockManagedQueryExecution(1, "query_id", 1, new Duration(1, SECONDS));
    query1.startWaitingForPrerequisites();
    root.run(query1);
    assertEquals(query1.getState(), RUNNING);
    MockManagedQueryExecution query2 = new MockManagedQueryExecution(0);
    query2.startWaitingForPrerequisites();
    root.run(query2);
    assertEquals(query2.getState(), RUNNING);
    MockManagedQueryExecution query3 = new MockManagedQueryExecution(0);
    query3.startWaitingForPrerequisites();
    root.run(query3);
    assertEquals(query3.getState(), QUEUED);
    query1.complete();
    root.processQueuedQueries();
    assertEquals(query2.getState(), RUNNING);
    assertEquals(query3.getState(), QUEUED);
    root.generateCpuQuota(2);
    root.processQueuedQueries();
    assertEquals(query2.getState(), RUNNING);
    assertEquals(query3.getState(), RUNNING);
}

Example 8 with MockManagedQueryExecution

use of com.facebook.presto.execution.MockManagedQueryExecution in project presto by prestodb.

the class TestResourceGroups method testGetInfo.

@Test
public void testGetInfo() {
    RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {
    }, directExecutor(), ignored -> Optional.empty(), rg -> false);
    root.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    root.setMaxQueuedQueries(40);
    // Start with zero capacity, so that nothing starts running until we've added all the queries
    root.setHardConcurrencyLimit(0);
    root.setSchedulingPolicy(WEIGHTED);
    InternalResourceGroup rootA = root.getOrCreateSubGroup("a", true);
    rootA.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    rootA.setMaxQueuedQueries(20);
    rootA.setHardConcurrencyLimit(2);
    InternalResourceGroup rootB = root.getOrCreateSubGroup("b", true);
    rootB.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    rootB.setMaxQueuedQueries(20);
    rootB.setHardConcurrencyLimit(2);
    rootB.setSchedulingWeight(2);
    rootB.setSchedulingPolicy(QUERY_PRIORITY);
    InternalResourceGroup rootAX = rootA.getOrCreateSubGroup("x", true);
    rootAX.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    rootAX.setMaxQueuedQueries(10);
    rootAX.setHardConcurrencyLimit(10);
    InternalResourceGroup rootAY = rootA.getOrCreateSubGroup("y", true);
    rootAY.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    rootAY.setMaxQueuedQueries(10);
    rootAY.setHardConcurrencyLimit(10);
    InternalResourceGroup rootBX = rootB.getOrCreateSubGroup("x", true);
    rootBX.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    rootBX.setMaxQueuedQueries(10);
    rootBX.setHardConcurrencyLimit(10);
    InternalResourceGroup rootBY = rootB.getOrCreateSubGroup("y", true);
    rootBY.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    rootBY.setMaxQueuedQueries(10);
    rootBY.setHardConcurrencyLimit(10);
    // Queue 40 queries (= maxQueuedQueries (40) + maxRunningQueries (0))
    Set<MockManagedQueryExecution> queries = fillGroupTo(rootAX, ImmutableSet.of(), 10, false);
    queries.addAll(fillGroupTo(rootAY, ImmutableSet.of(), 10, false));
    queries.addAll(fillGroupTo(rootBX, ImmutableSet.of(), 10, true));
    queries.addAll(fillGroupTo(rootBY, ImmutableSet.of(), 10, true));
    ResourceGroupInfo info = root.getInfo();
    assertEquals(info.getNumRunningQueries(), 0);
    assertEquals(info.getNumQueuedQueries(), 40);
    // root.maxRunningQueries = 4, root.a.maxRunningQueries = 2, root.b.maxRunningQueries = 2. Will have 4 queries running and 36 left queued.
    root.setHardConcurrencyLimit(4);
    root.processQueuedQueries();
    info = root.getInfo();
    assertEquals(info.getNumRunningQueries(), 4);
    assertEquals(info.getNumQueuedQueries(), 36);
    // Complete running queries
    Iterator<MockManagedQueryExecution> iterator = queries.iterator();
    while (iterator.hasNext()) {
        MockManagedQueryExecution query = iterator.next();
        if (query.getState() == RUNNING) {
            query.complete();
            iterator.remove();
        }
    }
    // 4 more queries start running, 32 left queued.
    root.processQueuedQueries();
    info = root.getInfo();
    assertEquals(info.getNumRunningQueries(), 4);
    assertEquals(info.getNumQueuedQueries(), 32);
    // root.maxRunningQueries = 10, root.a.maxRunningQueries = 2, root.b.maxRunningQueries = 2. Still only have 4 running queries and 32 left queued.
    root.setHardConcurrencyLimit(10);
    root.processQueuedQueries();
    info = root.getInfo();
    assertEquals(info.getNumRunningQueries(), 4);
    assertEquals(info.getNumQueuedQueries(), 32);
    // root.maxRunningQueries = 10, root.a.maxRunningQueries = 2, root.b.maxRunningQueries = 10. Will have 10 running queries and 26 left queued.
    rootB.setHardConcurrencyLimit(10);
    root.processQueuedQueries();
    info = root.getInfo();
    assertEquals(info.getNumRunningQueries(), 10);
    assertEquals(info.getNumQueuedQueries(), 26);
}

Example 9 with MockManagedQueryExecution

use of com.facebook.presto.execution.MockManagedQueryExecution in project presto by prestodb.

the class TestResourceManagerClusterStatusSender method testQueryHeartbeat.

@Test(timeOut = 4_000)
public void testQueryHeartbeat() throws Exception {
    MockManagedQueryExecution queryExecution = new MockManagedQueryExecution(1);
    sender.registerQuery(queryExecution);
    Thread.sleep(SLEEP_DURATION);
    int queryHeartbeats = resourceManagerClient.getQueryHeartbeats();
    assertTrue(queryHeartbeats > TARGET_HEARTBEATS * 0.5 && queryHeartbeats <= TARGET_HEARTBEATS * 1.5, format("Expect number of heartbeats to fall within target range (%s), +/- 50%%.  Was: %s", TARGET_HEARTBEATS, queryHeartbeats));
    // Completing the query stops the heartbeats
    queryExecution.complete();
    queryHeartbeats = resourceManagerClient.getQueryHeartbeats();
    Thread.sleep(SLEEP_DURATION);
    assertTrue(resourceManagerClient.getQueryHeartbeats() <= (queryHeartbeats + 1));
}

Example 10 with MockManagedQueryExecution

use of com.facebook.presto.execution.MockManagedQueryExecution in project presto by prestodb.

the class TestResourceGroups method testGetResourceGroupStateInfo.

@Test
public void testGetResourceGroupStateInfo() {
    RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {
    }, directExecutor(), ignored -> Optional.empty(), rg -> false);
    root.setSoftMemoryLimit(new DataSize(1, GIGABYTE));
    root.setMaxQueuedQueries(40);
    root.setHardConcurrencyLimit(10);
    root.setSchedulingPolicy(WEIGHTED);
    InternalResourceGroup rootA = root.getOrCreateSubGroup("a", true);
    rootA.setSoftMemoryLimit(new DataSize(10, MEGABYTE));
    rootA.setMaxQueuedQueries(20);
    rootA.setHardConcurrencyLimit(0);
    InternalResourceGroup rootB = root.getOrCreateSubGroup("b", true);
    rootB.setSoftMemoryLimit(new DataSize(5, MEGABYTE));
    rootB.setMaxQueuedQueries(20);
    rootB.setHardConcurrencyLimit(1);
    rootB.setSchedulingWeight(2);
    rootB.setSchedulingPolicy(QUERY_PRIORITY);
    InternalResourceGroup rootAX = rootA.getOrCreateSubGroup("x", true);
    rootAX.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    rootAX.setMaxQueuedQueries(10);
    rootAX.setHardConcurrencyLimit(10);
    InternalResourceGroup rootAY = rootA.getOrCreateSubGroup("y", true);
    rootAY.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    rootAY.setMaxQueuedQueries(10);
    rootAY.setHardConcurrencyLimit(10);
    Set<MockManagedQueryExecution> queries = fillGroupTo(rootAX, ImmutableSet.of(), 5, false);
    queries.addAll(fillGroupTo(rootAY, ImmutableSet.of(), 5, false));
    queries.addAll(fillGroupTo(rootB, ImmutableSet.of(), 10, true));
    ResourceGroupInfo rootInfo = root.getResourceGroupInfo(true, true, false);
    assertEquals(rootInfo.getId(), root.getId());
    assertEquals(rootInfo.getState(), CAN_RUN);
    assertEquals(rootInfo.getSoftMemoryLimit(), root.getSoftMemoryLimit());
    assertEquals(rootInfo.getMemoryUsage(), new DataSize(0, BYTE));
    assertEquals(rootInfo.getSubGroups().size(), 2);
    assertGroupInfoEquals(rootInfo.getSubGroups().get(0), rootA.getInfo());
    assertEquals(rootInfo.getSubGroups().get(0).getId(), rootA.getId());
    assertEquals(rootInfo.getSubGroups().get(0).getState(), CAN_QUEUE);
    assertEquals(rootInfo.getSubGroups().get(0).getSoftMemoryLimit(), rootA.getSoftMemoryLimit());
    assertEquals(rootInfo.getSubGroups().get(0).getHardConcurrencyLimit(), rootA.getHardConcurrencyLimit());
    assertEquals(rootInfo.getSubGroups().get(0).getMaxQueuedQueries(), rootA.getMaxQueuedQueries());
    assertEquals(rootInfo.getSubGroups().get(0).getNumEligibleSubGroups(), 2);
    assertEquals(rootInfo.getSubGroups().get(0).getNumRunningQueries(), 0);
    assertEquals(rootInfo.getSubGroups().get(0).getNumQueuedQueries(), 10);
    assertGroupInfoEquals(rootInfo.getSubGroups().get(1), rootB.getInfo());
    assertEquals(rootInfo.getSubGroups().get(1).getId(), rootB.getId());
    assertEquals(rootInfo.getSubGroups().get(1).getState(), CAN_QUEUE);
    assertEquals(rootInfo.getSubGroups().get(1).getSoftMemoryLimit(), rootB.getSoftMemoryLimit());
    assertEquals(rootInfo.getSubGroups().get(1).getHardConcurrencyLimit(), rootB.getHardConcurrencyLimit());
    assertEquals(rootInfo.getSubGroups().get(1).getMaxQueuedQueries(), rootB.getMaxQueuedQueries());
    assertEquals(rootInfo.getSubGroups().get(1).getNumEligibleSubGroups(), 0);
    assertEquals(rootInfo.getSubGroups().get(1).getNumRunningQueries(), 1);
    assertEquals(rootInfo.getSubGroups().get(1).getNumQueuedQueries(), 9);
    assertEquals(rootInfo.getSoftConcurrencyLimit(), root.getSoftConcurrencyLimit());
    assertEquals(rootInfo.getHardConcurrencyLimit(), root.getHardConcurrencyLimit());
    assertEquals(rootInfo.getMaxQueuedQueries(), root.getMaxQueuedQueries());
    assertEquals(rootInfo.getNumQueuedQueries(), 19);
    assertEquals(rootInfo.getRunningQueries().size(), 1);
    QueryStateInfo queryInfo = rootInfo.getRunningQueries().get(0);
    assertEquals(queryInfo.getResourceGroupId(), Optional.of(rootB.getId()));
}