Examples with MockManagedQueryExecution com.facebook.presto.execution.MockManagedQueryExecution used on opensource projects

Example 11 with MockManagedQueryExecution

use of com.facebook.presto.execution.MockManagedQueryExecution in project presto by prestodb.

the class TestResourceGroups method testWeightedFairScheduling.

@Test(timeOut = 10_000)
public void testWeightedFairScheduling() {
    RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {
    }, directExecutor(), ignored -> Optional.empty(), rg -> false);
    root.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    root.setMaxQueuedQueries(50);
    // Start with zero capacity, so that nothing starts running until we've added all the queries
    root.setHardConcurrencyLimit(0);
    root.setSchedulingPolicy(WEIGHTED_FAIR);
    InternalResourceGroup group1 = root.getOrCreateSubGroup("1", true);
    group1.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    group1.setMaxQueuedQueries(50);
    group1.setHardConcurrencyLimit(2);
    group1.setSoftConcurrencyLimit(2);
    group1.setSchedulingWeight(1);
    InternalResourceGroup group2 = root.getOrCreateSubGroup("2", true);
    group2.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    group2.setMaxQueuedQueries(50);
    group2.setHardConcurrencyLimit(2);
    group2.setSoftConcurrencyLimit(2);
    group2.setSchedulingWeight(2);
    Set<MockManagedQueryExecution> group1Queries = fillGroupTo(group1, ImmutableSet.of(), 4);
    Set<MockManagedQueryExecution> group2Queries = fillGroupTo(group2, ImmutableSet.of(), 4);
    root.setHardConcurrencyLimit(3);
    int group1Ran = 0;
    int group2Ran = 0;
    for (int i = 0; i < 1000; i++) {
        group1Ran += completeGroupQueries(group1Queries);
        group2Ran += completeGroupQueries(group2Queries);
        root.processQueuedQueries();
        group1Queries = fillGroupTo(group1, group1Queries, 4);
        group2Queries = fillGroupTo(group2, group2Queries, 4);
    }
    // group1 has a weight of 1 and group2 has a weight of 2, so group2 should account for (2 / (1 + 2)) * 3000 queries.
    assertBetweenInclusive(group1Ran, 995, 1000);
    assertBetweenInclusive(group2Ran, 1995, 2000);
}

Example 12 with MockManagedQueryExecution

use of com.facebook.presto.execution.MockManagedQueryExecution in project presto by prestodb.

the class TestResourceGroups method testWeightedScheduling.

@Test(timeOut = 10_000)
public void testWeightedScheduling() {
    RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {
    }, directExecutor(), ignored -> Optional.empty(), rg -> false);
    root.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    root.setMaxQueuedQueries(4);
    // Start with zero capacity, so that nothing starts running until we've added all the queries
    root.setHardConcurrencyLimit(0);
    root.setSchedulingPolicy(WEIGHTED);
    InternalResourceGroup group1 = root.getOrCreateSubGroup("1", true);
    group1.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    group1.setMaxQueuedQueries(2);
    group1.setHardConcurrencyLimit(2);
    group1.setSoftConcurrencyLimit(2);
    InternalResourceGroup group2 = root.getOrCreateSubGroup("2", true);
    group2.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    group2.setMaxQueuedQueries(2);
    group2.setHardConcurrencyLimit(2);
    group2.setSoftConcurrencyLimit(2);
    group2.setSchedulingWeight(2);
    Set<MockManagedQueryExecution> group1Queries = fillGroupTo(group1, ImmutableSet.of(), 2);
    Set<MockManagedQueryExecution> group2Queries = fillGroupTo(group2, ImmutableSet.of(), 2);
    root.setHardConcurrencyLimit(1);
    int group2Ran = 0;
    for (int i = 0; i < 1000; i++) {
        for (Iterator<MockManagedQueryExecution> iterator = group1Queries.iterator(); iterator.hasNext(); ) {
            MockManagedQueryExecution query = iterator.next();
            if (query.getState() == RUNNING) {
                query.complete();
                iterator.remove();
            }
        }
        group2Ran += completeGroupQueries(group2Queries);
        root.processQueuedQueries();
        group1Queries = fillGroupTo(group1, group1Queries, 2);
        group2Queries = fillGroupTo(group2, group2Queries, 2);
    }
    // group1 has a weight of 1 and group2 has a weight of 2, so group2 should account for (2 / (1 + 2)) of the queries.
    // since this is stochastic, we check that the result of 1000 trials are 2/3 with 99.9999% confidence
    BinomialDistribution binomial = new BinomialDistribution(1000, 2.0 / 3.0);
    int lowerBound = binomial.inverseCumulativeProbability(0.000001);
    int upperBound = binomial.inverseCumulativeProbability(0.999999);
    assertLessThan(group2Ran, upperBound);
    assertGreaterThan(group2Ran, lowerBound);
}

Example 13 with MockManagedQueryExecution

use of com.facebook.presto.execution.MockManagedQueryExecution in project presto by prestodb.

the class TestResourceGroups method testPriorityScheduling.

@Test(timeOut = 10_000)
public void testPriorityScheduling() {
    RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {
    }, directExecutor(), ignored -> Optional.empty(), rg -> false);
    root.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    root.setMaxQueuedQueries(100);
    // Start with zero capacity, so that nothing starts running until we've added all the queries
    root.setHardConcurrencyLimit(0);
    root.setSchedulingPolicy(QUERY_PRIORITY);
    InternalResourceGroup group1 = root.getOrCreateSubGroup("1", true);
    group1.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    group1.setMaxQueuedQueries(100);
    group1.setHardConcurrencyLimit(1);
    InternalResourceGroup group2 = root.getOrCreateSubGroup("2", true);
    group2.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    group2.setMaxQueuedQueries(100);
    group2.setHardConcurrencyLimit(1);
    SortedMap<Integer, MockManagedQueryExecution> queries = new TreeMap<>();
    Random random = new Random();
    for (int i = 0; i < 100; i++) {
        int priority;
        do {
            priority = random.nextInt(1_000_000) + 1;
        } while (queries.containsKey(priority));
        MockManagedQueryExecution query = new MockManagedQueryExecution(0, "query_id", priority);
        if (random.nextBoolean()) {
            group1.run(query);
        } else {
            group2.run(query);
        }
        queries.put(priority, query);
    }
    root.setHardConcurrencyLimit(1);
    List<MockManagedQueryExecution> orderedQueries = new ArrayList<>(queries.values());
    reverse(orderedQueries);
    for (MockManagedQueryExecution query : orderedQueries) {
        root.processQueuedQueries();
        assertEquals(query.getState(), RUNNING);
        query.complete();
    }
}

Example 14 with MockManagedQueryExecution

use of com.facebook.presto.execution.MockManagedQueryExecution in project presto by prestodb.

the class TestResourceGroups method testFairEligibility.

@Test(timeOut = 10_000)
public void testFairEligibility() {
    RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {
    }, directExecutor(), ignored -> Optional.empty(), rg -> false);
    root.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    root.setMaxQueuedQueries(4);
    root.setHardConcurrencyLimit(1);
    InternalResourceGroup group1 = root.getOrCreateSubGroup("1", true);
    group1.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    group1.setMaxQueuedQueries(4);
    group1.setHardConcurrencyLimit(1);
    InternalResourceGroup group2 = root.getOrCreateSubGroup("2", true);
    group2.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    group2.setMaxQueuedQueries(4);
    group2.setHardConcurrencyLimit(1);
    InternalResourceGroup group3 = root.getOrCreateSubGroup("3", true);
    group3.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    group3.setMaxQueuedQueries(4);
    group3.setHardConcurrencyLimit(1);
    MockManagedQueryExecution query1a = new MockManagedQueryExecution(0);
    query1a.startWaitingForPrerequisites();
    group1.run(query1a);
    assertEquals(query1a.getState(), RUNNING);
    MockManagedQueryExecution query1b = new MockManagedQueryExecution(0);
    query1b.startWaitingForPrerequisites();
    group1.run(query1b);
    assertEquals(query1b.getState(), QUEUED);
    MockManagedQueryExecution query2a = new MockManagedQueryExecution(0);
    query2a.startWaitingForPrerequisites();
    group2.run(query2a);
    assertEquals(query2a.getState(), QUEUED);
    MockManagedQueryExecution query2b = new MockManagedQueryExecution(0);
    query2b.startWaitingForPrerequisites();
    group2.run(query2b);
    assertEquals(query2b.getState(), QUEUED);
    MockManagedQueryExecution query3a = new MockManagedQueryExecution(0);
    query3a.startWaitingForPrerequisites();
    group3.run(query3a);
    assertEquals(query3a.getState(), QUEUED);
    query1a.complete();
    root.processQueuedQueries();
    // 2a and not 1b should have started, as group1 was not eligible to start a second query
    assertEquals(query1b.getState(), QUEUED);
    assertEquals(query2a.getState(), RUNNING);
    assertEquals(query2b.getState(), QUEUED);
    assertEquals(query3a.getState(), QUEUED);
    query2a.complete();
    root.processQueuedQueries();
    assertEquals(query3a.getState(), RUNNING);
    assertEquals(query2b.getState(), QUEUED);
    assertEquals(query1b.getState(), QUEUED);
    query3a.complete();
    root.processQueuedQueries();
    assertEquals(query1b.getState(), RUNNING);
    assertEquals(query2b.getState(), QUEUED);
}

Example 15 with MockManagedQueryExecution

use of com.facebook.presto.execution.MockManagedQueryExecution in project presto by prestodb.

the class TestResourceGroups method testHardCpuLimit.

@Test(timeOut = 10_000)
public void testHardCpuLimit() {
    RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {
    }, directExecutor(), ignored -> Optional.empty(), rg -> false);
    root.setSoftMemoryLimit(new DataSize(1, BYTE));
    root.setHardCpuLimit(new Duration(1, SECONDS));
    root.setCpuQuotaGenerationMillisPerSecond(2000);
    root.setMaxQueuedQueries(1);
    root.setHardConcurrencyLimit(1);
    MockManagedQueryExecution query1 = new MockManagedQueryExecution(1, "query_id", 1, new Duration(2, SECONDS));
    query1.startWaitingForPrerequisites();
    root.run(query1);
    assertEquals(query1.getState(), RUNNING);
    MockManagedQueryExecution query2 = new MockManagedQueryExecution(0);
    query2.startWaitingForPrerequisites();
    root.run(query2);
    assertEquals(query2.getState(), QUEUED);
    query1.complete();
    root.processQueuedQueries();
    assertEquals(query2.getState(), QUEUED);
    root.generateCpuQuota(2);
    root.processQueuedQueries();
    assertEquals(query2.getState(), RUNNING);
}