Examples with MockManagedQueryExecution io.prestosql.execution.MockManagedQueryExecution used on opensource projects

Example 26 with MockManagedQueryExecution

use of io.prestosql.execution.MockManagedQueryExecution in project hetu-core by openlookeng.

the class TestResourceGroups method testWeightedFairSchedulingEqualWeights.

@Test(timeOut = 10_000)
public void testWeightedFairSchedulingEqualWeights() {
    InternalResourceGroup root = new InternalResourceGroup(Optional.empty(), "root", (group, export) -> {
    }, directExecutor());
    root.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    root.setMaxQueuedQueries(50);
    // Start with zero capacity, so that nothing starts running until we've added all the queries
    root.setHardConcurrencyLimit(0);
    root.setSchedulingPolicy(WEIGHTED_FAIR);
    InternalResourceGroup group1 = root.getOrCreateSubGroup("1");
    group1.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    group1.setMaxQueuedQueries(50);
    group1.setHardConcurrencyLimit(2);
    group1.setSoftConcurrencyLimit(2);
    group1.setSchedulingWeight(1);
    InternalResourceGroup group2 = root.getOrCreateSubGroup("2");
    group2.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    group2.setMaxQueuedQueries(50);
    group2.setHardConcurrencyLimit(2);
    group2.setSoftConcurrencyLimit(2);
    group2.setSchedulingWeight(1);
    InternalResourceGroup group3 = root.getOrCreateSubGroup("3");
    group3.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
    group3.setMaxQueuedQueries(50);
    group3.setHardConcurrencyLimit(2);
    group3.setSoftConcurrencyLimit(2);
    group3.setSchedulingWeight(2);
    Set<MockManagedQueryExecution> group1Queries = fillGroupTo(group1, ImmutableSet.of(), 4);
    Set<MockManagedQueryExecution> group2Queries = fillGroupTo(group2, ImmutableSet.of(), 4);
    Set<MockManagedQueryExecution> group3Queries = fillGroupTo(group3, ImmutableSet.of(), 4);
    root.setHardConcurrencyLimit(4);
    int group1Ran = 0;
    int group2Ran = 0;
    int group3Ran = 0;
    for (int i = 0; i < 1000; i++) {
        group1Ran += completeGroupQueries(group1Queries);
        group2Ran += completeGroupQueries(group2Queries);
        group3Ran += completeGroupQueries(group3Queries);
        root.processQueuedQueries();
        group1Queries = fillGroupTo(group1, group1Queries, 4);
        group2Queries = fillGroupTo(group2, group2Queries, 4);
        group3Queries = fillGroupTo(group3, group3Queries, 4);
    }
    // group 3 should run approximately 2x the number of queries of 1 and 2
    BinomialDistribution binomial = new BinomialDistribution(4000, 1.0 / 4.0);
    int lowerBound = binomial.inverseCumulativeProbability(0.000001);
    int upperBound = binomial.inverseCumulativeProbability(0.999999);
    assertBetweenInclusive(group1Ran, lowerBound, upperBound);
    assertBetweenInclusive(group2Ran, lowerBound, upperBound);
    assertBetweenInclusive(group3Ran, 2 * lowerBound, 2 * upperBound);
}

Example 27 with MockManagedQueryExecution

use of io.prestosql.execution.MockManagedQueryExecution in project hetu-core by openlookeng.

the class TestDistributedResourceGroup method testMemoryLimit.

@Test
public void testMemoryLimit() {
    synchronized (lock) {
        DistributedResourceGroup root = new DistributedResourceGroup(Optional.empty(), "root", (group, export) -> {
        }, directExecutor(), statestore);
        resourceGroupBasicSetUp(root, ONE_BYTE, 3, 4);
        Set<MockManagedQueryExecution> queries = new HashSet<>();
        // query1 running in remote
        MockManagedQueryExecution query1 = new MockManagedQueryExecution(2);
        query1.startWaitingForResources();
        query1.setResourceGroupId(root.getId());
        updateQueryStateCache(query1);
        // Process the group to refresh stats
        root.processQueuedQueries();
        assertEquals(query1.getState(), RUNNING);
        MockManagedQueryExecution query2 = new MockManagedQueryExecution(0);
        query2.setResourceGroupId(root.getId());
        root.run(query2);
        updateQueryStateCache(query2);
        assertEquals(query2.getState(), QUEUED);
        query1.complete();
        updateQueryStateCache(query1);
        root.run(query2);
        root.processQueuedQueries();
        assertEquals(query2.getState(), RUNNING);
        StateCacheStore.get().resetCachedStates();
    }
}

Example 28 with MockManagedQueryExecution

use of io.prestosql.execution.MockManagedQueryExecution in project hetu-core by openlookeng.

the class TestDistributedResourceGroup method testHardCpuLimit.

@Test
public void testHardCpuLimit() {
    synchronized (lock) {
        DistributedResourceGroup root = new DistributedResourceGroup(Optional.empty(), "root", (group, export) -> {
        }, directExecutor(), statestore);
        resourceGroupBasicSetUp(root, ONE_BYTE, 1, 1);
        root.setHardCpuLimit(new Duration(1, SECONDS));
        root.setCpuQuotaGenerationMillisPerSecond(2000);
        Set<MockManagedQueryExecution> queries = new HashSet<>();
        MockManagedQueryExecution query1 = new MockManagedQueryExecution(1, "query_id", 1, new Duration(2, SECONDS));
        root.run(query1);
        assertEquals(query1.getState(), RUNNING);
        query1.setResourceGroupId(root.getId());
        queries.add(query1);
        updateStateCacheStore(queries);
        MockManagedQueryExecution query2 = new MockManagedQueryExecution(0);
        root.run(query2);
        assertEquals(query2.getState(), QUEUED);
        query2.setResourceGroupId(root.getId());
        queries.add(query2);
        updateStateCacheStore(queries);
        query1.complete();
        updateStateCacheStore(queries);
        // update query1's cpu usage(2000ms) to root
        setResourceGroupCpuUsage(root, 2000);
        root.processQueuedQueries();
        assertEquals(query2.getState(), QUEUED);
        StateCacheStore.get().resetCachedStates();
    }
}

Example 29 with MockManagedQueryExecution

use of io.prestosql.execution.MockManagedQueryExecution in project hetu-core by openlookeng.

the class TestDistributedResourceGroup method testStateStoreFetchAndUpdate.

@Test
public void testStateStoreFetchAndUpdate() throws JsonProcessingException {
    synchronized (lock) {
        DistributedResourceGroupTemp root = new DistributedResourceGroupTemp(Optional.empty(), "root", (group, export) -> {
        }, directExecutor(), statestore, internalNodeManager);
        resourceGroupBasicSetUp(root, ONE_MEGABYTE, 1, 1);
        MockManagedQueryExecution query1 = new MockManagedQueryExecution(100);
        query1.setResourceGroupId(root.getId());
        MockManagedQueryExecution query2 = new MockManagedQueryExecution(0);
        query2.setResourceGroupId(root.getId());
        Map<String, String> mockMap = new HashMap<>();
        MockStateMap<String, String> mockStateMap = new MockStateMap<>("127.0.0.1-resourceaggrstats", mockMap);
        when(statestore.getOrCreateStateCollection(anyString(), anyObject())).thenReturn(mockStateMap);
        when(internalNodeManager.getCurrentNode()).thenReturn(new InternalNode("node1", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
        when(internalNodeManager.getCoordinators()).thenReturn(ImmutableSet.of(new InternalNode("node1", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true)));
        root.run(query1);
        DistributedResourceGroupAggrStats rootStats = MAPPER.readerFor(DistributedResourceGroupAggrStats.class).readValue(mockMap.get("root"));
        assertEquals(rootStats.getRunningQueries(), 1);
        root.run(query2);
        rootStats = MAPPER.readerFor(DistributedResourceGroupAggrStats.class).readValue(mockMap.get("root"));
        assertEquals(rootStats.getQueuedQueries(), 1);
        assertEquals(rootStats.getCachedMemoryUsageBytes(), 100);
        when(internalNodeManager.getCurrentNode()).thenReturn(new InternalNode("node2", URI.create("local://127.0.0.2"), NodeVersion.UNKNOWN, true));
        assertEquals(root.getGlobalCachedMemoryUsageBytes(), 200);
    }
}

Example 30 with MockManagedQueryExecution

use of io.prestosql.execution.MockManagedQueryExecution in project hetu-core by openlookeng.

the class TestDistributedResourceGroup method testLeastRecentlyExecutedSubgroup.

@Test
public void testLeastRecentlyExecutedSubgroup() {
    synchronized (lock) {
        DistributedResourceGroup root = new DistributedResourceGroup(Optional.empty(), "root", (group, export) -> {
        }, directExecutor(), statestore);
        resourceGroupBasicSetUp(root, ONE_MEGABYTE, 1, 4);
        DistributedResourceGroup group1 = root.getOrCreateSubGroup("1");
        resourceGroupBasicSetUp(group1, ONE_MEGABYTE, 1, 4);
        DistributedResourceGroup group2 = root.getOrCreateSubGroup("2");
        resourceGroupBasicSetUp(group2, ONE_MEGABYTE, 1, 4);
        DistributedResourceGroup group3 = root.getOrCreateSubGroup("3");
        resourceGroupBasicSetUp(group3, ONE_MEGABYTE, 1, 4);
        // remote query query1a
        MockManagedQueryExecution query1a = new MockManagedQueryExecution(0);
        query1a.startWaitingForResources();
        assertEquals(query1a.getState(), RUNNING);
        query1a.setResourceGroupId(group1.getId());
        updateQueryStateCache(query1a);
        // local queries
        MockManagedQueryExecution query1b = new MockManagedQueryExecution(0);
        group1.run(query1b);
        assertEquals(query1b.getState(), QUEUED);
        query1b.setResourceGroupId(group1.getId());
        updateQueryStateCache(query1b);
        MockManagedQueryExecution query2a = new MockManagedQueryExecution(0);
        group2.run(query2a);
        assertEquals(query2a.getState(), QUEUED);
        query2a.setResourceGroupId(group2.getId());
        updateQueryStateCache(query2a);
        MockManagedQueryExecution query2b = new MockManagedQueryExecution(0);
        group2.run(query2b);
        assertEquals(query2b.getState(), QUEUED);
        query2b.setResourceGroupId(group2.getId());
        updateQueryStateCache(query2b);
        MockManagedQueryExecution query3a = new MockManagedQueryExecution(0);
        group3.run(query3a);
        assertEquals(query3a.getState(), QUEUED);
        query3a.setResourceGroupId(group3.getId());
        updateQueryStateCache(query3a);
        query1a.complete();
        updateQueryStateCache(query1a);
        query2a.complete();
        updateQueryStateCache(query2a);
        root.processQueuedQueries();
        // group1 and group2 have executed queries, group3 has not.
        // group3 is the least recently executed subgroup
        assertEquals(query1b.getState(), QUEUED);
        assertEquals(query2b.getState(), QUEUED);
        assertEquals(query3a.getState(), RUNNING);
        query3a.complete();
        updateQueryStateCache(query3a);
        root.processQueuedQueries();
        // group1 is the least recently executed subgroup
        assertEquals(query1b.getState(), RUNNING);
        assertEquals(query2b.getState(), QUEUED);
        query1b.complete();
        updateQueryStateCache(query1b);
        root.processQueuedQueries();
        assertEquals(query2b.getState(), RUNNING);
        StateCacheStore.get().resetCachedStates();
    }
}