use of io.prestosql.server.ResourceGroupInfo in project hetu-core by openlookeng.
the class TestResourceGroupIntegration method waitForGlobalResourceGroup.
public static void waitForGlobalResourceGroup(DistributedQueryRunner queryRunner) throws InterruptedException {
long startTime = System.nanoTime();
while (true) {
SECONDS.sleep(1);
ResourceGroupInfo global = getResourceGroupManager(queryRunner).getResourceGroupInfo(new ResourceGroupId("global"));
if (global.getSoftMemoryLimit().toBytes() > 0) {
break;
}
assertLessThan(nanosSince(startTime).roundTo(SECONDS), 60L);
}
}
use of io.prestosql.server.ResourceGroupInfo in project hetu-core by openlookeng.
the class TestDistributedResourceGroup method testGetResourceGroupStateInfo.
@Test
public void testGetResourceGroupStateInfo() {
synchronized (lock) {
DistributedResourceGroup root = new DistributedResourceGroup(Optional.empty(), "root", (group, export) -> {
}, directExecutor(), statestore);
resourceGroupBasicSetUp(root, ONE_GIGABYTE, 20, 40);
root.setSoftConcurrencyLimit(20);
DistributedResourceGroup rootA = root.getOrCreateSubGroup("a");
resourceGroupBasicSetUp(rootA, ONE_MEGABYTE, 10, 20);
DistributedResourceGroup rootB = root.getOrCreateSubGroup("b");
resourceGroupBasicSetUp(rootB, ONE_MEGABYTE, 10, 20);
DistributedResourceGroup rootAX = rootA.getOrCreateSubGroup("x");
resourceGroupBasicSetUp(rootAX, ONE_MEGABYTE, 10, 10);
DistributedResourceGroup rootAY = rootA.getOrCreateSubGroup("y");
resourceGroupBasicSetUp(rootAY, ONE_MEGABYTE, 10, 10);
Set<MockManagedQueryExecution> queries = fillGroupTo(rootAX, ImmutableSet.of(), 5, false);
queries.addAll(fillGroupTo(rootAY, ImmutableSet.of(), 5, false));
queries.addAll(fillGroupTo(rootB, ImmutableSet.of(), 10, true));
updateStateCacheStore(queries);
root.processQueuedQueries();
ResourceGroupInfo rootInfo = root.getFullInfo();
assertEquals(rootInfo.getId(), root.getId());
assertEquals(rootInfo.getState(), CAN_QUEUE);
assertEquals(rootInfo.getSoftMemoryLimit(), root.getSoftMemoryLimit());
assertEquals(rootInfo.getMemoryUsage(), new DataSize(0, BYTE));
assertEquals(rootInfo.getSubGroups().size(), 2);
assertGroupInfoEquals(rootInfo.getSubGroups().get(0), rootA.getInfo());
assertEquals(rootInfo.getSubGroups().get(0).getId(), rootA.getId());
assertEquals(rootInfo.getSubGroups().get(0).getState(), CAN_QUEUE);
assertEquals(rootInfo.getSubGroups().get(0).getSoftMemoryLimit(), rootA.getSoftMemoryLimit());
assertEquals(rootInfo.getSubGroups().get(0).getSoftReservedMemory(), rootA.getSoftReservedMemory());
assertEquals(rootInfo.getSubGroups().get(0).getHardReservedConcurrency(), rootA.getHardReservedConcurrency());
assertEquals(rootInfo.getSubGroups().get(0).getSchedulingWeight(), rootA.getSchedulingWeight());
assertEquals(rootInfo.getSubGroups().get(0).getSchedulingPolicy(), rootA.getSchedulingPolicy());
assertEquals(rootInfo.getSubGroups().get(0).getHardConcurrencyLimit(), rootA.getHardConcurrencyLimit());
assertEquals(rootInfo.getSubGroups().get(0).getMaxQueuedQueries(), rootA.getMaxQueuedQueries());
assertEquals(rootInfo.getSubGroups().get(0).getNumRunningQueries(), 10);
assertEquals(rootInfo.getSubGroups().get(0).getNumQueuedQueries(), 0);
assertGroupInfoEquals(rootInfo.getSubGroups().get(1), rootB.getInfo());
assertEquals(rootInfo.getSubGroups().get(1).getId(), rootB.getId());
assertEquals(rootInfo.getSubGroups().get(1).getState(), CAN_QUEUE);
assertEquals(rootInfo.getSubGroups().get(1).getSoftMemoryLimit(), rootB.getSoftMemoryLimit());
assertEquals(rootInfo.getSubGroups().get(1).getSoftReservedMemory(), rootA.getSoftReservedMemory());
assertEquals(rootInfo.getSubGroups().get(1).getHardReservedConcurrency(), rootA.getHardReservedConcurrency());
assertEquals(rootInfo.getSubGroups().get(1).getSchedulingWeight(), rootA.getSchedulingWeight());
assertEquals(rootInfo.getSubGroups().get(1).getSchedulingPolicy(), rootA.getSchedulingPolicy());
assertEquals(rootInfo.getSubGroups().get(1).getHardConcurrencyLimit(), rootB.getHardConcurrencyLimit());
assertEquals(rootInfo.getSubGroups().get(1).getMaxQueuedQueries(), rootB.getMaxQueuedQueries());
assertEquals(rootInfo.getSubGroups().get(1).getNumEligibleSubGroups(), 0);
assertEquals(rootInfo.getSubGroups().get(1).getNumRunningQueries(), 10);
assertEquals(rootInfo.getSubGroups().get(1).getNumQueuedQueries(), 0);
assertEquals(rootInfo.getSoftConcurrencyLimit(), root.getSoftConcurrencyLimit());
assertEquals(rootInfo.getHardConcurrencyLimit(), root.getHardConcurrencyLimit());
assertEquals(rootInfo.getMaxQueuedQueries(), root.getMaxQueuedQueries());
assertEquals(rootInfo.getNumQueuedQueries(), 0);
assertEquals(rootInfo.getRunningQueries().size(), 20);
QueryStateInfo queryInfo = rootInfo.getRunningQueries().get(0);
assertEquals(queryInfo.getResourceGroupId(), Optional.of(rootAX.getId()));
StateCacheStore.get().resetCachedStates();
}
}
use of io.prestosql.server.ResourceGroupInfo in project hetu-core by openlookeng.
the class TestDistributedResourceGroup method testGetInfo.
@Test
public void testGetInfo() {
synchronized (lock) {
DistributedResourceGroup root = new DistributedResourceGroup(Optional.empty(), "root", (group, export) -> {
}, directExecutor(), statestore);
// Start with zero capacity, so that nothing starts running until we've added all the queries
resourceGroupBasicSetUp(root, ONE_MEGABYTE, 0, 40);
DistributedResourceGroup rootA = root.getOrCreateSubGroup("a");
resourceGroupBasicSetUp(rootA, ONE_MEGABYTE, 10, 20);
DistributedResourceGroup rootB = root.getOrCreateSubGroup("b");
resourceGroupBasicSetUp(rootB, ONE_MEGABYTE, 10, 20);
DistributedResourceGroup rootAX = rootA.getOrCreateSubGroup("x");
resourceGroupBasicSetUp(rootAX, ONE_MEGABYTE, 10, 10);
DistributedResourceGroup rootAY = rootA.getOrCreateSubGroup("y");
resourceGroupBasicSetUp(rootAY, ONE_MEGABYTE, 10, 10);
DistributedResourceGroup rootBX = rootB.getOrCreateSubGroup("x");
resourceGroupBasicSetUp(rootBX, ONE_MEGABYTE, 10, 10);
DistributedResourceGroup rootBY = rootB.getOrCreateSubGroup("y");
resourceGroupBasicSetUp(rootBY, ONE_MEGABYTE, 10, 10);
// Queue 40 queries (= maxQueuedQueries (40) + maxRunningQueries (0))
Set<MockManagedQueryExecution> queries = fillGroupTo(rootAX, ImmutableSet.of(), 10, false);
queries.addAll(fillGroupTo(rootAY, ImmutableSet.of(), 10, false));
queries.addAll(fillGroupTo(rootBX, ImmutableSet.of(), 10, true));
queries.addAll(fillGroupTo(rootBY, ImmutableSet.of(), 10, true));
updateStateCacheStore(queries);
root.processQueuedQueries();
ResourceGroupInfo info = root.getInfo();
assertEquals(info.getNumRunningQueries(), 0);
assertEquals(info.getNumQueuedQueries(), 40);
root.setHardConcurrencyLimit(10);
root.processQueuedQueries();
updateStateCacheStore(queries);
root.processQueuedQueries();
info = root.getInfo();
assertEquals(info.getNumRunningQueries(), 10);
assertEquals(info.getNumQueuedQueries(), 30);
// Complete running queries
Iterator<MockManagedQueryExecution> iterator = queries.iterator();
while (iterator.hasNext()) {
MockManagedQueryExecution query = iterator.next();
if (query.getState() == RUNNING) {
query.complete();
iterator.remove();
updateQueryStateCache(query);
}
}
// 10 more queries start running, 20 left queued.
root.processQueuedQueries();
updateStateCacheStore(queries);
root.processQueuedQueries();
info = root.getInfo();
assertEquals(info.getNumRunningQueries(), 10);
assertEquals(info.getNumQueuedQueries(), 20);
StateCacheStore.get().resetCachedStates();
}
}
use of io.prestosql.server.ResourceGroupInfo in project hetu-core by openlookeng.
the class TestResourceGroups method testGetResourceGroupStateInfo.
@Test
public void testGetResourceGroupStateInfo() {
InternalResourceGroup root = new InternalResourceGroup(Optional.empty(), "root", (group, export) -> {
}, directExecutor());
root.setSoftMemoryLimit(new DataSize(1, GIGABYTE));
root.setMaxQueuedQueries(40);
root.setHardConcurrencyLimit(10);
root.setSchedulingPolicy(WEIGHTED);
InternalResourceGroup rootA = root.getOrCreateSubGroup("a");
rootA.setSoftMemoryLimit(new DataSize(10, MEGABYTE));
rootA.setMaxQueuedQueries(20);
rootA.setHardConcurrencyLimit(0);
InternalResourceGroup rootB = root.getOrCreateSubGroup("b");
rootB.setSoftMemoryLimit(new DataSize(5, MEGABYTE));
rootB.setMaxQueuedQueries(20);
rootB.setHardConcurrencyLimit(1);
rootB.setSchedulingWeight(2);
rootB.setSchedulingPolicy(QUERY_PRIORITY);
InternalResourceGroup rootAX = rootA.getOrCreateSubGroup("x");
rootAX.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
rootAX.setMaxQueuedQueries(10);
rootAX.setHardConcurrencyLimit(10);
InternalResourceGroup rootAY = rootA.getOrCreateSubGroup("y");
rootAY.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
rootAY.setMaxQueuedQueries(10);
rootAY.setHardConcurrencyLimit(10);
Set<MockManagedQueryExecution> queries = fillGroupTo(rootAX, ImmutableSet.of(), 5, false);
queries.addAll(fillGroupTo(rootAY, ImmutableSet.of(), 5, false));
queries.addAll(fillGroupTo(rootB, ImmutableSet.of(), 10, true));
ResourceGroupInfo rootInfo = root.getFullInfo();
assertEquals(rootInfo.getId(), root.getId());
assertEquals(rootInfo.getState(), CAN_RUN);
assertEquals(rootInfo.getSoftMemoryLimit(), root.getSoftMemoryLimit());
assertEquals(rootInfo.getMemoryUsage(), new DataSize(0, BYTE));
assertEquals(rootInfo.getSubGroups().size(), 2);
assertGroupInfoEquals(rootInfo.getSubGroups().get(0), rootA.getInfo());
assertEquals(rootInfo.getSubGroups().get(0).getId(), rootA.getId());
assertEquals(rootInfo.getSubGroups().get(0).getState(), CAN_QUEUE);
assertEquals(rootInfo.getSubGroups().get(0).getSoftMemoryLimit(), rootA.getSoftMemoryLimit());
assertEquals(rootInfo.getSubGroups().get(0).getHardConcurrencyLimit(), rootA.getHardConcurrencyLimit());
assertEquals(rootInfo.getSubGroups().get(0).getMaxQueuedQueries(), rootA.getMaxQueuedQueries());
assertEquals(rootInfo.getSubGroups().get(0).getNumEligibleSubGroups(), 2);
assertEquals(rootInfo.getSubGroups().get(0).getNumRunningQueries(), 0);
assertEquals(rootInfo.getSubGroups().get(0).getNumQueuedQueries(), 10);
assertGroupInfoEquals(rootInfo.getSubGroups().get(1), rootB.getInfo());
assertEquals(rootInfo.getSubGroups().get(1).getId(), rootB.getId());
assertEquals(rootInfo.getSubGroups().get(1).getState(), CAN_QUEUE);
assertEquals(rootInfo.getSubGroups().get(1).getSoftMemoryLimit(), rootB.getSoftMemoryLimit());
assertEquals(rootInfo.getSubGroups().get(1).getHardConcurrencyLimit(), rootB.getHardConcurrencyLimit());
assertEquals(rootInfo.getSubGroups().get(1).getMaxQueuedQueries(), rootB.getMaxQueuedQueries());
assertEquals(rootInfo.getSubGroups().get(1).getNumEligibleSubGroups(), 0);
assertEquals(rootInfo.getSubGroups().get(1).getNumRunningQueries(), 1);
assertEquals(rootInfo.getSubGroups().get(1).getNumQueuedQueries(), 9);
assertEquals(rootInfo.getSoftConcurrencyLimit(), root.getSoftConcurrencyLimit());
assertEquals(rootInfo.getHardConcurrencyLimit(), root.getHardConcurrencyLimit());
assertEquals(rootInfo.getMaxQueuedQueries(), root.getMaxQueuedQueries());
assertEquals(rootInfo.getNumQueuedQueries(), 19);
assertEquals(rootInfo.getRunningQueries().size(), 1);
QueryStateInfo queryInfo = rootInfo.getRunningQueries().get(0);
assertEquals(queryInfo.getResourceGroupId(), Optional.of(rootB.getId()));
}
use of io.prestosql.server.ResourceGroupInfo in project hetu-core by openlookeng.
the class TestResourceGroups method testGetInfo.
@Test
public void testGetInfo() {
InternalResourceGroup root = new InternalResourceGroup(Optional.empty(), "root", (group, export) -> {
}, directExecutor());
root.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
root.setMaxQueuedQueries(40);
// Start with zero capacity, so that nothing starts running until we've added all the queries
root.setHardConcurrencyLimit(0);
root.setSchedulingPolicy(WEIGHTED);
InternalResourceGroup rootA = root.getOrCreateSubGroup("a");
rootA.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
rootA.setMaxQueuedQueries(20);
rootA.setHardConcurrencyLimit(2);
InternalResourceGroup rootB = root.getOrCreateSubGroup("b");
rootB.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
rootB.setMaxQueuedQueries(20);
rootB.setHardConcurrencyLimit(2);
rootB.setSchedulingWeight(2);
rootB.setSchedulingPolicy(QUERY_PRIORITY);
InternalResourceGroup rootAX = rootA.getOrCreateSubGroup("x");
rootAX.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
rootAX.setMaxQueuedQueries(10);
rootAX.setHardConcurrencyLimit(10);
InternalResourceGroup rootAY = rootA.getOrCreateSubGroup("y");
rootAY.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
rootAY.setMaxQueuedQueries(10);
rootAY.setHardConcurrencyLimit(10);
InternalResourceGroup rootBX = rootB.getOrCreateSubGroup("x");
rootBX.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
rootBX.setMaxQueuedQueries(10);
rootBX.setHardConcurrencyLimit(10);
InternalResourceGroup rootBY = rootB.getOrCreateSubGroup("y");
rootBY.setSoftMemoryLimit(new DataSize(1, MEGABYTE));
rootBY.setMaxQueuedQueries(10);
rootBY.setHardConcurrencyLimit(10);
// Queue 40 queries (= maxQueuedQueries (40) + maxRunningQueries (0))
Set<MockManagedQueryExecution> queries = fillGroupTo(rootAX, ImmutableSet.of(), 10, false);
queries.addAll(fillGroupTo(rootAY, ImmutableSet.of(), 10, false));
queries.addAll(fillGroupTo(rootBX, ImmutableSet.of(), 10, true));
queries.addAll(fillGroupTo(rootBY, ImmutableSet.of(), 10, true));
ResourceGroupInfo info = root.getInfo();
assertEquals(info.getNumRunningQueries(), 0);
assertEquals(info.getNumQueuedQueries(), 40);
// root.maxRunningQueries = 4, root.a.maxRunningQueries = 2, root.b.maxRunningQueries = 2. Will have 4 queries running and 36 left queued.
root.setHardConcurrencyLimit(4);
root.processQueuedQueries();
info = root.getInfo();
assertEquals(info.getNumRunningQueries(), 4);
assertEquals(info.getNumQueuedQueries(), 36);
// Complete running queries
Iterator<MockManagedQueryExecution> iterator = queries.iterator();
while (iterator.hasNext()) {
MockManagedQueryExecution query = iterator.next();
if (query.getState() == RUNNING) {
query.complete();
iterator.remove();
}
}
// 4 more queries start running, 32 left queued.
root.processQueuedQueries();
info = root.getInfo();
assertEquals(info.getNumRunningQueries(), 4);
assertEquals(info.getNumQueuedQueries(), 32);
// root.maxRunningQueries = 10, root.a.maxRunningQueries = 2, root.b.maxRunningQueries = 2. Still only have 4 running queries and 32 left queued.
root.setHardConcurrencyLimit(10);
root.processQueuedQueries();
info = root.getInfo();
assertEquals(info.getNumRunningQueries(), 4);
assertEquals(info.getNumQueuedQueries(), 32);
// root.maxRunningQueries = 10, root.a.maxRunningQueries = 2, root.b.maxRunningQueries = 10. Will have 10 running queries and 26 left queued.
rootB.setHardConcurrencyLimit(10);
root.processQueuedQueries();
info = root.getInfo();
assertEquals(info.getNumRunningQueries(), 10);
assertEquals(info.getNumQueuedQueries(), 26);
}
Aggregations