use of com.facebook.presto.server.BasicQueryInfo in project presto by prestodb.
the class TestDistributedQueryResource method assertStateCounts.
private void assertStateCounts(List<BasicQueryInfo> infos, int expectedFinished, int expectedFailed, int expectedRunning, int expectedQueued) {
int failed = 0;
int finished = 0;
int running = 0;
int queued = 0;
for (BasicQueryInfo info : infos) {
switch(info.getState()) {
case RUNNING:
case FINISHING:
running++;
break;
case WAITING_FOR_RESOURCES:
case PLANNING:
case DISPATCHING:
case QUEUED:
queued++;
break;
case FINISHED:
finished++;
break;
case FAILED:
failed++;
break;
default:
fail("Unexpected query state " + info.getState());
}
}
assertEquals(failed, expectedFailed);
assertEquals(finished, expectedFinished);
assertEquals(running, expectedRunning);
assertEquals(queued, expectedQueued);
}
use of com.facebook.presto.server.BasicQueryInfo in project presto by prestodb.
the class TestMemoryManager method testClusterPoolsMultiCoordinator.
@Test(timeOut = 60_000)
public void testClusterPoolsMultiCoordinator() throws Exception {
Map<String, String> properties = ImmutableMap.<String, String>builder().put("task.verbose-stats", "true").put("resource-manager.memory-pool-fetch-interval", "10ms").put("resource-manager.query-heartbeat-interval", "10ms").put("resource-manager.node-status-timeout", "5s").build();
try (DistributedQueryRunner queryRunner = createQueryRunner(properties, ImmutableMap.of(), properties, 2)) {
// Reserve all the memory
QueryId fakeQueryId = new QueryId("fake");
for (TestingPrestoServer server : queryRunner.getCoordinatorWorkers()) {
for (MemoryPool pool : server.getLocalMemoryManager().getPools()) {
assertTrue(pool.tryReserve(fakeQueryId, "test", pool.getMaxBytes()));
}
}
List<Future<?>> queryFutures = new ArrayList<>();
for (int i = 0; i < 2; i++) {
int coordinator = i;
queryFutures.add(executor.submit(() -> queryRunner.execute(coordinator, "SELECT COUNT(*), clerk FROM orders GROUP BY clerk")));
}
ClusterMemoryManager memoryManager = queryRunner.getCoordinator(0).getClusterMemoryManager();
ClusterMemoryPoolInfo reservedPool;
while ((reservedPool = memoryManager.getClusterInfo(RESERVED_POOL)) == null) {
MILLISECONDS.sleep(10);
}
ClusterMemoryPoolInfo generalPool = memoryManager.getClusterInfo(GENERAL_POOL);
assertNotNull(generalPool);
// Wait for the queries to start running and get assigned to the expected pools
while (generalPool.getAssignedQueries() != 1 || reservedPool.getAssignedQueries() != 1 || generalPool.getBlockedNodes() != 3 || reservedPool.getBlockedNodes() != 3) {
generalPool = memoryManager.getClusterInfo(GENERAL_POOL);
reservedPool = memoryManager.getClusterInfo(RESERVED_POOL);
MILLISECONDS.sleep(10);
}
// Make sure the queries are blocked
List<BasicQueryInfo> currentQueryInfos;
do {
currentQueryInfos = queryRunner.getCoordinators().stream().map(TestingPrestoServer::getQueryManager).map(QueryManager::getQueries).flatMap(Collection::stream).collect(toImmutableList());
MILLISECONDS.sleep(10);
} while (currentQueryInfos.size() != 2);
for (BasicQueryInfo info : currentQueryInfos) {
assertFalse(info.getState().isDone());
}
// Check that the pool information propagated to the query objects
assertNotEquals(currentQueryInfos.get(0).getMemoryPool(), currentQueryInfos.get(1).getMemoryPool());
while (!currentQueryInfos.stream().allMatch(TestMemoryManager::isBlockedWaitingForMemory)) {
MILLISECONDS.sleep(10);
currentQueryInfos = queryRunner.getCoordinators().stream().map(TestingPrestoServer::getQueryManager).map(QueryManager::getQueries).flatMap(Collection::stream).collect(toImmutableList());
for (BasicQueryInfo info : currentQueryInfos) {
assertFalse(info.getState().isDone());
}
}
// Release the memory in the reserved pool
for (TestingPrestoServer server : queryRunner.getCoordinatorWorkers()) {
Optional<MemoryPool> reserved = server.getLocalMemoryManager().getReservedPool();
assertTrue(reserved.isPresent());
// Free up the entire pool
reserved.get().free(fakeQueryId, "test", reserved.get().getMaxBytes());
assertTrue(reserved.get().getFreeBytes() > 0);
}
// This also checks that the query in the general pool is successfully moved to the reserved pool.
for (Future<?> query : queryFutures) {
query.get();
}
queryRunner.getCoordinators().stream().map(TestingPrestoServer::getQueryManager).map(QueryManager::getQueries).flatMap(Collection::stream).forEach(info -> assertEquals(info.getState(), FINISHED));
// Make sure we didn't leak any memory on the workers
for (TestingPrestoServer worker : queryRunner.getCoordinatorWorkers()) {
Optional<MemoryPool> reserved = worker.getLocalMemoryManager().getReservedPool();
assertTrue(reserved.isPresent());
assertEquals(reserved.get().getMaxBytes(), reserved.get().getFreeBytes());
MemoryPool general = worker.getLocalMemoryManager().getGeneralPool();
// Free up the memory we reserved earlier
general.free(fakeQueryId, "test", general.getMaxBytes());
assertEquals(general.getMaxBytes(), general.getFreeBytes());
}
}
}
Aggregations