use of io.prestosql.dispatcher.DispatchManager in project hetu-core by openlookeng.
the class TestQueuesDb method testRejection.
@Test(timeOut = 60_000)
public void testRejection() throws Exception {
InternalResourceGroupManager<?> manager = queryRunner.getCoordinator().getResourceGroupManager().get();
DbResourceGroupConfigurationManager dbConfigurationManager = (DbResourceGroupConfigurationManager) manager.getConfigurationManager();
// Verify the query cannot be submitted
QueryId queryId = createQuery(queryRunner, rejectingSession(), LONG_LASTING_QUERY);
waitForQueryState(queryRunner, queryId, FAILED);
DispatchManager dispatchManager = queryRunner.getCoordinator().getDispatchManager();
assertEquals(dispatchManager.getQueryInfo(queryId).getErrorCode(), QUERY_REJECTED.toErrorCode());
int selectorCount = getSelectors(queryRunner).size();
dao.insertSelector(4, 100_000, "user.*", "(?i).*reject.*", null, null, null);
dbConfigurationManager.load();
assertEquals(getSelectors(queryRunner).size(), selectorCount + 1);
// Verify the query can be submitted
queryId = createQuery(queryRunner, rejectingSession(), LONG_LASTING_QUERY);
waitForQueryState(queryRunner, queryId, RUNNING);
dao.deleteSelector(4, "user.*", "(?i).*reject.*", null);
dbConfigurationManager.load();
// Verify the query cannot be submitted
queryId = createQuery(queryRunner, rejectingSession(), LONG_LASTING_QUERY);
waitForQueryState(queryRunner, queryId, FAILED);
}
use of io.prestosql.dispatcher.DispatchManager in project hetu-core by openlookeng.
the class TestQueryManager method testFailQuery.
@Test(timeOut = 60_000L)
public void testFailQuery() throws Exception {
DispatchManager dispatchManager = queryRunner.getCoordinator().getDispatchManager();
QueryId queryId = dispatchManager.createQueryId();
dispatchManager.createQuery(queryId, "slug", new TestingSessionContext(TEST_SESSION), "SELECT * FROM lineitem").get();
// wait until query starts running
while (true) {
QueryState state = dispatchManager.getQueryInfo(queryId).getState();
if (state.isDone()) {
fail("unexpected query state: " + state);
}
if (state == RUNNING) {
break;
}
Thread.sleep(100);
}
// cancel query
QueryManager queryManager = queryRunner.getCoordinator().getQueryManager();
queryManager.failQuery(queryId, new PrestoException(GENERIC_INTERNAL_ERROR, "mock exception"));
QueryInfo queryInfo = queryManager.getFullQueryInfo(queryId);
assertEquals(queryInfo.getState(), FAILED);
assertEquals(queryInfo.getErrorCode(), GENERIC_INTERNAL_ERROR.toErrorCode());
assertNotNull(queryInfo.getFailureInfo());
assertEquals(queryInfo.getFailureInfo().getMessage(), "mock exception");
}
use of io.prestosql.dispatcher.DispatchManager in project hetu-core by openlookeng.
the class AbstractTestDistributedQueries method testQueryLoggingCount.
@Test
public void testQueryLoggingCount() {
QueryManager queryManager = ((DistributedQueryRunner) getQueryRunner()).getCoordinator().getQueryManager();
executeExclusively(() -> {
assertUntilTimeout(() -> assertEquals(queryManager.getQueries().stream().map(BasicQueryInfo::getQueryId).map(queryManager::getFullQueryInfo).filter(info -> !info.isFinalQueryInfo()).collect(toList()), ImmutableList.of()), new Duration(1, MINUTES));
// We cannot simply get the number of completed queries as soon as all the queries are completed, because this counter may not be up-to-date at that point.
// The completed queries counter is updated in a final query info listener, which is called eventually.
// Therefore, here we wait until the value of this counter gets stable.
DispatchManager dispatchManager = ((DistributedQueryRunner) getQueryRunner()).getCoordinator().getDispatchManager();
long beforeCompletedQueriesCount = waitUntilStable(() -> dispatchManager.getStats().getCompletedQueries().getTotalCount(), new Duration(5, SECONDS));
long beforeSubmittedQueriesCount = dispatchManager.getStats().getSubmittedQueries().getTotalCount();
assertUpdate("CREATE TABLE test_query_logging_count AS SELECT 1 foo_1, 2 foo_2_4", 1);
assertQuery("SELECT foo_1, foo_2_4 FROM test_query_logging_count", "SELECT 1, 2");
assertUpdate("DROP TABLE test_query_logging_count");
assertQueryFails("SELECT * FROM test_query_logging_count", ".*Table .* does not exist");
// TODO: Figure out a better way of synchronization
assertUntilTimeout(() -> assertEquals(dispatchManager.getStats().getCompletedQueries().getTotalCount() - beforeCompletedQueriesCount, 4), new Duration(1, MINUTES));
assertEquals(dispatchManager.getStats().getSubmittedQueries().getTotalCount() - beforeSubmittedQueriesCount, 4);
});
}
use of io.prestosql.dispatcher.DispatchManager in project hetu-core by openlookeng.
the class TestQueuesDb method testSelectorPriority.
@Test(timeOut = 60_000)
public void testSelectorPriority() throws Exception {
InternalResourceGroupManager<?> manager = queryRunner.getCoordinator().getResourceGroupManager().get();
QueryManager queryManager = queryRunner.getCoordinator().getQueryManager();
DbResourceGroupConfigurationManager dbConfigurationManager = (DbResourceGroupConfigurationManager) manager.getConfigurationManager();
QueryId firstQuery = createQuery(queryRunner, dashboardSession(), LONG_LASTING_QUERY);
waitForQueryState(queryRunner, firstQuery, RUNNING);
Optional<ResourceGroupId> resourceGroup = queryManager.getFullQueryInfo(firstQuery).getResourceGroupId();
assertTrue(resourceGroup.isPresent());
assertEquals(resourceGroup.get().toString(), "global.user-user.dashboard-user");
// create a new resource group that rejects all queries submitted to it
// Hetu: add parameters softReservedMemory and hardReservedConcurrency
dao.insertResourceGroup(8, "reject-all-queries", "1MB", "1MB", 0, 0, 0, 0, null, null, null, null, null, "RECENT_QUERIES", 3L, TEST_ENVIRONMENT);
// add a new selector that has a higher priority than the existing dashboard selector and that routes queries to the "reject-all-queries" resource group
dao.insertSelector(8, 200, "user.*", "(?i).*dashboard.*", null, null, null);
// reload the configuration
dbConfigurationManager.load();
QueryId secondQuery = createQuery(queryRunner, dashboardSession(), LONG_LASTING_QUERY);
waitForQueryState(queryRunner, secondQuery, FAILED);
DispatchManager dispatchManager = queryRunner.getCoordinator().getDispatchManager();
BasicQueryInfo basicQueryInfo = dispatchManager.getQueryInfo(secondQuery);
assertEquals(basicQueryInfo.getErrorCode(), QUERY_QUEUE_FULL.toErrorCode());
}
use of io.prestosql.dispatcher.DispatchManager in project hetu-core by openlookeng.
the class TestQueuesDb method testQueryExecutionTimeLimit.
@Test(timeOut = 60_000)
public void testQueryExecutionTimeLimit() throws Exception {
QueryManager queryManager = queryRunner.getCoordinator().getQueryManager();
InternalResourceGroupManager<?> manager = queryRunner.getCoordinator().getResourceGroupManager().get();
DbResourceGroupConfigurationManager dbConfigurationManager = (DbResourceGroupConfigurationManager) manager.getConfigurationManager();
QueryId firstQuery = createQuery(queryRunner, testSessionBuilder().setCatalog("tpch").setSchema("sf100000").setSource("dashboard").setSystemProperty(QUERY_MAX_EXECUTION_TIME, "1ms").build(), LONG_LASTING_QUERY);
waitForQueryState(queryRunner, firstQuery, FAILED);
assertEquals(queryManager.getFullQueryInfo(firstQuery).getErrorCode(), EXCEEDED_TIME_LIMIT.toErrorCode());
assertContains(queryManager.getFullQueryInfo(firstQuery).getFailureInfo().getMessage(), "Query exceeded the maximum execution time limit of 1.00ms");
// set max running queries to 0 for the dashboard resource group so that new queries get queued immediately
// Hetu: add parameters softReservedMemory and hardReservedConcurrency
dao.updateResourceGroup(5, "dashboard-${USER}", "1MB", "1MB", 1, null, 0, 0, null, null, null, null, null, "RECENT_QUERIES", 3L, TEST_ENVIRONMENT);
dbConfigurationManager.load();
QueryId secondQuery = createQuery(queryRunner, testSessionBuilder().setCatalog("tpch").setSchema("sf100000").setSource("dashboard").setSystemProperty(QUERY_MAX_EXECUTION_TIME, "1ms").build(), LONG_LASTING_QUERY);
// this query should immediately get queued
waitForQueryState(queryRunner, secondQuery, QUEUED);
// after a 5s wait this query should still be QUEUED, not FAILED as the max execution time should be enforced after the query starts running
Thread.sleep(5_000);
DispatchManager dispatchManager = queryRunner.getCoordinator().getDispatchManager();
assertEquals(dispatchManager.getQueryInfo(secondQuery).getState(), QUEUED);
// reconfigure the resource group to run the second query
// Hetu: add parameters softReservedMemory and hardReservedConcurrency
dao.updateResourceGroup(5, "dashboard-${USER}", "1MB", "1MB", 1, null, 1, 0, null, null, null, null, null, "RECENT_QUERIES", 3L, TEST_ENVIRONMENT);
dbConfigurationManager.load();
// cancel the first one and let the second one start
dispatchManager.cancelQuery(firstQuery);
// wait until the second one is FAILED
waitForQueryState(queryRunner, secondQuery, FAILED);
}
Aggregations