Search in sources :

Example 1 with DispatchManager

use of io.prestosql.dispatcher.DispatchManager in project hetu-core by openlookeng.

the class TestQueuesDb method testRejection.

@Test(timeOut = 60_000)
public void testRejection() throws Exception {
    InternalResourceGroupManager<?> manager = queryRunner.getCoordinator().getResourceGroupManager().get();
    DbResourceGroupConfigurationManager dbConfigurationManager = (DbResourceGroupConfigurationManager) manager.getConfigurationManager();
    // Verify the query cannot be submitted
    QueryId queryId = createQuery(queryRunner, rejectingSession(), LONG_LASTING_QUERY);
    waitForQueryState(queryRunner, queryId, FAILED);
    DispatchManager dispatchManager = queryRunner.getCoordinator().getDispatchManager();
    assertEquals(dispatchManager.getQueryInfo(queryId).getErrorCode(), QUERY_REJECTED.toErrorCode());
    int selectorCount = getSelectors(queryRunner).size();
    dao.insertSelector(4, 100_000, "user.*", "(?i).*reject.*", null, null, null);
    dbConfigurationManager.load();
    assertEquals(getSelectors(queryRunner).size(), selectorCount + 1);
    // Verify the query can be submitted
    queryId = createQuery(queryRunner, rejectingSession(), LONG_LASTING_QUERY);
    waitForQueryState(queryRunner, queryId, RUNNING);
    dao.deleteSelector(4, "user.*", "(?i).*reject.*", null);
    dbConfigurationManager.load();
    // Verify the query cannot be submitted
    queryId = createQuery(queryRunner, rejectingSession(), LONG_LASTING_QUERY);
    waitForQueryState(queryRunner, queryId, FAILED);
}
Also used : DbResourceGroupConfigurationManager(io.prestosql.plugin.resourcegroups.db.DbResourceGroupConfigurationManager) DispatchManager(io.prestosql.dispatcher.DispatchManager) QueryId(io.prestosql.spi.QueryId) Test(org.testng.annotations.Test)

Example 2 with DispatchManager

use of io.prestosql.dispatcher.DispatchManager in project hetu-core by openlookeng.

the class TestQueryManager method testFailQuery.

@Test(timeOut = 60_000L)
public void testFailQuery() throws Exception {
    DispatchManager dispatchManager = queryRunner.getCoordinator().getDispatchManager();
    QueryId queryId = dispatchManager.createQueryId();
    dispatchManager.createQuery(queryId, "slug", new TestingSessionContext(TEST_SESSION), "SELECT * FROM lineitem").get();
    // wait until query starts running
    while (true) {
        QueryState state = dispatchManager.getQueryInfo(queryId).getState();
        if (state.isDone()) {
            fail("unexpected query state: " + state);
        }
        if (state == RUNNING) {
            break;
        }
        Thread.sleep(100);
    }
    // cancel query
    QueryManager queryManager = queryRunner.getCoordinator().getQueryManager();
    queryManager.failQuery(queryId, new PrestoException(GENERIC_INTERNAL_ERROR, "mock exception"));
    QueryInfo queryInfo = queryManager.getFullQueryInfo(queryId);
    assertEquals(queryInfo.getState(), FAILED);
    assertEquals(queryInfo.getErrorCode(), GENERIC_INTERNAL_ERROR.toErrorCode());
    assertNotNull(queryInfo.getFailureInfo());
    assertEquals(queryInfo.getFailureInfo().getMessage(), "mock exception");
}
Also used : DispatchManager(io.prestosql.dispatcher.DispatchManager) TestingSessionContext(io.prestosql.execution.TestingSessionContext) QueryId(io.prestosql.spi.QueryId) QueryManager(io.prestosql.execution.QueryManager) PrestoException(io.prestosql.spi.PrestoException) TestQueryRunnerUtil.waitForQueryState(io.prestosql.execution.TestQueryRunnerUtil.waitForQueryState) QueryState(io.prestosql.execution.QueryState) QueryInfo(io.prestosql.execution.QueryInfo) BasicQueryInfo(io.prestosql.server.BasicQueryInfo) Test(org.testng.annotations.Test)

Example 3 with DispatchManager

use of io.prestosql.dispatcher.DispatchManager in project hetu-core by openlookeng.

the class AbstractTestDistributedQueries method testQueryLoggingCount.

@Test
public void testQueryLoggingCount() {
    QueryManager queryManager = ((DistributedQueryRunner) getQueryRunner()).getCoordinator().getQueryManager();
    executeExclusively(() -> {
        assertUntilTimeout(() -> assertEquals(queryManager.getQueries().stream().map(BasicQueryInfo::getQueryId).map(queryManager::getFullQueryInfo).filter(info -> !info.isFinalQueryInfo()).collect(toList()), ImmutableList.of()), new Duration(1, MINUTES));
        // We cannot simply get the number of completed queries as soon as all the queries are completed, because this counter may not be up-to-date at that point.
        // The completed queries counter is updated in a final query info listener, which is called eventually.
        // Therefore, here we wait until the value of this counter gets stable.
        DispatchManager dispatchManager = ((DistributedQueryRunner) getQueryRunner()).getCoordinator().getDispatchManager();
        long beforeCompletedQueriesCount = waitUntilStable(() -> dispatchManager.getStats().getCompletedQueries().getTotalCount(), new Duration(5, SECONDS));
        long beforeSubmittedQueriesCount = dispatchManager.getStats().getSubmittedQueries().getTotalCount();
        assertUpdate("CREATE TABLE test_query_logging_count AS SELECT 1 foo_1, 2 foo_2_4", 1);
        assertQuery("SELECT foo_1, foo_2_4 FROM test_query_logging_count", "SELECT 1, 2");
        assertUpdate("DROP TABLE test_query_logging_count");
        assertQueryFails("SELECT * FROM test_query_logging_count", ".*Table .* does not exist");
        // TODO: Figure out a better way of synchronization
        assertUntilTimeout(() -> assertEquals(dispatchManager.getStats().getCompletedQueries().getTotalCount() - beforeCompletedQueriesCount, 4), new Duration(1, MINUTES));
        assertEquals(dispatchManager.getStats().getSubmittedQueries().getTotalCount() - beforeSubmittedQueriesCount, 4);
    });
}
Also used : Arrays(java.util.Arrays) TestingSession(io.prestosql.testing.TestingSession) CREATE_VIEW_WITH_SELECT_COLUMNS(io.prestosql.testing.TestingAccessControlManager.TestingPrivilegeType.CREATE_VIEW_WITH_SELECT_COLUMNS) SystemSessionProperties(io.prestosql.SystemSessionProperties) QueryInfo(io.prestosql.execution.QueryInfo) Test(org.testng.annotations.Test) Thread.currentThread(java.lang.Thread.currentThread) INFORMATION_SCHEMA(io.prestosql.connector.informationschema.InformationSchemaMetadata.INFORMATION_SCHEMA) MaterializedResult(io.prestosql.testing.MaterializedResult) Duration(io.airlift.units.Duration) ADD_COLUMN(io.prestosql.testing.TestingAccessControlManager.TestingPrivilegeType.ADD_COLUMN) RENAME_TABLE(io.prestosql.testing.TestingAccessControlManager.TestingPrivilegeType.RENAME_TABLE) SET_SESSION(io.prestosql.testing.TestingAccessControlManager.TestingPrivilegeType.SET_SESSION) Assertions(io.airlift.testing.Assertions) QueryManager(io.prestosql.execution.QueryManager) Assert.assertEquals(io.prestosql.testing.assertions.Assert.assertEquals) Duration.nanosSince(io.airlift.units.Duration.nanosSince) Assert.assertFalse(org.testng.Assert.assertFalse) CREATE_TABLE(io.prestosql.testing.TestingAccessControlManager.TestingPrivilegeType.CREATE_TABLE) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) Collections.nCopies(java.util.Collections.nCopies) Identity(io.prestosql.spi.security.Identity) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) Uninterruptibles.sleepUninterruptibly(com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly) DROP_COLUMN(io.prestosql.testing.TestingAccessControlManager.TestingPrivilegeType.DROP_COLUMN) String.format(java.lang.String.format) Preconditions.checkState(com.google.common.base.Preconditions.checkState) List(java.util.List) Optional(java.util.Optional) MaterializedResult.resultBuilder(io.prestosql.testing.MaterializedResult.resultBuilder) CREATE_VIEW(io.prestosql.testing.TestingAccessControlManager.TestingPrivilegeType.CREATE_VIEW) SET_USER(io.prestosql.testing.TestingAccessControlManager.TestingPrivilegeType.SET_USER) Joiner(com.google.common.base.Joiner) Assert.assertNull(org.testng.Assert.assertNull) QueryAssertions.assertContains(io.prestosql.tests.QueryAssertions.assertContains) TESTING_CATALOG(io.prestosql.testing.TestingSession.TESTING_CATALOG) MINUTES(java.util.concurrent.TimeUnit.MINUTES) Supplier(java.util.function.Supplier) VARCHAR(io.prestosql.spi.type.VarcharType.VARCHAR) TestingAccessControlManager.privilege(io.prestosql.testing.TestingAccessControlManager.privilege) ImmutableList(com.google.common.collect.ImmutableList) BasicQueryInfo(io.prestosql.server.BasicQueryInfo) Session(io.prestosql.Session) UncheckedTimeoutException(com.google.common.util.concurrent.UncheckedTimeoutException) DispatchManager(io.prestosql.dispatcher.DispatchManager) Language(org.intellij.lang.annotations.Language) QUERY_MAX_MEMORY(io.prestosql.SystemSessionProperties.QUERY_MAX_MEMORY) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) SELECT_COLUMN(io.prestosql.testing.TestingAccessControlManager.TestingPrivilegeType.SELECT_COLUMN) MaterializedRow(io.prestosql.testing.MaterializedRow) Collectors.toList(java.util.stream.Collectors.toList) DROP_TABLE(io.prestosql.testing.TestingAccessControlManager.TestingPrivilegeType.DROP_TABLE) Assert.assertTrue(org.testng.Assert.assertTrue) RENAME_COLUMN(io.prestosql.testing.TestingAccessControlManager.TestingPrivilegeType.RENAME_COLUMN) SECONDS(java.util.concurrent.TimeUnit.SECONDS) DispatchManager(io.prestosql.dispatcher.DispatchManager) QueryManager(io.prestosql.execution.QueryManager) Duration(io.airlift.units.Duration) Test(org.testng.annotations.Test)

Example 4 with DispatchManager

use of io.prestosql.dispatcher.DispatchManager in project hetu-core by openlookeng.

the class TestQueuesDb method testSelectorPriority.

@Test(timeOut = 60_000)
public void testSelectorPriority() throws Exception {
    InternalResourceGroupManager<?> manager = queryRunner.getCoordinator().getResourceGroupManager().get();
    QueryManager queryManager = queryRunner.getCoordinator().getQueryManager();
    DbResourceGroupConfigurationManager dbConfigurationManager = (DbResourceGroupConfigurationManager) manager.getConfigurationManager();
    QueryId firstQuery = createQuery(queryRunner, dashboardSession(), LONG_LASTING_QUERY);
    waitForQueryState(queryRunner, firstQuery, RUNNING);
    Optional<ResourceGroupId> resourceGroup = queryManager.getFullQueryInfo(firstQuery).getResourceGroupId();
    assertTrue(resourceGroup.isPresent());
    assertEquals(resourceGroup.get().toString(), "global.user-user.dashboard-user");
    // create a new resource group that rejects all queries submitted to it
    // Hetu: add parameters softReservedMemory and hardReservedConcurrency
    dao.insertResourceGroup(8, "reject-all-queries", "1MB", "1MB", 0, 0, 0, 0, null, null, null, null, null, "RECENT_QUERIES", 3L, TEST_ENVIRONMENT);
    // add a new selector that has a higher priority than the existing dashboard selector and that routes queries to the "reject-all-queries" resource group
    dao.insertSelector(8, 200, "user.*", "(?i).*dashboard.*", null, null, null);
    // reload the configuration
    dbConfigurationManager.load();
    QueryId secondQuery = createQuery(queryRunner, dashboardSession(), LONG_LASTING_QUERY);
    waitForQueryState(queryRunner, secondQuery, FAILED);
    DispatchManager dispatchManager = queryRunner.getCoordinator().getDispatchManager();
    BasicQueryInfo basicQueryInfo = dispatchManager.getQueryInfo(secondQuery);
    assertEquals(basicQueryInfo.getErrorCode(), QUERY_QUEUE_FULL.toErrorCode());
}
Also used : DbResourceGroupConfigurationManager(io.prestosql.plugin.resourcegroups.db.DbResourceGroupConfigurationManager) ResourceGroupId(io.prestosql.spi.resourcegroups.ResourceGroupId) TestQueues.createResourceGroupId(io.prestosql.execution.TestQueues.createResourceGroupId) DispatchManager(io.prestosql.dispatcher.DispatchManager) QueryId(io.prestosql.spi.QueryId) BasicQueryInfo(io.prestosql.server.BasicQueryInfo) QueryManager(io.prestosql.execution.QueryManager) Test(org.testng.annotations.Test)

Example 5 with DispatchManager

use of io.prestosql.dispatcher.DispatchManager in project hetu-core by openlookeng.

the class TestQueuesDb method testQueryExecutionTimeLimit.

@Test(timeOut = 60_000)
public void testQueryExecutionTimeLimit() throws Exception {
    QueryManager queryManager = queryRunner.getCoordinator().getQueryManager();
    InternalResourceGroupManager<?> manager = queryRunner.getCoordinator().getResourceGroupManager().get();
    DbResourceGroupConfigurationManager dbConfigurationManager = (DbResourceGroupConfigurationManager) manager.getConfigurationManager();
    QueryId firstQuery = createQuery(queryRunner, testSessionBuilder().setCatalog("tpch").setSchema("sf100000").setSource("dashboard").setSystemProperty(QUERY_MAX_EXECUTION_TIME, "1ms").build(), LONG_LASTING_QUERY);
    waitForQueryState(queryRunner, firstQuery, FAILED);
    assertEquals(queryManager.getFullQueryInfo(firstQuery).getErrorCode(), EXCEEDED_TIME_LIMIT.toErrorCode());
    assertContains(queryManager.getFullQueryInfo(firstQuery).getFailureInfo().getMessage(), "Query exceeded the maximum execution time limit of 1.00ms");
    // set max running queries to 0 for the dashboard resource group so that new queries get queued immediately
    // Hetu: add parameters softReservedMemory and hardReservedConcurrency
    dao.updateResourceGroup(5, "dashboard-${USER}", "1MB", "1MB", 1, null, 0, 0, null, null, null, null, null, "RECENT_QUERIES", 3L, TEST_ENVIRONMENT);
    dbConfigurationManager.load();
    QueryId secondQuery = createQuery(queryRunner, testSessionBuilder().setCatalog("tpch").setSchema("sf100000").setSource("dashboard").setSystemProperty(QUERY_MAX_EXECUTION_TIME, "1ms").build(), LONG_LASTING_QUERY);
    // this query should immediately get queued
    waitForQueryState(queryRunner, secondQuery, QUEUED);
    // after a 5s wait this query should still be QUEUED, not FAILED as the max execution time should be enforced after the query starts running
    Thread.sleep(5_000);
    DispatchManager dispatchManager = queryRunner.getCoordinator().getDispatchManager();
    assertEquals(dispatchManager.getQueryInfo(secondQuery).getState(), QUEUED);
    // reconfigure the resource group to run the second query
    // Hetu: add parameters softReservedMemory and hardReservedConcurrency
    dao.updateResourceGroup(5, "dashboard-${USER}", "1MB", "1MB", 1, null, 1, 0, null, null, null, null, null, "RECENT_QUERIES", 3L, TEST_ENVIRONMENT);
    dbConfigurationManager.load();
    // cancel the first one and let the second one start
    dispatchManager.cancelQuery(firstQuery);
    // wait until the second one is FAILED
    waitForQueryState(queryRunner, secondQuery, FAILED);
}
Also used : DbResourceGroupConfigurationManager(io.prestosql.plugin.resourcegroups.db.DbResourceGroupConfigurationManager) DispatchManager(io.prestosql.dispatcher.DispatchManager) QueryId(io.prestosql.spi.QueryId) QueryManager(io.prestosql.execution.QueryManager) Test(org.testng.annotations.Test)

Aggregations

DispatchManager (io.prestosql.dispatcher.DispatchManager)9 Test (org.testng.annotations.Test)7 QueryId (io.prestosql.spi.QueryId)6 QueryManager (io.prestosql.execution.QueryManager)5 BasicQueryInfo (io.prestosql.server.BasicQueryInfo)5 QueryInfo (io.prestosql.execution.QueryInfo)3 DbResourceGroupConfigurationManager (io.prestosql.plugin.resourcegroups.db.DbResourceGroupConfigurationManager)3 ImmutableList (com.google.common.collect.ImmutableList)2 ImmutableMap (com.google.common.collect.ImmutableMap)2 ImmutableSet (com.google.common.collect.ImmutableSet)2 Iterables.getOnlyElement (com.google.common.collect.Iterables.getOnlyElement)2 UncheckedTimeoutException (com.google.common.util.concurrent.UncheckedTimeoutException)2 Uninterruptibles.sleepUninterruptibly (com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly)2 Assertions (io.airlift.testing.Assertions)2 Duration (io.airlift.units.Duration)2 Duration.nanosSince (io.airlift.units.Duration.nanosSince)2 Session (io.prestosql.Session)2 SystemSessionProperties (io.prestosql.SystemSessionProperties)2 QUERY_MAX_MEMORY (io.prestosql.SystemSessionProperties.QUERY_MAX_MEMORY)2 INFORMATION_SCHEMA (io.prestosql.connector.informationschema.InformationSchemaMetadata.INFORMATION_SCHEMA)2