Search in sources :

Example 1 with ReloadingResourceGroupConfigurationManager

use of com.facebook.presto.resourceGroups.reloading.ReloadingResourceGroupConfigurationManager in project presto by prestodb.

the class TestQueuesDb method testRejection.

@Test(timeOut = 60_000)
public void testRejection() throws Exception {
    InternalResourceGroupManager manager = queryRunner.getCoordinator().getResourceGroupManager().get();
    ReloadingResourceGroupConfigurationManager reloadingConfigurationManager = (ReloadingResourceGroupConfigurationManager) manager.getConfigurationManager();
    // Verify the query cannot be submitted
    QueryId queryId = createQuery(queryRunner, rejectingSession(), LONG_LASTING_QUERY);
    waitForQueryState(queryRunner, queryId, FAILED);
    DispatchManager dispatchManager = queryRunner.getCoordinator().getDispatchManager();
    assertEquals(dispatchManager.getQueryInfo(queryId).getErrorCode(), QUERY_REJECTED.toErrorCode());
    int selectorCount = getSelectors(queryRunner).size();
    dao.insertSelector(4, 100_000, "user.*", "(?i).*reject.*", null, null, null);
    reloadingConfigurationManager.load();
    assertEquals(getSelectors(queryRunner).size(), selectorCount + 1);
    // Verify the query can be submitted
    queryId = createQuery(queryRunner, rejectingSession(), LONG_LASTING_QUERY);
    waitForQueryState(queryRunner, queryId, RUNNING);
    dao.deleteSelector(4, "user.*", "(?i).*reject.*", null);
    reloadingConfigurationManager.load();
    // Verify the query cannot be submitted
    queryId = createQuery(queryRunner, rejectingSession(), LONG_LASTING_QUERY);
    waitForQueryState(queryRunner, queryId, FAILED);
}
Also used : ReloadingResourceGroupConfigurationManager(com.facebook.presto.resourceGroups.reloading.ReloadingResourceGroupConfigurationManager) DispatchManager(com.facebook.presto.dispatcher.DispatchManager) QueryId(com.facebook.presto.spi.QueryId) InternalResourceGroupManager(com.facebook.presto.execution.resourceGroups.InternalResourceGroupManager) Test(org.testng.annotations.Test)

Example 2 with ReloadingResourceGroupConfigurationManager

use of com.facebook.presto.resourceGroups.reloading.ReloadingResourceGroupConfigurationManager in project presto by prestodb.

the class TestQueuesDb method testTooManyQueries.

// @Test(timeOut = 90_000)
public void testTooManyQueries() throws Exception {
    QueryId firstDashboardQuery = createQuery(queryRunner, dashboardSession(), LONG_LASTING_QUERY);
    waitForQueryState(queryRunner, firstDashboardQuery, RUNNING);
    QueryId secondDashboardQuery = createQuery(queryRunner, dashboardSession(), LONG_LASTING_QUERY);
    waitForQueryState(queryRunner, secondDashboardQuery, QUEUED);
    QueryId thirdDashboardQuery = createQuery(queryRunner, dashboardSession(), LONG_LASTING_QUERY);
    waitForQueryState(queryRunner, thirdDashboardQuery, FAILED);
    // Allow one more query to run and resubmit third query
    dao.updateResourceGroup(3, "user-${USER}", "1MB", 3, 4, 4, null, null, null, null, null, null, null, null, 1L, TEST_ENVIRONMENT);
    dao.updateResourceGroup(5, "dashboard-${USER}", "1MB", 1, 2, 2, null, null, null, null, null, null, null, null, 3L, TEST_ENVIRONMENT);
    InternalResourceGroupManager manager = queryRunner.getCoordinator().getResourceGroupManager().get();
    ReloadingResourceGroupConfigurationManager reloadingConfigurationManager = (ReloadingResourceGroupConfigurationManager) manager.getConfigurationManager();
    // Trigger reload to make the test more deterministic
    reloadingConfigurationManager.load();
    waitForQueryState(queryRunner, secondDashboardQuery, RUNNING);
    thirdDashboardQuery = createQuery(queryRunner, dashboardSession(), LONG_LASTING_QUERY);
    waitForQueryState(queryRunner, thirdDashboardQuery, QUEUED);
    // Lower running queries in dashboard resource groups and reload the config
    dao.updateResourceGroup(5, "dashboard-${USER}", "1MB", 1, 1, 1, null, null, null, null, null, null, null, null, 3L, TEST_ENVIRONMENT);
    reloadingConfigurationManager.load();
    // Cancel query and verify that third query is still queued
    cancelQuery(queryRunner, firstDashboardQuery);
    waitForQueryState(queryRunner, firstDashboardQuery, FAILED);
    MILLISECONDS.sleep(2000);
    waitForQueryState(queryRunner, thirdDashboardQuery, QUEUED);
}
Also used : ReloadingResourceGroupConfigurationManager(com.facebook.presto.resourceGroups.reloading.ReloadingResourceGroupConfigurationManager) QueryId(com.facebook.presto.spi.QueryId) InternalResourceGroupManager(com.facebook.presto.execution.resourceGroups.InternalResourceGroupManager)

Example 3 with ReloadingResourceGroupConfigurationManager

use of com.facebook.presto.resourceGroups.reloading.ReloadingResourceGroupConfigurationManager in project presto by prestodb.

the class TestQueuesDb method testSelectorPriority.

@Test(timeOut = 60_000)
public void testSelectorPriority() throws Exception {
    InternalResourceGroupManager manager = queryRunner.getCoordinator().getResourceGroupManager().get();
    QueryManager queryManager = queryRunner.getCoordinator().getQueryManager();
    ReloadingResourceGroupConfigurationManager reloadingConfigurationManager = (ReloadingResourceGroupConfigurationManager) manager.getConfigurationManager();
    QueryId firstQuery = createQuery(queryRunner, dashboardSession(), LONG_LASTING_QUERY);
    waitForQueryState(queryRunner, firstQuery, RUNNING);
    Optional<ResourceGroupId> resourceGroup = queryManager.getFullQueryInfo(firstQuery).getResourceGroupId();
    assertTrue(resourceGroup.isPresent());
    assertEquals(resourceGroup.get().toString(), "global.user-user.dashboard-user");
    // create a new resource group that rejects all queries submitted to it
    dao.insertResourceGroup(10, "reject-all-queries", "1MB", 0, 0, 0, null, null, null, null, null, null, null, null, 3L, TEST_ENVIRONMENT);
    // add a new selector that has a higher priority than the existing dashboard selector and that routes queries to the "reject-all-queries" resource group
    dao.insertSelector(10, 200, "user.*", "(?i).*dashboard.*", null, null, null);
    // reload the configuration
    reloadingConfigurationManager.load();
    QueryId secondQuery = createQuery(queryRunner, dashboardSession(), LONG_LASTING_QUERY);
    waitForQueryState(queryRunner, secondQuery, FAILED);
    DispatchManager dispatchManager = queryRunner.getCoordinator().getDispatchManager();
    BasicQueryInfo basicQueryInfo = dispatchManager.getQueryInfo(secondQuery);
    assertEquals(basicQueryInfo.getErrorCode(), QUERY_QUEUE_FULL.toErrorCode());
}
Also used : ReloadingResourceGroupConfigurationManager(com.facebook.presto.resourceGroups.reloading.ReloadingResourceGroupConfigurationManager) ResourceGroupId(com.facebook.presto.spi.resourceGroups.ResourceGroupId) TestQueues.createResourceGroupId(com.facebook.presto.execution.TestQueues.createResourceGroupId) DispatchManager(com.facebook.presto.dispatcher.DispatchManager) QueryId(com.facebook.presto.spi.QueryId) BasicQueryInfo(com.facebook.presto.server.BasicQueryInfo) QueryManager(com.facebook.presto.execution.QueryManager) InternalResourceGroupManager(com.facebook.presto.execution.resourceGroups.InternalResourceGroupManager) Test(org.testng.annotations.Test)

Example 4 with ReloadingResourceGroupConfigurationManager

use of com.facebook.presto.resourceGroups.reloading.ReloadingResourceGroupConfigurationManager in project presto by prestodb.

the class TestQueuesDb method testNonLeafGroup.

@Test
public void testNonLeafGroup() throws Exception {
    Session session = testSessionBuilder().setCatalog("tpch").setSchema("sf100000").setSource("non-leaf").build();
    QueryManager queryManager = queryRunner.getCoordinator().getQueryManager();
    InternalResourceGroupManager manager = queryRunner.getCoordinator().getResourceGroupManager().get();
    ReloadingResourceGroupConfigurationManager reloadingConfigurationManager = (ReloadingResourceGroupConfigurationManager) manager.getConfigurationManager();
    int originalSize = getSelectors(queryRunner).size();
    // Add a selector for a non leaf group
    dao.insertSelector(3, 100, "user.*", "(?i).*non-leaf.*", null, null, null);
    reloadingConfigurationManager.load();
    while (getSelectors(queryRunner).size() != originalSize + 1) {
        MILLISECONDS.sleep(500);
    }
    // Submit query with side effect of creating resource groups
    QueryId firstDashboardQuery = createQuery(queryRunner, dashboardSession(), LONG_LASTING_QUERY);
    waitForQueryState(queryRunner, firstDashboardQuery, RUNNING);
    cancelQuery(queryRunner, firstDashboardQuery);
    waitForQueryState(queryRunner, firstDashboardQuery, FAILED);
    // Submit a query to a non-leaf resource group
    QueryId invalidResourceGroupQuery = createQuery(queryRunner, session, LONG_LASTING_QUERY);
    waitForQueryState(queryRunner, invalidResourceGroupQuery, FAILED);
    assertEquals(queryRunner.getCoordinator().getDispatchManager().getQueryInfo(invalidResourceGroupQuery).getErrorCode(), INVALID_RESOURCE_GROUP.toErrorCode());
}
Also used : ReloadingResourceGroupConfigurationManager(com.facebook.presto.resourceGroups.reloading.ReloadingResourceGroupConfigurationManager) QueryId(com.facebook.presto.spi.QueryId) QueryManager(com.facebook.presto.execution.QueryManager) InternalResourceGroupManager(com.facebook.presto.execution.resourceGroups.InternalResourceGroupManager) H2TestUtil.adhocSession(com.facebook.presto.execution.resourceGroups.db.H2TestUtil.adhocSession) H2TestUtil.dashboardSession(com.facebook.presto.execution.resourceGroups.db.H2TestUtil.dashboardSession) H2TestUtil.rejectingSession(com.facebook.presto.execution.resourceGroups.db.H2TestUtil.rejectingSession) Session(com.facebook.presto.Session) Test(org.testng.annotations.Test)

Example 5 with ReloadingResourceGroupConfigurationManager

use of com.facebook.presto.resourceGroups.reloading.ReloadingResourceGroupConfigurationManager in project presto by prestodb.

the class TestQueuesDb method testQueryExecutionTimeLimit.

@Test(timeOut = 60_000)
public void testQueryExecutionTimeLimit() throws Exception {
    QueryManager queryManager = queryRunner.getCoordinator().getQueryManager();
    InternalResourceGroupManager manager = queryRunner.getCoordinator().getResourceGroupManager().get();
    ReloadingResourceGroupConfigurationManager reloadingConfigurationManager = (ReloadingResourceGroupConfigurationManager) manager.getConfigurationManager();
    QueryId firstQuery = createQuery(queryRunner, testSessionBuilder().setCatalog("tpch").setSchema("sf100000").setSource("dashboard").setSystemProperty(QUERY_MAX_EXECUTION_TIME, "1ms").build(), LONG_LASTING_QUERY);
    waitForQueryState(queryRunner, firstQuery, FAILED);
    assertEquals(queryManager.getFullQueryInfo(firstQuery).getErrorCode(), EXCEEDED_TIME_LIMIT.toErrorCode());
    assertContains(queryManager.getFullQueryInfo(firstQuery).getFailureInfo().getMessage(), "Query exceeded the maximum execution time limit of 1.00ms");
    // set max running queries to 0 for the dashboard resource group so that new queries get queued immediately
    dao.updateResourceGroup(5, "dashboard-${USER}", "1MB", 1, null, 0, null, null, null, null, null, null, null, null, 3L, TEST_ENVIRONMENT);
    reloadingConfigurationManager.load();
    QueryId secondQuery = createQuery(queryRunner, testSessionBuilder().setCatalog("tpch").setSchema("sf100000").setSource("dashboard").setSystemProperty(QUERY_MAX_EXECUTION_TIME, "1ms").build(), LONG_LASTING_QUERY);
    // this query should immediately get queued
    waitForQueryState(queryRunner, secondQuery, QUEUED);
    // after a 5s wait this query should still be QUEUED, not FAILED as the max execution time should be enforced after the query starts running
    Thread.sleep(5_000);
    DispatchManager dispatchManager = queryRunner.getCoordinator().getDispatchManager();
    assertEquals(dispatchManager.getQueryInfo(secondQuery).getState(), QUEUED);
    // reconfigure the resource group to run the second query
    dao.updateResourceGroup(5, "dashboard-${USER}", "1MB", 1, null, 1, null, null, null, null, null, null, null, null, 3L, TEST_ENVIRONMENT);
    reloadingConfigurationManager.load();
    // cancel the first one and let the second one start
    dispatchManager.cancelQuery(firstQuery);
    // wait until the second one is FAILED
    waitForQueryState(queryRunner, secondQuery, FAILED);
}
Also used : ReloadingResourceGroupConfigurationManager(com.facebook.presto.resourceGroups.reloading.ReloadingResourceGroupConfigurationManager) DispatchManager(com.facebook.presto.dispatcher.DispatchManager) QueryId(com.facebook.presto.spi.QueryId) QueryManager(com.facebook.presto.execution.QueryManager) InternalResourceGroupManager(com.facebook.presto.execution.resourceGroups.InternalResourceGroupManager) Test(org.testng.annotations.Test)

Aggregations

InternalResourceGroupManager (com.facebook.presto.execution.resourceGroups.InternalResourceGroupManager)5 ReloadingResourceGroupConfigurationManager (com.facebook.presto.resourceGroups.reloading.ReloadingResourceGroupConfigurationManager)5 QueryId (com.facebook.presto.spi.QueryId)5 Test (org.testng.annotations.Test)4 DispatchManager (com.facebook.presto.dispatcher.DispatchManager)3 QueryManager (com.facebook.presto.execution.QueryManager)3 Session (com.facebook.presto.Session)1 TestQueues.createResourceGroupId (com.facebook.presto.execution.TestQueues.createResourceGroupId)1 H2TestUtil.adhocSession (com.facebook.presto.execution.resourceGroups.db.H2TestUtil.adhocSession)1 H2TestUtil.dashboardSession (com.facebook.presto.execution.resourceGroups.db.H2TestUtil.dashboardSession)1 H2TestUtil.rejectingSession (com.facebook.presto.execution.resourceGroups.db.H2TestUtil.rejectingSession)1 BasicQueryInfo (com.facebook.presto.server.BasicQueryInfo)1 ResourceGroupId (com.facebook.presto.spi.resourceGroups.ResourceGroupId)1