Search in sources :

Example 96 with QueryId

use of io.prestosql.spi.QueryId in project hetu-core by openlookeng.

the class TestQueues method testExceedSoftLimits.

@Test(timeOut = 240_000)
public void testExceedSoftLimits() throws Exception {
    try (DistributedQueryRunner queryRunner = createQueryRunner()) {
        queryRunner.installPlugin(new ResourceGroupManagerPlugin());
        queryRunner.getCoordinator().getResourceGroupManager().get().setConfigurationManager("file", ImmutableMap.of("resource-groups.config-file", getResourceFilePath("resource_groups_config_soft_limits.json")));
        QueryId scheduled1 = createScheduledQuery(queryRunner);
        waitForQueryState(queryRunner, scheduled1, RUNNING);
        QueryId scheduled2 = createScheduledQuery(queryRunner);
        waitForQueryState(queryRunner, scheduled2, RUNNING);
        QueryId scheduled3 = createScheduledQuery(queryRunner);
        waitForQueryState(queryRunner, scheduled3, RUNNING);
        // cluster is now 'at capacity' - scheduled is running 3 (i.e. over soft limit)
        QueryId backfill1 = createBackfill(queryRunner);
        QueryId scheduled4 = createScheduledQuery(queryRunner);
        cancelQuery(queryRunner, scheduled1);
        // backfill should be chosen to run next
        waitForQueryState(queryRunner, backfill1, RUNNING);
        cancelQuery(queryRunner, scheduled2);
        cancelQuery(queryRunner, scheduled3);
        cancelQuery(queryRunner, scheduled4);
        QueryId backfill2 = createBackfill(queryRunner);
        waitForQueryState(queryRunner, backfill2, RUNNING);
        QueryId backfill3 = createBackfill(queryRunner);
        waitForQueryState(queryRunner, backfill3, RUNNING);
        // cluster is now 'at capacity' - backfills is running 3 (i.e. over soft limit)
        QueryId backfill4 = createBackfill(queryRunner);
        QueryId scheduled5 = createScheduledQuery(queryRunner);
        cancelQuery(queryRunner, backfill1);
        // scheduled should be chosen to run next
        waitForQueryState(queryRunner, scheduled5, RUNNING);
        cancelQuery(queryRunner, backfill2);
        cancelQuery(queryRunner, backfill3);
        cancelQuery(queryRunner, backfill4);
        cancelQuery(queryRunner, scheduled5);
        waitForQueryState(queryRunner, scheduled5, FAILED);
    }
}
Also used : DistributedQueryRunner(io.prestosql.tests.DistributedQueryRunner) QueryId(io.prestosql.spi.QueryId) ResourceGroupManagerPlugin(io.prestosql.plugin.resourcegroups.ResourceGroupManagerPlugin) Test(org.testng.annotations.Test)

Example 97 with QueryId

use of io.prestosql.spi.QueryId in project hetu-core by openlookeng.

the class TestQueues method testRejection.

private void testRejection() throws Exception {
    try (DistributedQueryRunner queryRunner = createQueryRunner()) {
        queryRunner.installPlugin(new ResourceGroupManagerPlugin());
        queryRunner.getCoordinator().getResourceGroupManager().get().setConfigurationManager("file", ImmutableMap.of("resource-groups.config-file", getResourceFilePath("resource_groups_config_dashboard.json")));
        QueryId queryId = createQuery(queryRunner, newRejectionSession(), LONG_LASTING_QUERY);
        waitForQueryState(queryRunner, queryId, FAILED);
        DispatchManager dispatchManager = queryRunner.getCoordinator().getDispatchManager();
        assertEquals(dispatchManager.getQueryInfo(queryId).getErrorCode(), QUERY_REJECTED.toErrorCode());
    }
}
Also used : DistributedQueryRunner(io.prestosql.tests.DistributedQueryRunner) DispatchManager(io.prestosql.dispatcher.DispatchManager) QueryId(io.prestosql.spi.QueryId) ResourceGroupManagerPlugin(io.prestosql.plugin.resourcegroups.ResourceGroupManagerPlugin)

Example 98 with QueryId

use of io.prestosql.spi.QueryId in project hetu-core by openlookeng.

the class TestQueues method testResourceGroupManagerWithTooManyQueriesScheduled.

@Test(timeOut = 240_000)
public void testResourceGroupManagerWithTooManyQueriesScheduled() throws Exception {
    try (DistributedQueryRunner queryRunner = createQueryRunner()) {
        queryRunner.installPlugin(new ResourceGroupManagerPlugin());
        queryRunner.getCoordinator().getResourceGroupManager().get().setConfigurationManager("file", ImmutableMap.of("resource-groups.config-file", getResourceFilePath("resource_groups_config_dashboard.json")));
        QueryId firstDashboardQuery = createDashboardQuery(queryRunner);
        waitForQueryState(queryRunner, firstDashboardQuery, RUNNING);
        QueryId secondDashboardQuery = createDashboardQuery(queryRunner);
        waitForQueryState(queryRunner, secondDashboardQuery, QUEUED);
        QueryId thirdDashboardQuery = createDashboardQuery(queryRunner);
        waitForQueryState(queryRunner, thirdDashboardQuery, FAILED);
    }
}
Also used : DistributedQueryRunner(io.prestosql.tests.DistributedQueryRunner) QueryId(io.prestosql.spi.QueryId) ResourceGroupManagerPlugin(io.prestosql.plugin.resourcegroups.ResourceGroupManagerPlugin) Test(org.testng.annotations.Test)

Example 99 with QueryId

use of io.prestosql.spi.QueryId in project hetu-core by openlookeng.

the class TestQueues method assertResourceGroup.

private void assertResourceGroup(DistributedQueryRunner queryRunner, Session session, String query, ResourceGroupId expectedResourceGroup) throws InterruptedException {
    QueryId queryId = createQuery(queryRunner, session, query);
    waitForQueryState(queryRunner, queryId, ImmutableSet.of(RUNNING, FINISHED));
    Optional<ResourceGroupId> resourceGroupId = queryRunner.getCoordinator().getQueryManager().getFullQueryInfo(queryId).getResourceGroupId();
    assertTrue(resourceGroupId.isPresent(), "Query should have a resource group");
    assertEquals(resourceGroupId.get(), expectedResourceGroup, format("Expected: '%s' resource group, found: %s", expectedResourceGroup, resourceGroupId.get()));
}
Also used : ResourceGroupId(io.prestosql.spi.resourcegroups.ResourceGroupId) QueryId(io.prestosql.spi.QueryId)

Example 100 with QueryId

use of io.prestosql.spi.QueryId in project hetu-core by openlookeng.

the class TestQueuesDb method testNonLeafGroup.

@Test
public void testNonLeafGroup() throws Exception {
    Session session = testSessionBuilder().setCatalog("tpch").setSchema("sf100000").setSource("non-leaf").build();
    QueryManager queryManager = queryRunner.getCoordinator().getQueryManager();
    InternalResourceGroupManager<?> manager = queryRunner.getCoordinator().getResourceGroupManager().get();
    DbResourceGroupConfigurationManager dbConfigurationManager = (DbResourceGroupConfigurationManager) manager.getConfigurationManager();
    int originalSize = getSelectors(queryRunner).size();
    // Add a selector for a non leaf group
    dao.insertSelector(3, 100, "user.*", "(?i).*non-leaf.*", null, null, null);
    dbConfigurationManager.load();
    while (getSelectors(queryRunner).size() != originalSize + 1) {
        MILLISECONDS.sleep(500);
    }
    // Submit query with side effect of creating resource groups
    QueryId firstDashboardQuery = createQuery(queryRunner, dashboardSession(), LONG_LASTING_QUERY);
    waitForQueryState(queryRunner, firstDashboardQuery, RUNNING);
    cancelQuery(queryRunner, firstDashboardQuery);
    waitForQueryState(queryRunner, firstDashboardQuery, FAILED);
    // Submit a query to a non-leaf resource group
    QueryId invalidResourceGroupQuery = createQuery(queryRunner, session, LONG_LASTING_QUERY);
    waitForQueryState(queryRunner, invalidResourceGroupQuery, FAILED);
    assertEquals(queryRunner.getCoordinator().getDispatchManager().getQueryInfo(invalidResourceGroupQuery).getErrorCode(), INVALID_RESOURCE_GROUP.toErrorCode());
}
Also used : DbResourceGroupConfigurationManager(io.prestosql.plugin.resourcegroups.db.DbResourceGroupConfigurationManager) QueryId(io.prestosql.spi.QueryId) QueryManager(io.prestosql.execution.QueryManager) H2TestUtil.adhocSession(io.prestosql.execution.resourcegroups.db.H2TestUtil.adhocSession) H2TestUtil.dashboardSession(io.prestosql.execution.resourcegroups.db.H2TestUtil.dashboardSession) H2TestUtil.rejectingSession(io.prestosql.execution.resourcegroups.db.H2TestUtil.rejectingSession) Session(io.prestosql.Session) Test(org.testng.annotations.Test)

Aggregations

QueryId (io.prestosql.spi.QueryId)100 Test (org.testng.annotations.Test)69 TaskId (io.prestosql.execution.TaskId)25 DataSize (io.airlift.units.DataSize)18 Session (io.prestosql.Session)18 PrestoException (io.prestosql.spi.PrestoException)13 MemoryPoolId (io.prestosql.spi.memory.MemoryPoolId)13 DistributedQueryRunner (io.prestosql.tests.DistributedQueryRunner)13 NoOpFailureDetector (io.prestosql.failuredetector.NoOpFailureDetector)11 List (java.util.List)11 Map (java.util.Map)11 ImmutableMap (com.google.common.collect.ImmutableMap)10 InternalNode (io.prestosql.metadata.InternalNode)10 PlanNodeId (io.prestosql.spi.plan.PlanNodeId)10 ImmutableList (com.google.common.collect.ImmutableList)9 TestingGcMonitor (io.airlift.stats.TestingGcMonitor)9 ArrayList (java.util.ArrayList)9 Duration (io.airlift.units.Duration)8 Optional (java.util.Optional)8 ColumnHandle (io.prestosql.spi.connector.ColumnHandle)7