Search in sources :

Example 1 with ResourceGroupId

use of io.prestosql.spi.resourcegroups.ResourceGroupId in project hetu-core by openlookeng.

the class TestQueuesDb method assertResourceGroupWithClientTags.

private void assertResourceGroupWithClientTags(Set<String> clientTags, ResourceGroupId expectedResourceGroup) throws InterruptedException {
    Session session = testSessionBuilder().setCatalog("tpch").setSchema("sf100000").setSource("client_tags").setClientTags(clientTags).build();
    QueryId queryId = createQuery(queryRunner, session, LONG_LASTING_QUERY);
    waitForQueryState(queryRunner, queryId, ImmutableSet.of(RUNNING, FINISHED));
    Optional<ResourceGroupId> resourceGroupId = queryRunner.getCoordinator().getQueryManager().getFullQueryInfo(queryId).getResourceGroupId();
    assertTrue(resourceGroupId.isPresent(), "Query should have a resource group");
    assertEquals(resourceGroupId.get(), expectedResourceGroup, format("Expected: '%s' resource group, found: %s", expectedResourceGroup, resourceGroupId.get()));
}
Also used : ResourceGroupId(io.prestosql.spi.resourcegroups.ResourceGroupId) TestQueues.createResourceGroupId(io.prestosql.execution.TestQueues.createResourceGroupId) QueryId(io.prestosql.spi.QueryId) H2TestUtil.adhocSession(io.prestosql.execution.resourcegroups.db.H2TestUtil.adhocSession) H2TestUtil.dashboardSession(io.prestosql.execution.resourcegroups.db.H2TestUtil.dashboardSession) H2TestUtil.rejectingSession(io.prestosql.execution.resourcegroups.db.H2TestUtil.rejectingSession) Session(io.prestosql.Session)

Example 2 with ResourceGroupId

use of io.prestosql.spi.resourcegroups.ResourceGroupId in project hetu-core by openlookeng.

the class TestDbResourceGroupConfigurationManager method testEnvironments.

@Test
public void testEnvironments() {
    H2DaoProvider daoProvider = setup("test_configuration");
    H2ResourceGroupsDao dao = daoProvider.get();
    dao.createResourceGroupsGlobalPropertiesTable();
    dao.createResourceGroupsTable();
    dao.createSelectorsTable();
    String prodEnvironment = "prod";
    String devEnvironment = "dev";
    dao.insertResourceGroupsGlobalProperties("cpu_quota_period", "1h");
    // two resource groups are the same except the group for the prod environment has a larger softMemoryLimit
    // Hetu: add values for new parameters softReservedMemory and hardReservedConcurrency
    dao.insertResourceGroup(1, "prod_global", "10MB", "5MB", 1000, 100, 100, 50, "weighted", null, true, "1h", "1d", "RECENT_QUERIES", null, prodEnvironment);
    dao.insertResourceGroup(2, "dev_global", "1MB", "5MB", 1000, 100, 100, 50, "weighted", null, true, "1h", "1d", "RECENT_QUERIES", null, devEnvironment);
    dao.insertSelector(1, 1, ".*prod_user.*", null, null, null, null);
    dao.insertSelector(2, 2, ".*dev_user.*", null, null, null, null);
    // check the prod configuration
    DbResourceGroupConfigurationManager manager = new DbResourceGroupConfigurationManager((poolId, listener) -> {
    }, new DbResourceGroupConfig(), daoProvider.get(), prodEnvironment);
    List<ResourceGroupSpec> groups = manager.getRootGroups();
    assertEquals(groups.size(), 1);
    InternalResourceGroup prodGlobal = new InternalResourceGroup(Optional.empty(), "prod_global", (group, export) -> {
    }, directExecutor());
    manager.configure(prodGlobal, new SelectionContext<>(prodGlobal.getId(), new ResourceGroupIdTemplate("prod_global")));
    // Hetu: add values for new parameters softReservedMemory and hardReservedConcurrency
    assertEqualsResourceGroup(prodGlobal, "10MB", 1000, 100, 100, WEIGHTED, DEFAULT_WEIGHT, true, new Duration(1, HOURS), new Duration(1, DAYS));
    assertEquals(manager.getSelectors().size(), 1);
    ResourceGroupSelector prodSelector = manager.getSelectors().get(0);
    ResourceGroupId prodResourceGroupId = prodSelector.match(new SelectionCriteria(true, "prod_user", Optional.empty(), ImmutableSet.of(), EMPTY_RESOURCE_ESTIMATES, Optional.empty())).get().getResourceGroupId();
    assertEquals(prodResourceGroupId.toString(), "prod_global");
    // check the dev configuration
    manager = new DbResourceGroupConfigurationManager((poolId, listener) -> {
    }, new DbResourceGroupConfig(), daoProvider.get(), devEnvironment);
    assertEquals(groups.size(), 1);
    InternalResourceGroup devGlobal = new InternalResourceGroup(Optional.empty(), "dev_global", (group, export) -> {
    }, directExecutor());
    manager.configure(devGlobal, new SelectionContext<>(prodGlobal.getId(), new ResourceGroupIdTemplate("dev_global")));
    assertEqualsResourceGroup(devGlobal, "1MB", 1000, 100, 100, WEIGHTED, DEFAULT_WEIGHT, true, new Duration(1, HOURS), new Duration(1, DAYS));
    assertEquals(manager.getSelectors().size(), 1);
    ResourceGroupSelector devSelector = manager.getSelectors().get(0);
    ResourceGroupId devResourceGroupId = devSelector.match(new SelectionCriteria(true, "dev_user", Optional.empty(), ImmutableSet.of(), EMPTY_RESOURCE_ESTIMATES, Optional.empty())).get().getResourceGroupId();
    assertEquals(devResourceGroupId.toString(), "dev_global");
}
Also used : WEIGHTED(io.prestosql.spi.resourcegroups.SchedulingPolicy.WEIGHTED) ResourceGroupIdTemplate(io.prestosql.plugin.resourcegroups.ResourceGroupIdTemplate) DEFAULT_WEIGHT(io.prestosql.execution.resourcegroups.BaseResourceGroup.DEFAULT_WEIGHT) SelectionContext(io.prestosql.spi.resourcegroups.SelectionContext) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Assert.assertEquals(org.testng.Assert.assertEquals) Test(org.testng.annotations.Test) ResourceGroupSpec(io.prestosql.plugin.resourcegroups.ResourceGroupSpec) Duration(io.airlift.units.Duration) ArrayList(java.util.ArrayList) UnableToExecuteStatementException(org.jdbi.v3.core.statement.UnableToExecuteStatementException) Assertions.assertThatThrownBy(org.assertj.core.api.Assertions.assertThatThrownBy) ResourceGroupSelector(io.prestosql.plugin.resourcegroups.ResourceGroupSelector) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) DAYS(java.util.concurrent.TimeUnit.DAYS) Assert.assertFalse(org.testng.Assert.assertFalse) PrestoException(io.prestosql.spi.PrestoException) ImmutableSet(com.google.common.collect.ImmutableSet) Assert.fail(org.testng.Assert.fail) ResourceEstimates(io.prestosql.spi.session.ResourceEstimates) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) InternalResourceGroup(io.prestosql.execution.resourcegroups.InternalResourceGroup) MoreExecutors.directExecutor(com.google.common.util.concurrent.MoreExecutors.directExecutor) ResourceGroupId(io.prestosql.spi.resourcegroups.ResourceGroupId) DataSize(io.airlift.units.DataSize) List(java.util.List) StaticSelector(io.prestosql.plugin.resourcegroups.StaticSelector) FAIR(io.prestosql.spi.resourcegroups.SchedulingPolicy.FAIR) SchedulingPolicy(io.prestosql.spi.resourcegroups.SchedulingPolicy) Optional(java.util.Optional) HOURS(java.util.concurrent.TimeUnit.HOURS) Assert.assertTrue(org.testng.Assert.assertTrue) SelectionCriteria(io.prestosql.spi.resourcegroups.SelectionCriteria) Pattern(java.util.regex.Pattern) Comparator(java.util.Comparator) ResourceGroupSelector(io.prestosql.plugin.resourcegroups.ResourceGroupSelector) ResourceGroupId(io.prestosql.spi.resourcegroups.ResourceGroupId) ResourceGroupSpec(io.prestosql.plugin.resourcegroups.ResourceGroupSpec) Duration(io.airlift.units.Duration) ResourceGroupIdTemplate(io.prestosql.plugin.resourcegroups.ResourceGroupIdTemplate) InternalResourceGroup(io.prestosql.execution.resourcegroups.InternalResourceGroup) SelectionCriteria(io.prestosql.spi.resourcegroups.SelectionCriteria) Test(org.testng.annotations.Test)

Example 3 with ResourceGroupId

use of io.prestosql.spi.resourcegroups.ResourceGroupId in project hetu-core by openlookeng.

the class TestDbSourceExactMatchSelector method testMatch.

@Test
public void testMatch() {
    ResourceGroupId resourceGroupId1 = new ResourceGroupId(ImmutableList.of("global", "test", "user", "insert"));
    ResourceGroupId resourceGroupId2 = new ResourceGroupId(ImmutableList.of("global", "test", "user", "select"));
    dao.insertExactMatchSelector("test", "@test@test_pipeline", INSERT.name(), CODEC.toJson(resourceGroupId1));
    dao.insertExactMatchSelector("test", "@test@test_pipeline", SELECT.name(), CODEC.toJson(resourceGroupId2));
    DbSourceExactMatchSelector selector = new DbSourceExactMatchSelector("test", dao);
    assertEquals(selector.match(new SelectionCriteria(true, "testuser", Optional.of("@test@test_pipeline"), ImmutableSet.of("tag"), EMPTY_RESOURCE_ESTIMATES, Optional.empty())), Optional.empty());
    assertEquals(selector.match(new SelectionCriteria(true, "testuser", Optional.of("@test@test_pipeline"), ImmutableSet.of("tag"), EMPTY_RESOURCE_ESTIMATES, Optional.of(INSERT.name()))).map(SelectionContext::getResourceGroupId), Optional.of(resourceGroupId1));
    assertEquals(selector.match(new SelectionCriteria(true, "testuser", Optional.of("@test@test_pipeline"), ImmutableSet.of("tag"), EMPTY_RESOURCE_ESTIMATES, Optional.of(SELECT.name()))).map(SelectionContext::getResourceGroupId), Optional.of(resourceGroupId2));
    assertEquals(selector.match(new SelectionCriteria(true, "testuser", Optional.of("@test@test_pipeline"), ImmutableSet.of("tag"), EMPTY_RESOURCE_ESTIMATES, Optional.of(DELETE.name()))), Optional.empty());
    assertEquals(selector.match(new SelectionCriteria(true, "testuser", Optional.of("@test@test_new"), ImmutableSet.of(), EMPTY_RESOURCE_ESTIMATES, Optional.of(INSERT.name()))), Optional.empty());
}
Also used : ResourceGroupId(io.prestosql.spi.resourcegroups.ResourceGroupId) SelectionCriteria(io.prestosql.spi.resourcegroups.SelectionCriteria) Test(org.testng.annotations.Test)

Example 4 with ResourceGroupId

use of io.prestosql.spi.resourcegroups.ResourceGroupId in project hetu-core by openlookeng.

the class TestFileResourceGroupConfigurationManager method assertMatch.

private static void assertMatch(FileResourceGroupConfigurationManager manager, SelectionCriteria criteria, String expectedResourceGroup) {
    ResourceGroupId resourceGroupId = match(manager, criteria).getResourceGroupId();
    assertEquals(resourceGroupId.toString(), expectedResourceGroup, format("Expected: '%s' resource group, found: %s", expectedResourceGroup, resourceGroupId));
}
Also used : ResourceGroupId(io.prestosql.spi.resourcegroups.ResourceGroupId)

Example 5 with ResourceGroupId

use of io.prestosql.spi.resourcegroups.ResourceGroupId in project hetu-core by openlookeng.

the class TestResourceGroupIdTemplate method testExpansion.

@Test
public void testExpansion() {
    ResourceGroupIdTemplate template = new ResourceGroupIdTemplate("test.${USER}.${SOURCE}");
    ResourceGroupId expected = new ResourceGroupId(new ResourceGroupId(new ResourceGroupId("test"), "u"), "s");
    assertEquals(template.expandTemplate(new VariableMap(ImmutableMap.of("USER", "u", "SOURCE", "s"))), expected);
    template = new ResourceGroupIdTemplate("test.${USER}");
    assertEquals(template.expandTemplate(new VariableMap(ImmutableMap.of("USER", "alice.smith", "SOURCE", "s"))), new ResourceGroupId(new ResourceGroupId("test"), "alice.smith"));
}
Also used : ResourceGroupId(io.prestosql.spi.resourcegroups.ResourceGroupId) Test(org.testng.annotations.Test)

Aggregations

ResourceGroupId (io.prestosql.spi.resourcegroups.ResourceGroupId)37 Test (org.testng.annotations.Test)20 NoOpResourceGroupManager (io.prestosql.execution.resourcegroups.NoOpResourceGroupManager)6 FileSystemClientManager (io.prestosql.filesystem.FileSystemClientManager)5 HeuristicIndexerManager (io.prestosql.heuristicindex.HeuristicIndexerManager)5 HetuMetaStoreManager (io.prestosql.metastore.HetuMetaStoreManager)5 QueryId (io.prestosql.spi.QueryId)5 TestQueues.createResourceGroupId (io.prestosql.execution.TestQueues.createResourceGroupId)4 SelectionCriteria (io.prestosql.spi.resourcegroups.SelectionCriteria)4 DataSize (io.airlift.units.DataSize)3 Duration (io.airlift.units.Duration)3 Session (io.prestosql.Session)3 MockManagedQueryExecution (io.prestosql.execution.MockManagedQueryExecution)3 AccessControlManager (io.prestosql.security.AccessControlManager)3 ResourceGroupInfo (io.prestosql.server.ResourceGroupInfo)3 SharedQueryState (io.prestosql.statestore.SharedQueryState)3 SharedResourceGroupState (io.prestosql.statestore.SharedResourceGroupState)3 HashMap (java.util.HashMap)3 ImmutableMap (com.google.common.collect.ImmutableMap)2 H2TestUtil.adhocSession (io.prestosql.execution.resourcegroups.db.H2TestUtil.adhocSession)2