Examples with ResourceGroupIdTemplate io.trino.plugin.resourcegroups.ResourceGroupIdTemplate used on opensource projects

Example 1 with ResourceGroupIdTemplate

use of io.trino.plugin.resourcegroups.ResourceGroupIdTemplate in project trino by trinodb.

the class DbResourceGroupConfigurationManager method load.

@VisibleForTesting
public synchronized void load() {
    try {
        Map.Entry<ManagerSpec, Map<ResourceGroupIdTemplate, ResourceGroupSpec>> specsFromDb = buildSpecsFromDb();
        ManagerSpec managerSpec = specsFromDb.getKey();
        Map<ResourceGroupIdTemplate, ResourceGroupSpec> resourceGroupSpecs = specsFromDb.getValue();
        Set<ResourceGroupIdTemplate> changedSpecs = new HashSet<>();
        Set<ResourceGroupIdTemplate> deletedSpecs = Sets.difference(this.resourceGroupSpecs.keySet(), resourceGroupSpecs.keySet());
        for (Map.Entry<ResourceGroupIdTemplate, ResourceGroupSpec> entry : resourceGroupSpecs.entrySet()) {
            if (!entry.getValue().sameConfig(this.resourceGroupSpecs.get(entry.getKey()))) {
                changedSpecs.add(entry.getKey());
            }
        }
        this.resourceGroupSpecs = resourceGroupSpecs;
        this.cpuQuotaPeriod.set(managerSpec.getCpuQuotaPeriod());
        this.rootGroups.set(managerSpec.getRootGroups());
        List<ResourceGroupSelector> selectors = buildSelectors(managerSpec);
        if (exactMatchSelectorEnabled) {
            ImmutableList.Builder<ResourceGroupSelector> builder = ImmutableList.builder();
            builder.add(new DbSourceExactMatchSelector(environment, dao));
            builder.addAll(selectors);
            this.selectors.set(builder.build());
        } else {
            this.selectors.set(selectors);
        }
        configureChangedGroups(changedSpecs);
        disableDeletedGroups(deletedSpecs);
        if (lastRefresh.get() > 0) {
            for (ResourceGroupIdTemplate deleted : deletedSpecs) {
                log.info("Resource group spec deleted %s", deleted);
            }
            for (ResourceGroupIdTemplate changed : changedSpecs) {
                log.info("Resource group spec %s changed to %s", changed, resourceGroupSpecs.get(changed));
            }
        } else {
            log.info("Loaded %s selectors and %s resource groups from database", this.selectors.get().size(), this.resourceGroupSpecs.size());
        }
        lastRefresh.set(System.nanoTime());
    } catch (Throwable e) {
        if (succinctNanos(System.nanoTime() - lastRefresh.get()).compareTo(maxRefreshInterval) > 0) {
            lastRefresh.set(0);
        }
        refreshFailures.update(1);
        log.error(e, "Error loading configuration from db");
    }
}

Example 2 with ResourceGroupIdTemplate

use of io.trino.plugin.resourcegroups.ResourceGroupIdTemplate in project trino by trinodb.

the class DbResourceGroupConfigurationManager method populateFromDbHelper.

// Populate temporary data structures to build resource group specs and selectors from db
private synchronized void populateFromDbHelper(Map<Long, ResourceGroupSpecBuilder> recordMap, Set<Long> rootGroupIds, Map<Long, ResourceGroupIdTemplate> resourceGroupIdTemplateMap, Map<Long, Set<Long>> subGroupIdsToBuild) {
    List<ResourceGroupSpecBuilder> records = dao.getResourceGroups(environment);
    for (ResourceGroupSpecBuilder record : records) {
        recordMap.put(record.getId(), record);
        if (record.getParentId().isEmpty()) {
            rootGroupIds.add(record.getId());
            resourceGroupIdTemplateMap.put(record.getId(), new ResourceGroupIdTemplate(record.getNameTemplate().toString()));
        } else {
            subGroupIdsToBuild.computeIfAbsent(record.getParentId().get(), k -> new HashSet<>()).add(record.getId());
        }
    }
}

Example 3 with ResourceGroupIdTemplate

use of io.trino.plugin.resourcegroups.ResourceGroupIdTemplate in project trino by trinodb.

the class DbSourceExactMatchSelector method toTemplate.

private static ResourceGroupIdTemplate toTemplate(ResourceGroupId groupId) {
    ResourceGroupIdTemplate template = new ResourceGroupIdTemplate(groupId.toString());
    checkArgument(template.getVariableNames().isEmpty(), "Group ID contains variable references: %s", groupId);
    return template;
}

Example 4 with ResourceGroupIdTemplate

use of io.trino.plugin.resourcegroups.ResourceGroupIdTemplate in project trino by trinodb.

the class TestDbResourceGroupConfigurationManager method testEnvironments.

@Test
public void testEnvironments() {
    H2DaoProvider daoProvider = setup("test_configuration");
    H2ResourceGroupsDao dao = daoProvider.get();
    dao.createResourceGroupsGlobalPropertiesTable();
    dao.createResourceGroupsTable();
    dao.createSelectorsTable();
    String prodEnvironment = "prod";
    String devEnvironment = "dev";
    dao.insertResourceGroupsGlobalProperties("cpu_quota_period", "1h");
    // two resource groups are the same except the group for the prod environment has a larger softMemoryLimit
    dao.insertResourceGroup(1, "prod_global", "10MB", 1000, 100, 100, "weighted", null, true, "1h", "1d", null, prodEnvironment);
    dao.insertResourceGroup(2, "dev_global", "1MB", 1000, 100, 100, "weighted", null, true, "1h", "1d", null, devEnvironment);
    dao.insertSelector(1, 1, ".*prod_user.*", null, null, null, null, null);
    dao.insertSelector(2, 2, ".*dev_user.*", null, null, null, null, null);
    // check the prod configuration
    DbResourceGroupConfigurationManager manager = new DbResourceGroupConfigurationManager(listener -> {
    }, new DbResourceGroupConfig(), daoProvider.get(), prodEnvironment);
    List<ResourceGroupSpec> groups = manager.getRootGroups();
    assertEquals(groups.size(), 1);
    InternalResourceGroup prodGlobal = new InternalResourceGroup("prod_global", (group, export) -> {
    }, directExecutor());
    manager.configure(prodGlobal, new SelectionContext<>(prodGlobal.getId(), new ResourceGroupIdTemplate("prod_global")));
    assertEqualsResourceGroup(prodGlobal, "10MB", 1000, 100, 100, WEIGHTED, DEFAULT_WEIGHT, true, Duration.ofHours(1), Duration.ofDays(1));
    assertEquals(manager.getSelectors().size(), 1);
    ResourceGroupSelector prodSelector = manager.getSelectors().get(0);
    ResourceGroupId prodResourceGroupId = prodSelector.match(new SelectionCriteria(true, "prod_user", ImmutableSet.of(), Optional.empty(), ImmutableSet.of(), EMPTY_RESOURCE_ESTIMATES, Optional.empty())).get().getResourceGroupId();
    assertEquals(prodResourceGroupId.toString(), "prod_global");
    // check the dev configuration
    manager = new DbResourceGroupConfigurationManager(listener -> {
    }, new DbResourceGroupConfig(), daoProvider.get(), devEnvironment);
    assertEquals(groups.size(), 1);
    InternalResourceGroup devGlobal = new InternalResourceGroup("dev_global", (group, export) -> {
    }, directExecutor());
    manager.configure(devGlobal, new SelectionContext<>(prodGlobal.getId(), new ResourceGroupIdTemplate("dev_global")));
    assertEqualsResourceGroup(devGlobal, "1MB", 1000, 100, 100, WEIGHTED, DEFAULT_WEIGHT, true, Duration.ofHours(1), Duration.ofDays(1));
    assertEquals(manager.getSelectors().size(), 1);
    ResourceGroupSelector devSelector = manager.getSelectors().get(0);
    ResourceGroupId devResourceGroupId = devSelector.match(new SelectionCriteria(true, "dev_user", ImmutableSet.of(), Optional.empty(), ImmutableSet.of(), EMPTY_RESOURCE_ESTIMATES, Optional.empty())).get().getResourceGroupId();
    assertEquals(devResourceGroupId.toString(), "dev_global");
}

Example 5 with ResourceGroupIdTemplate

use of io.trino.plugin.resourcegroups.ResourceGroupIdTemplate in project trino by trinodb.

the class TestDbResourceGroupConfigurationManager method testReconfig.

@Test(timeOut = 60_000)
public void testReconfig() throws Exception {
    H2DaoProvider daoProvider = setup("test_reconfig");
    H2ResourceGroupsDao dao = daoProvider.get();
    dao.createResourceGroupsGlobalPropertiesTable();
    dao.createResourceGroupsTable();
    dao.createSelectorsTable();
    dao.insertResourceGroup(1, "global", "1MB", 1000, 100, 100, "weighted", null, true, "1h", "1d", null, ENVIRONMENT);
    dao.insertResourceGroup(2, "sub", "2MB", 4, 3, 3, null, 5, null, null, null, 1L, ENVIRONMENT);
    dao.insertSelector(2, 1, null, null, null, null, null, null);
    dao.insertResourceGroupsGlobalProperties("cpu_quota_period", "1h");
    DbResourceGroupConfigurationManager manager = new DbResourceGroupConfigurationManager(listener -> {
    }, new DbResourceGroupConfig(), daoProvider.get(), ENVIRONMENT);
    manager.start();
    AtomicBoolean exported = new AtomicBoolean();
    InternalResourceGroup global = new InternalResourceGroup("global", (group, export) -> exported.set(export), directExecutor());
    manager.configure(global, new SelectionContext<>(global.getId(), new ResourceGroupIdTemplate("global")));
    InternalResourceGroup globalSub = global.getOrCreateSubGroup("sub");
    manager.configure(globalSub, new SelectionContext<>(globalSub.getId(), new ResourceGroupIdTemplate("global.sub")));
    // Verify record exists
    assertEqualsResourceGroup(globalSub, "2MB", 4, 3, 3, FAIR, 5, false, Duration.ofMillis(Long.MAX_VALUE), Duration.ofMillis(Long.MAX_VALUE));
    dao.updateResourceGroup(2, "sub", "3MB", 2, 1, 1, "weighted", 6, true, "1h", "1d", 1L, ENVIRONMENT);
    do {
        MILLISECONDS.sleep(500);
    } while (globalSub.getJmxExport() == false);
    // Verify update
    assertEqualsResourceGroup(globalSub, "3MB", 2, 1, 1, WEIGHTED, 6, true, Duration.ofHours(1), Duration.ofDays(1));
    // Verify delete
    dao.deleteSelectors(2);
    dao.deleteResourceGroup(2);
    do {
        MILLISECONDS.sleep(500);
    } while (globalSub.getMaxQueuedQueries() != 0 || globalSub.getHardConcurrencyLimit() != 0);
}