Examples with ResourceGroupSpec io.trino.plugin.resourcegroups.ResourceGroupSpec used on opensource projects

Example 1 with ResourceGroupSpec

use of io.trino.plugin.resourcegroups.ResourceGroupSpec in project trino by trinodb.

the class DbResourceGroupConfigurationManager method load.

@VisibleForTesting
public synchronized void load() {
    try {
        Map.Entry<ManagerSpec, Map<ResourceGroupIdTemplate, ResourceGroupSpec>> specsFromDb = buildSpecsFromDb();
        ManagerSpec managerSpec = specsFromDb.getKey();
        Map<ResourceGroupIdTemplate, ResourceGroupSpec> resourceGroupSpecs = specsFromDb.getValue();
        Set<ResourceGroupIdTemplate> changedSpecs = new HashSet<>();
        Set<ResourceGroupIdTemplate> deletedSpecs = Sets.difference(this.resourceGroupSpecs.keySet(), resourceGroupSpecs.keySet());
        for (Map.Entry<ResourceGroupIdTemplate, ResourceGroupSpec> entry : resourceGroupSpecs.entrySet()) {
            if (!entry.getValue().sameConfig(this.resourceGroupSpecs.get(entry.getKey()))) {
                changedSpecs.add(entry.getKey());
            }
        }
        this.resourceGroupSpecs = resourceGroupSpecs;
        this.cpuQuotaPeriod.set(managerSpec.getCpuQuotaPeriod());
        this.rootGroups.set(managerSpec.getRootGroups());
        List<ResourceGroupSelector> selectors = buildSelectors(managerSpec);
        if (exactMatchSelectorEnabled) {
            ImmutableList.Builder<ResourceGroupSelector> builder = ImmutableList.builder();
            builder.add(new DbSourceExactMatchSelector(environment, dao));
            builder.addAll(selectors);
            this.selectors.set(builder.build());
        } else {
            this.selectors.set(selectors);
        }
        configureChangedGroups(changedSpecs);
        disableDeletedGroups(deletedSpecs);
        if (lastRefresh.get() > 0) {
            for (ResourceGroupIdTemplate deleted : deletedSpecs) {
                log.info("Resource group spec deleted %s", deleted);
            }
            for (ResourceGroupIdTemplate changed : changedSpecs) {
                log.info("Resource group spec %s changed to %s", changed, resourceGroupSpecs.get(changed));
            }
        } else {
            log.info("Loaded %s selectors and %s resource groups from database", this.selectors.get().size(), this.resourceGroupSpecs.size());
        }
        lastRefresh.set(System.nanoTime());
    } catch (Throwable e) {
        if (succinctNanos(System.nanoTime() - lastRefresh.get()).compareTo(maxRefreshInterval) > 0) {
            lastRefresh.set(0);
        }
        refreshFailures.update(1);
        log.error(e, "Error loading configuration from db");
    }
}

Example 2 with ResourceGroupSpec

use of io.trino.plugin.resourcegroups.ResourceGroupSpec in project trino by trinodb.

the class TestDbResourceGroupConfigurationManager method testEnvironments.

@Test
public void testEnvironments() {
    H2DaoProvider daoProvider = setup("test_configuration");
    H2ResourceGroupsDao dao = daoProvider.get();
    dao.createResourceGroupsGlobalPropertiesTable();
    dao.createResourceGroupsTable();
    dao.createSelectorsTable();
    String prodEnvironment = "prod";
    String devEnvironment = "dev";
    dao.insertResourceGroupsGlobalProperties("cpu_quota_period", "1h");
    // two resource groups are the same except the group for the prod environment has a larger softMemoryLimit
    dao.insertResourceGroup(1, "prod_global", "10MB", 1000, 100, 100, "weighted", null, true, "1h", "1d", null, prodEnvironment);
    dao.insertResourceGroup(2, "dev_global", "1MB", 1000, 100, 100, "weighted", null, true, "1h", "1d", null, devEnvironment);
    dao.insertSelector(1, 1, ".*prod_user.*", null, null, null, null, null);
    dao.insertSelector(2, 2, ".*dev_user.*", null, null, null, null, null);
    // check the prod configuration
    DbResourceGroupConfigurationManager manager = new DbResourceGroupConfigurationManager(listener -> {
    }, new DbResourceGroupConfig(), daoProvider.get(), prodEnvironment);
    List<ResourceGroupSpec> groups = manager.getRootGroups();
    assertEquals(groups.size(), 1);
    InternalResourceGroup prodGlobal = new InternalResourceGroup("prod_global", (group, export) -> {
    }, directExecutor());
    manager.configure(prodGlobal, new SelectionContext<>(prodGlobal.getId(), new ResourceGroupIdTemplate("prod_global")));
    assertEqualsResourceGroup(prodGlobal, "10MB", 1000, 100, 100, WEIGHTED, DEFAULT_WEIGHT, true, Duration.ofHours(1), Duration.ofDays(1));
    assertEquals(manager.getSelectors().size(), 1);
    ResourceGroupSelector prodSelector = manager.getSelectors().get(0);
    ResourceGroupId prodResourceGroupId = prodSelector.match(new SelectionCriteria(true, "prod_user", ImmutableSet.of(), Optional.empty(), ImmutableSet.of(), EMPTY_RESOURCE_ESTIMATES, Optional.empty())).get().getResourceGroupId();
    assertEquals(prodResourceGroupId.toString(), "prod_global");
    // check the dev configuration
    manager = new DbResourceGroupConfigurationManager(listener -> {
    }, new DbResourceGroupConfig(), daoProvider.get(), devEnvironment);
    assertEquals(groups.size(), 1);
    InternalResourceGroup devGlobal = new InternalResourceGroup("dev_global", (group, export) -> {
    }, directExecutor());
    manager.configure(devGlobal, new SelectionContext<>(prodGlobal.getId(), new ResourceGroupIdTemplate("dev_global")));
    assertEqualsResourceGroup(devGlobal, "1MB", 1000, 100, 100, WEIGHTED, DEFAULT_WEIGHT, true, Duration.ofHours(1), Duration.ofDays(1));
    assertEquals(manager.getSelectors().size(), 1);
    ResourceGroupSelector devSelector = manager.getSelectors().get(0);
    ResourceGroupId devResourceGroupId = devSelector.match(new SelectionCriteria(true, "dev_user", ImmutableSet.of(), Optional.empty(), ImmutableSet.of(), EMPTY_RESOURCE_ESTIMATES, Optional.empty())).get().getResourceGroupId();
    assertEquals(devResourceGroupId.toString(), "dev_global");
}

Example 3 with ResourceGroupSpec

use of io.trino.plugin.resourcegroups.ResourceGroupSpec in project trino by trinodb.

the class DbResourceGroupConfigurationManager method buildSpecsFromDb.

private synchronized Map.Entry<ManagerSpec, Map<ResourceGroupIdTemplate, ResourceGroupSpec>> buildSpecsFromDb() {
    // New resource group spec map
    Map<ResourceGroupIdTemplate, ResourceGroupSpec> resourceGroupSpecs = new HashMap<>();
    // Set of root group db ids
    Set<Long> rootGroupIds = new HashSet<>();
    // Map of id from db to resource group spec
    Map<Long, ResourceGroupSpec> resourceGroupSpecMap = new HashMap<>();
    // Map of id from db to resource group template id
    Map<Long, ResourceGroupIdTemplate> resourceGroupIdTemplateMap = new HashMap<>();
    // Map of id from db to resource group spec builder
    Map<Long, ResourceGroupSpecBuilder> recordMap = new HashMap<>();
    // Map of subgroup id's not yet built
    Map<Long, Set<Long>> subGroupIdsToBuild = new HashMap<>();
    populateFromDbHelper(recordMap, rootGroupIds, resourceGroupIdTemplateMap, subGroupIdsToBuild);
    // Build up resource group specs from leaf to root
    for (LinkedList<Long> queue = new LinkedList<>(rootGroupIds); !queue.isEmpty(); ) {
        Long id = queue.pollFirst();
        resourceGroupIdTemplateMap.computeIfAbsent(id, k -> {
            ResourceGroupSpecBuilder builder = recordMap.get(id);
            return ResourceGroupIdTemplate.forSubGroupNamed(resourceGroupIdTemplateMap.get(builder.getParentId().get()), builder.getNameTemplate().toString());
        });
        Set<Long> childrenToBuild = subGroupIdsToBuild.getOrDefault(id, ImmutableSet.of());
        // Add to resource group specs if no more child resource groups are left to build
        if (childrenToBuild.isEmpty()) {
            ResourceGroupSpecBuilder builder = recordMap.get(id);
            ResourceGroupSpec resourceGroupSpec = builder.build();
            resourceGroupSpecMap.put(id, resourceGroupSpec);
            // Add newly built spec to spec map
            resourceGroupSpecs.put(resourceGroupIdTemplateMap.get(id), resourceGroupSpec);
            // Add this resource group spec to parent subgroups and remove id from subgroup ids to build
            builder.getParentId().ifPresent(parentId -> {
                recordMap.get(parentId).addSubGroup(resourceGroupSpec);
                subGroupIdsToBuild.get(parentId).remove(id);
            });
        } else {
            // Add this group back to queue since it still has subgroups to build
            queue.addFirst(id);
            // Add this group's subgroups to the queue so that when this id is dequeued again childrenToBuild will be empty
            queue.addAll(0, childrenToBuild);
        }
    }
    // Specs are built from db records, validate and return manager spec
    List<ResourceGroupSpec> rootGroups = rootGroupIds.stream().map(resourceGroupSpecMap::get).collect(Collectors.toList());
    List<SelectorSpec> selectors = dao.getSelectors(environment).stream().map(selectorRecord -> new SelectorSpec(selectorRecord.getUserRegex(), selectorRecord.getUserGroupRegex(), selectorRecord.getSourceRegex(), selectorRecord.getQueryType(), selectorRecord.getClientTags(), selectorRecord.getSelectorResourceEstimate(), resourceGroupIdTemplateMap.get(selectorRecord.getResourceGroupId()))).collect(Collectors.toList());
    ManagerSpec managerSpec = new ManagerSpec(rootGroups, selectors, getCpuQuotaPeriodFromDb());
    validateRootGroups(managerSpec);
    return new AbstractMap.SimpleImmutableEntry<>(managerSpec, resourceGroupSpecs);
}