Examples with ResourceConsumption com.netflix.titus.master.service.management.ResourceConsumption used on opensource projects

Example 1 with ResourceConsumption

use of com.netflix.titus.master.service.management.ResourceConsumption in project titus-control-plane by Netflix.

the class ResourceConsumptionEvaluatorTest method batchJobWithMultipleTasks.

@SuppressWarnings("unchecked")
@Test
public void batchJobWithMultipleTasks() {
    when(applicationSlaManagementService.getApplicationSLAs()).thenReturn(asList(ConsumptionModelGenerator.DEFAULT_SLA, ConsumptionModelGenerator.CRITICAL_SLA_1, ConsumptionModelGenerator.NOT_USED_SLA));
    // Job with defined capacity group SLA
    Job<BatchJobExt> goodCapacityJob = newBatchJob("goodCapacityJob", jd -> jd.toBuilder().withExtensions(jd.getExtensions().toBuilder().withSize(2).build()).withCapacityGroup(ConsumptionModelGenerator.CRITICAL_SLA_1.getAppName()).build()).getLeft();
    List<Task> goodCapacityTasks = jobComponentStub.getJobOperations().getTasks(goodCapacityJob.getId());
    // Job without appName defined
    Job<BatchJobExt> noAppNameJob = newBatchJob("badCapacityJob", jd -> jd.toBuilder().withApplicationName("").withExtensions(jd.getExtensions().toBuilder().withSize(2).build()).withCapacityGroup(ConsumptionModelGenerator.DEFAULT_SLA.getAppName()).build()).getLeft();
    List<Task> noAppNameTasks = jobComponentStub.getJobOperations().getTasks(noAppNameJob.getId());
    // Job with capacity group for which SLA is not defined
    Job<BatchJobExt> badCapacityJob = newBatchJob("badCapacityJob", jd -> jd.toBuilder().withExtensions(jd.getExtensions().toBuilder().withSize(2).build()).withCapacityGroup("missingCapacityGroup").build()).getLeft();
    List<Task> badCapacityTasks = jobComponentStub.getJobOperations().getTasks(badCapacityJob.getId());
    // Evaluate
    ResourceConsumptionEvaluator evaluator = new ResourceConsumptionEvaluator(applicationSlaManagementService, v3JobOperations);
    Set<String> undefined = evaluator.getUndefinedCapacityGroups();
    assertThat(undefined).contains("missingCapacityGroup");
    CompositeResourceConsumption systemConsumption = evaluator.getSystemConsumption();
    Map<String, ResourceConsumption> tierConsumptions = systemConsumption.getContributors();
    assertThat(tierConsumptions).containsKeys(Tier.Critical.name(), Tier.Flex.name());
    // Critical capacity group
    CompositeResourceConsumption criticalConsumption = (CompositeResourceConsumption) findConsumption(systemConsumption, Tier.Critical.name(), ConsumptionModelGenerator.CRITICAL_SLA_1.getAppName()).get();
    assertThat(criticalConsumption.getCurrentConsumption()).isEqualTo(expectedCurrentConsumptionForBatchJob(goodCapacityJob, goodCapacityTasks));
    assertThat(criticalConsumption.getMaxConsumption()).isEqualTo(expectedMaxConsumptionForBatchJob(goodCapacityJob));
    assertThat(criticalConsumption.getAllowedConsumption()).isEqualTo(ConsumptionModelGenerator.capacityGroupLimit(ConsumptionModelGenerator.CRITICAL_SLA_1));
    assertThat(criticalConsumption.isAboveLimit()).isTrue();
    // Default capacity group
    CompositeResourceConsumption defaultConsumption = (CompositeResourceConsumption) findConsumption(systemConsumption, Tier.Flex.name(), ConsumptionModelGenerator.DEFAULT_SLA.getAppName()).get();
    assertThat(defaultConsumption.getCurrentConsumption()).isEqualTo(ResourceDimensions.add(expectedCurrentConsumptionForBatchJob(noAppNameJob, noAppNameTasks), expectedCurrentConsumptionForBatchJob(badCapacityJob, badCapacityTasks)));
    assertThat(defaultConsumption.getMaxConsumption()).isEqualTo(ResourceDimensions.add(expectedMaxConsumptionForBatchJob(noAppNameJob), expectedMaxConsumptionForBatchJob(badCapacityJob)));
    assertThat(defaultConsumption.getAllowedConsumption()).isEqualTo(ConsumptionModelGenerator.capacityGroupLimit(ConsumptionModelGenerator.DEFAULT_SLA));
    assertThat(defaultConsumption.isAboveLimit()).isFalse();
    // Not used capacity group
    CompositeResourceConsumption notUsedConsumption = (CompositeResourceConsumption) findConsumption(systemConsumption, Tier.Critical.name(), ConsumptionModelGenerator.NOT_USED_SLA.getAppName()).get();
    assertThat(notUsedConsumption.getCurrentConsumption()).isEqualTo(ResourceDimension.empty());
    assertThat(notUsedConsumption.getAllowedConsumption()).isEqualTo(ConsumptionModelGenerator.capacityGroupLimit(ConsumptionModelGenerator.NOT_USED_SLA));
    assertThat(notUsedConsumption.isAboveLimit()).isFalse();
}

Example 2 with ResourceConsumption

use of com.netflix.titus.master.service.management.ResourceConsumption in project titus-control-plane by Netflix.

the class DefaultResourceConsumptionService method notifyAboutResourceConsumptionChange.

private void notifyAboutResourceConsumptionChange(ConsumptionEvaluationResult oldEvaluation) {
    Map<String, ResourceConsumption> newCapacityGroupConsumptions = ResourceConsumptions.groupBy(latestEvaluation.getSystemConsumption(), ConsumptionLevel.CapacityGroup);
    Map<String, ResourceConsumption> oldCapacityGroupConsumptions = oldEvaluation == null ? Collections.emptyMap() : ResourceConsumptions.groupBy(oldEvaluation.getSystemConsumption(), ConsumptionLevel.CapacityGroup);
    long now = worker.now();
    newCapacityGroupConsumptions.values().forEach(newConsumption -> {
        ResourceConsumption previous = oldCapacityGroupConsumptions.get(newConsumption.getConsumerName());
        if (previous == null || !previous.equals(newConsumption)) {
            publishEvent(new CapacityGroupAllocationEvent(newConsumption.getConsumerName(), now, (CompositeResourceConsumption) newConsumption));
        }
    });
}

Example 3 with ResourceConsumption

use of com.netflix.titus.master.service.management.ResourceConsumption in project titus-control-plane by Netflix.

the class ResourceConsumptionEvaluator method computeAllocationsByCapacityGroupAndAppName.

/**
 * @return capacityGroups -> apps -> instanceTypes -> consumption
 */
private Pair<Map<String, Map<String, ResourceConsumption>>, Set<String>> computeAllocationsByCapacityGroupAndAppName() {
    Map<String, Map<String, ResourceConsumption>> consumptionMap = new HashMap<>();
    Set<String> undefinedCapacityGroups = new HashSet<>();
    v3JobOperations.getJobsAndTasks().forEach(jobsAndTasks -> {
        Job job = jobsAndTasks.getLeft();
        List<Task> tasks = jobsAndTasks.getRight();
        List<Task> runningTasks = getRunningWorkers(tasks);
        ResourceDimension taskResources = perTaskResourceDimension(job);
        String appName = Evaluators.getOrDefault(job.getJobDescriptor().getApplicationName(), DEFAULT_APPLICATION);
        ResourceDimension currentConsumption = ResourceDimensions.multiply(taskResources, runningTasks.size());
        ResourceDimension maxConsumption = ResourceDimensions.multiply(taskResources, getMaxJobSize(job));
        Map<String, List<Task>> tasksByInstanceType = tasks.stream().collect(groupingBy(task -> task.getTaskContext().getOrDefault(TaskAttributes.TASK_ATTRIBUTES_AGENT_ITYPE, "unknown")));
        Map<String, ResourceConsumption> consumptionByInstanceType = CollectionsExt.mapValuesWithKeys(tasksByInstanceType, (instanceType, instanceTypeTasks) -> {
            List<Task> runningInstanceTypeTasks = getRunningWorkers(instanceTypeTasks);
            ResourceDimension instanceTypeConsumption = ResourceDimensions.multiply(taskResources, runningInstanceTypeTasks.size());
            return new ResourceConsumption(instanceType, ConsumptionLevel.InstanceType, instanceTypeConsumption, // maxConsumption is not relevant at ConsumptionLevel.InstanceType
            instanceTypeConsumption, getWorkerStateMap(instanceTypeTasks));
        }, HashMap::new);
        ResourceConsumption jobConsumption = new CompositeResourceConsumption(appName, ConsumptionLevel.Application, currentConsumption, maxConsumption, // allowedConsumption is not relevant at ConsumptionLevel.Application
        maxConsumption, getWorkerStateMap(tasks), consumptionByInstanceType, // we consider a job is always within its allowed usage since it can't go over its max
        false);
        String capacityGroup = resolveCapacityGroup(undefinedCapacityGroups, job, appName);
        updateConsumptionMap(appName, capacityGroup, jobConsumption, consumptionMap);
    });
    // Add unused capacity groups
    copyAndRemove(definedCapacityGroups, consumptionMap.keySet()).forEach(capacityGroup -> consumptionMap.put(capacityGroup, Collections.emptyMap()));
    return Pair.of(consumptionMap, undefinedCapacityGroups);
}

Example 4 with ResourceConsumption

use of com.netflix.titus.master.service.management.ResourceConsumption in project titus-control-plane by Netflix.

the class ResourceConsumptionEvaluator method updateConsumptionMap.

private void updateConsumptionMap(String applicationName, String capacityGroup, ResourceConsumption jobConsumption, Map<String, Map<String, ResourceConsumption>> consumptionMap) {
    Map<String, ResourceConsumption> capacityGroupAllocation = consumptionMap.computeIfAbsent(capacityGroup, k -> new HashMap<>());
    String effectiveAppName = applicationName == null ? DEFAULT_APPLICATION : applicationName;
    ResourceConsumption appAllocation = capacityGroupAllocation.get(effectiveAppName);
    if (appAllocation == null) {
        capacityGroupAllocation.put(effectiveAppName, jobConsumption);
    } else {
        capacityGroupAllocation.put(effectiveAppName, ResourceConsumptions.add(appAllocation, jobConsumption));
    }
}

Example 5 with ResourceConsumption

use of com.netflix.titus.master.service.management.ResourceConsumption in project titus-control-plane by Netflix.

the class ResourceConsumptionEvaluator method buildSystemConsumption.

private CompositeResourceConsumption buildSystemConsumption(Map<String, Map<String, ResourceConsumption>> capacityGroupConsumptionMap) {
    // Capacity group level
    Map<Tier, List<CompositeResourceConsumption>> tierConsumptions = new HashMap<>();
    capacityGroupConsumptionMap.forEach((capacityGroup, appConsumptions) -> {
        ApplicationSLA sla = applicationSlaMap.get(capacityGroup);
        ResourceDimension allowedConsumption = ResourceDimensions.multiply(sla.getResourceDimension(), sla.getInstanceCount());
        ResourceDimension maxConsumption = ResourceConsumptions.addMaxConsumptions(appConsumptions.values());
        List<Map<String, Object>> attrsList = appConsumptions.values().stream().map(ResourceConsumption::getAttributes).collect(Collectors.toList());
        CompositeResourceConsumption capacityGroupConsumption = new CompositeResourceConsumption(capacityGroup, ConsumptionLevel.CapacityGroup, ResourceConsumptions.addCurrentConsumptions(appConsumptions.values()), maxConsumption, allowedConsumption, ResourceConsumptions.mergeAttributes(attrsList), appConsumptions, !ResourceDimensions.isBigger(allowedConsumption, maxConsumption));
        tierConsumptions.computeIfAbsent(sla.getTier(), t -> new ArrayList<>()).add(capacityGroupConsumption);
    });
    // Tier level
    List<CompositeResourceConsumption> aggregatedTierConsumptions = new ArrayList<>();
    tierConsumptions.forEach((tier, consumptions) -> aggregatedTierConsumptions.add(ResourceConsumptions.aggregate(tier.name(), ConsumptionLevel.Tier, consumptions)));
    // System level
    return ResourceConsumptions.aggregate(SYSTEM_CONSUMER, ConsumptionLevel.System, aggregatedTierConsumptions);
}