Search in sources :

Example 1 with DEFAULT_APPLICATION

use of com.netflix.titus.master.service.management.ApplicationSlaManagementService.DEFAULT_APPLICATION in project titus-control-plane by Netflix.

the class ResourceConsumptionEvaluator method computeAllocationsByCapacityGroupAndAppName.

/**
 * @return capacityGroups -> apps -> instanceTypes -> consumption
 */
private Pair<Map<String, Map<String, ResourceConsumption>>, Set<String>> computeAllocationsByCapacityGroupAndAppName() {
    Map<String, Map<String, ResourceConsumption>> consumptionMap = new HashMap<>();
    Set<String> undefinedCapacityGroups = new HashSet<>();
    v3JobOperations.getJobsAndTasks().forEach(jobsAndTasks -> {
        Job job = jobsAndTasks.getLeft();
        List<Task> tasks = jobsAndTasks.getRight();
        List<Task> runningTasks = getRunningWorkers(tasks);
        ResourceDimension taskResources = perTaskResourceDimension(job);
        String appName = Evaluators.getOrDefault(job.getJobDescriptor().getApplicationName(), DEFAULT_APPLICATION);
        ResourceDimension currentConsumption = ResourceDimensions.multiply(taskResources, runningTasks.size());
        ResourceDimension maxConsumption = ResourceDimensions.multiply(taskResources, getMaxJobSize(job));
        Map<String, List<Task>> tasksByInstanceType = tasks.stream().collect(groupingBy(task -> task.getTaskContext().getOrDefault(TaskAttributes.TASK_ATTRIBUTES_AGENT_ITYPE, "unknown")));
        Map<String, ResourceConsumption> consumptionByInstanceType = CollectionsExt.mapValuesWithKeys(tasksByInstanceType, (instanceType, instanceTypeTasks) -> {
            List<Task> runningInstanceTypeTasks = getRunningWorkers(instanceTypeTasks);
            ResourceDimension instanceTypeConsumption = ResourceDimensions.multiply(taskResources, runningInstanceTypeTasks.size());
            return new ResourceConsumption(instanceType, ConsumptionLevel.InstanceType, instanceTypeConsumption, // maxConsumption is not relevant at ConsumptionLevel.InstanceType
            instanceTypeConsumption, getWorkerStateMap(instanceTypeTasks));
        }, HashMap::new);
        ResourceConsumption jobConsumption = new CompositeResourceConsumption(appName, ConsumptionLevel.Application, currentConsumption, maxConsumption, // allowedConsumption is not relevant at ConsumptionLevel.Application
        maxConsumption, getWorkerStateMap(tasks), consumptionByInstanceType, // we consider a job is always within its allowed usage since it can't go over its max
        false);
        String capacityGroup = resolveCapacityGroup(undefinedCapacityGroups, job, appName);
        updateConsumptionMap(appName, capacityGroup, jobConsumption, consumptionMap);
    });
    // Add unused capacity groups
    copyAndRemove(definedCapacityGroups, consumptionMap.keySet()).forEach(capacityGroup -> consumptionMap.put(capacityGroup, Collections.emptyMap()));
    return Pair.of(consumptionMap, undefinedCapacityGroups);
}
Also used : DEFAULT_APPLICATION(com.netflix.titus.master.service.management.ApplicationSlaManagementService.DEFAULT_APPLICATION) ApplicationSlaManagementService(com.netflix.titus.master.service.management.ApplicationSlaManagementService) Task(com.netflix.titus.api.jobmanager.model.job.Task) CollectionsExt(com.netflix.titus.common.util.CollectionsExt) LoggerFactory(org.slf4j.LoggerFactory) Collectors.groupingBy(java.util.stream.Collectors.groupingBy) HashMap(java.util.HashMap) ResourceDimensions(com.netflix.titus.master.model.ResourceDimensions) Function(java.util.function.Function) Supplier(java.util.function.Supplier) CollectionsExt.copyAndRemove(com.netflix.titus.common.util.CollectionsExt.copyAndRemove) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) ResourceConsumption(com.netflix.titus.master.service.management.ResourceConsumption) Pair(com.netflix.titus.common.util.tuple.Pair) Map(java.util.Map) BatchJobExt(com.netflix.titus.api.jobmanager.model.job.ext.BatchJobExt) ApplicationSLA(com.netflix.titus.api.model.ApplicationSLA) ConsumptionLevel(com.netflix.titus.master.service.management.ResourceConsumption.ConsumptionLevel) ResourceDimension(com.netflix.titus.api.model.ResourceDimension) SYSTEM_CONSUMER(com.netflix.titus.master.service.management.ResourceConsumption.SYSTEM_CONSUMER) Job(com.netflix.titus.api.jobmanager.model.job.Job) Logger(org.slf4j.Logger) ServiceJobExt(com.netflix.titus.api.jobmanager.model.job.ext.ServiceJobExt) Collection(java.util.Collection) Set(java.util.Set) JobFunctions(com.netflix.titus.api.jobmanager.model.job.JobFunctions) Tier(com.netflix.titus.api.model.Tier) Collectors(java.util.stream.Collectors) TaskState(com.netflix.titus.api.jobmanager.model.job.TaskState) List(java.util.List) V3JobOperations(com.netflix.titus.api.jobmanager.service.V3JobOperations) ContainerResources(com.netflix.titus.api.jobmanager.model.job.ContainerResources) TaskAttributes(com.netflix.titus.api.jobmanager.TaskAttributes) VisibleForTesting(com.google.common.annotations.VisibleForTesting) CompositeResourceConsumption(com.netflix.titus.master.service.management.CompositeResourceConsumption) Evaluators(com.netflix.titus.common.util.Evaluators) Collections(java.util.Collections) ResourceConsumptions(com.netflix.titus.master.service.management.ResourceConsumptions) Task(com.netflix.titus.api.jobmanager.model.job.Task) HashMap(java.util.HashMap) CompositeResourceConsumption(com.netflix.titus.master.service.management.CompositeResourceConsumption) ResourceConsumption(com.netflix.titus.master.service.management.ResourceConsumption) CompositeResourceConsumption(com.netflix.titus.master.service.management.CompositeResourceConsumption) ArrayList(java.util.ArrayList) List(java.util.List) Job(com.netflix.titus.api.jobmanager.model.job.Job) HashMap(java.util.HashMap) Map(java.util.Map) ResourceDimension(com.netflix.titus.api.model.ResourceDimension) HashSet(java.util.HashSet)

Aggregations

VisibleForTesting (com.google.common.annotations.VisibleForTesting)1 TaskAttributes (com.netflix.titus.api.jobmanager.TaskAttributes)1 ContainerResources (com.netflix.titus.api.jobmanager.model.job.ContainerResources)1 Job (com.netflix.titus.api.jobmanager.model.job.Job)1 JobFunctions (com.netflix.titus.api.jobmanager.model.job.JobFunctions)1 Task (com.netflix.titus.api.jobmanager.model.job.Task)1 TaskState (com.netflix.titus.api.jobmanager.model.job.TaskState)1 BatchJobExt (com.netflix.titus.api.jobmanager.model.job.ext.BatchJobExt)1 ServiceJobExt (com.netflix.titus.api.jobmanager.model.job.ext.ServiceJobExt)1 V3JobOperations (com.netflix.titus.api.jobmanager.service.V3JobOperations)1 ApplicationSLA (com.netflix.titus.api.model.ApplicationSLA)1 ResourceDimension (com.netflix.titus.api.model.ResourceDimension)1 Tier (com.netflix.titus.api.model.Tier)1 CollectionsExt (com.netflix.titus.common.util.CollectionsExt)1 CollectionsExt.copyAndRemove (com.netflix.titus.common.util.CollectionsExt.copyAndRemove)1 Evaluators (com.netflix.titus.common.util.Evaluators)1 Pair (com.netflix.titus.common.util.tuple.Pair)1 ResourceDimensions (com.netflix.titus.master.model.ResourceDimensions)1 ApplicationSlaManagementService (com.netflix.titus.master.service.management.ApplicationSlaManagementService)1 DEFAULT_APPLICATION (com.netflix.titus.master.service.management.ApplicationSlaManagementService.DEFAULT_APPLICATION)1