Examples with ResourceDimension com.netflix.titus.api.model.ResourceDimension used on opensource projects

Example 6 with ResourceDimension

use of com.netflix.titus.api.model.ResourceDimension in project titus-control-plane by Netflix.

the class ResourceDimensionsTest method testAligningUpToHigherMemory.

@Test
public void testAligningUpToHigherMemory() throws Exception {
    ResourceDimension small2X = ResourceDimensionSample.SmallWithGpuAndOpportunisticX2.build();
    ResourceDimension original = ResourceDimensionSample.SmallWithGpuAndOpportunistic.builder().withMemoryMB(small2X.getMemoryMB()).build();
    assertThat(ResourceDimensions.alignUp(original, small2X)).isEqualTo(small2X);
}

Example 7 with ResourceDimension

use of com.netflix.titus.api.model.ResourceDimension in project titus-control-plane by Netflix.

the class ResourceConsumptionLogTest method testAllocationEventLogging.

@Test
public void testAllocationEventLogging() throws Exception {
    CompositeResourceConsumption consumption = new CompositeResourceConsumption("myCapacityGroup", ResourceConsumption.ConsumptionLevel.CapacityGroup, // actual
    new ResourceDimension(1, 0, 1, 10, 10, 0), // max
    new ResourceDimension(2, 0, 2, 20, 20, 0), // limit
    new ResourceDimension(3, 0, 3, 30, 30, 0), Collections.singletonMap("attrKey", "attrValue"), Collections.emptyMap(), false);
    CapacityGroupAllocationEvent event = new CapacityGroupAllocationEvent("myCapacityGroup", System.currentTimeMillis(), consumption);
    String result = ResourceConsumptionLog.doLog(event);
    String expected = "Resource consumption change: group=myCapacityGroup [below limit] actual=[cpu=1.0, memoryMB=1, diskMB=10, networkMbs=10, gpu=0, opportunisticCpu=0], max=[cpu=2.0, memoryMB=2, diskMB=20, networkMbs=20, gpu=0, opportunisticCpu=0], limit=[cpu=3.0, memoryMB=3, diskMB=30, networkMbs=30, gpu=0, opportunisticCpu=0], attrs={attrKey=attrValue}";
    assertThat(result).isEqualTo(expected);
}

Example 8 with ResourceDimension

use of com.netflix.titus.api.model.ResourceDimension in project titus-control-plane by Netflix.

the class ResourceConsumptions method add.

/**
 * Add a collection of {@link ResourceConsumption} values of the same kind (the same consumption name and level).
 *
 * @return aggregated value of the same type as the input
 */
public static <C extends ResourceConsumption> C add(Collection<C> consumptions) {
    verifySameKind(consumptions);
    List<C> consumptionList = new ArrayList<>(consumptions);
    if (consumptionList.size() == 1) {
        return consumptionList.get(0);
    }
    ResourceDimension currentUsage = addCurrentConsumptions(consumptionList);
    ResourceDimension maxUsage = addMaxConsumptions(consumptionList);
    Map<String, Object> mergedAttrs = mergeAttributesOf(consumptions);
    C first = consumptionList.get(0);
    if (first instanceof CompositeResourceConsumption) {
        Map<String, ResourceConsumption> mergedContributors = new HashMap<>();
        consumptionList.stream().map(CompositeResourceConsumption.class::cast).flatMap(c -> c.getContributors().entrySet().stream()).forEach(entry -> mergedContributors.compute(entry.getKey(), (name, current) -> current == null ? entry.getValue() : ResourceConsumptions.add(current, entry.getValue())));
        ResourceDimension allowedUsage = addAllowedConsumptions((Collection<CompositeResourceConsumption>) consumptionList);
        return (C) new CompositeResourceConsumption(first.getConsumerName(), first.getConsumptionLevel(), currentUsage, maxUsage, allowedUsage, mergedAttrs, mergedContributors, !ResourceDimensions.isBigger(allowedUsage, maxUsage));
    }
    return (C) new ResourceConsumption(first.getConsumerName(), first.getConsumptionLevel(), currentUsage, maxUsage, mergedAttrs);
}

Example 9 with ResourceDimension

use of com.netflix.titus.api.model.ResourceDimension in project titus-control-plane by Netflix.

the class ResourceConsumptionEvaluator method computeAllocationsByCapacityGroupAndAppName.

/**
 * @return capacityGroups -> apps -> instanceTypes -> consumption
 */
private Pair<Map<String, Map<String, ResourceConsumption>>, Set<String>> computeAllocationsByCapacityGroupAndAppName() {
    Map<String, Map<String, ResourceConsumption>> consumptionMap = new HashMap<>();
    Set<String> undefinedCapacityGroups = new HashSet<>();
    v3JobOperations.getJobsAndTasks().forEach(jobsAndTasks -> {
        Job job = jobsAndTasks.getLeft();
        List<Task> tasks = jobsAndTasks.getRight();
        List<Task> runningTasks = getRunningWorkers(tasks);
        ResourceDimension taskResources = perTaskResourceDimension(job);
        String appName = Evaluators.getOrDefault(job.getJobDescriptor().getApplicationName(), DEFAULT_APPLICATION);
        ResourceDimension currentConsumption = ResourceDimensions.multiply(taskResources, runningTasks.size());
        ResourceDimension maxConsumption = ResourceDimensions.multiply(taskResources, getMaxJobSize(job));
        Map<String, List<Task>> tasksByInstanceType = tasks.stream().collect(groupingBy(task -> task.getTaskContext().getOrDefault(TaskAttributes.TASK_ATTRIBUTES_AGENT_ITYPE, "unknown")));
        Map<String, ResourceConsumption> consumptionByInstanceType = CollectionsExt.mapValuesWithKeys(tasksByInstanceType, (instanceType, instanceTypeTasks) -> {
            List<Task> runningInstanceTypeTasks = getRunningWorkers(instanceTypeTasks);
            ResourceDimension instanceTypeConsumption = ResourceDimensions.multiply(taskResources, runningInstanceTypeTasks.size());
            return new ResourceConsumption(instanceType, ConsumptionLevel.InstanceType, instanceTypeConsumption, // maxConsumption is not relevant at ConsumptionLevel.InstanceType
            instanceTypeConsumption, getWorkerStateMap(instanceTypeTasks));
        }, HashMap::new);
        ResourceConsumption jobConsumption = new CompositeResourceConsumption(appName, ConsumptionLevel.Application, currentConsumption, maxConsumption, // allowedConsumption is not relevant at ConsumptionLevel.Application
        maxConsumption, getWorkerStateMap(tasks), consumptionByInstanceType, // we consider a job is always within its allowed usage since it can't go over its max
        false);
        String capacityGroup = resolveCapacityGroup(undefinedCapacityGroups, job, appName);
        updateConsumptionMap(appName, capacityGroup, jobConsumption, consumptionMap);
    });
    // Add unused capacity groups
    copyAndRemove(definedCapacityGroups, consumptionMap.keySet()).forEach(capacityGroup -> consumptionMap.put(capacityGroup, Collections.emptyMap()));
    return Pair.of(consumptionMap, undefinedCapacityGroups);
}

Example 10 with ResourceDimension

use of com.netflix.titus.api.model.ResourceDimension in project titus-control-plane by Netflix.

the class ResourceConsumptionEvaluator method buildSystemConsumption.

private CompositeResourceConsumption buildSystemConsumption(Map<String, Map<String, ResourceConsumption>> capacityGroupConsumptionMap) {
    // Capacity group level
    Map<Tier, List<CompositeResourceConsumption>> tierConsumptions = new HashMap<>();
    capacityGroupConsumptionMap.forEach((capacityGroup, appConsumptions) -> {
        ApplicationSLA sla = applicationSlaMap.get(capacityGroup);
        ResourceDimension allowedConsumption = ResourceDimensions.multiply(sla.getResourceDimension(), sla.getInstanceCount());
        ResourceDimension maxConsumption = ResourceConsumptions.addMaxConsumptions(appConsumptions.values());
        List<Map<String, Object>> attrsList = appConsumptions.values().stream().map(ResourceConsumption::getAttributes).collect(Collectors.toList());
        CompositeResourceConsumption capacityGroupConsumption = new CompositeResourceConsumption(capacityGroup, ConsumptionLevel.CapacityGroup, ResourceConsumptions.addCurrentConsumptions(appConsumptions.values()), maxConsumption, allowedConsumption, ResourceConsumptions.mergeAttributes(attrsList), appConsumptions, !ResourceDimensions.isBigger(allowedConsumption, maxConsumption));
        tierConsumptions.computeIfAbsent(sla.getTier(), t -> new ArrayList<>()).add(capacityGroupConsumption);
    });
    // Tier level
    List<CompositeResourceConsumption> aggregatedTierConsumptions = new ArrayList<>();
    tierConsumptions.forEach((tier, consumptions) -> aggregatedTierConsumptions.add(ResourceConsumptions.aggregate(tier.name(), ConsumptionLevel.Tier, consumptions)));
    // System level
    return ResourceConsumptions.aggregate(SYSTEM_CONSUMER, ConsumptionLevel.System, aggregatedTierConsumptions);
}