Examples with CompositeResourceConsumption com.netflix.titus.master.service.management.CompositeResourceConsumption used on opensource projects

Example 1 with CompositeResourceConsumption

use of com.netflix.titus.master.service.management.CompositeResourceConsumption in project titus-control-plane by Netflix.

the class ResourceConsumptionLog method doLog.

/* Visible for testing */
static String doLog(ResourceConsumptionEvents.ResourceConsumptionEvent event) {
    if (event instanceof CapacityGroupAllocationEvent) {
        CapacityGroupAllocationEvent changeEvent = (CapacityGroupAllocationEvent) event;
        CompositeResourceConsumption consumption = changeEvent.getCapacityGroupConsumption();
        StringBuilder sb = new StringBuilder("Resource consumption change: group=");
        sb.append(consumption.getConsumerName());
        if (consumption.isAboveLimit()) {
            sb.append(" [above limit] ");
        } else {
            sb.append(" [below limit] ");
        }
        sb.append("actual=");
        ResourceDimensions.format(consumption.getCurrentConsumption(), sb);
        sb.append(", max=");
        ResourceDimensions.format(consumption.getMaxConsumption(), sb);
        sb.append(", limit=");
        ResourceDimensions.format(consumption.getAllowedConsumption(), sb);
        if (consumption.getAttributes().isEmpty()) {
            sb.append(", attrs={}");
        } else {
            sb.append(", attrs={");
            consumption.getAttributes().forEach((k, v) -> sb.append(k).append('=').append(v).append(','));
            sb.setCharAt(sb.length() - 1, '}');
        }
        String message = sb.toString();
        if (consumption.isAboveLimit()) {
            logger.warn(message);
        } else {
            logger.info(message);
        }
        return message;
    }
    if (event instanceof CapacityGroupUndefinedEvent) {
        String message = "Capacity group not defined: group=" + event.getCapacityGroup();
        logger.warn(message);
        return message;
    }
    if (event instanceof CapacityGroupRemovedEvent) {
        String message = "Capacity group no longer defined: group=" + event.getCapacityGroup();
        logger.info(message);
        return message;
    }
    String message = "Unrecognized resource consumption event type " + event.getClass();
    logger.error(message);
    return message;
}

Example 2 with CompositeResourceConsumption

use of com.netflix.titus.master.service.management.CompositeResourceConsumption in project titus-control-plane by Netflix.

the class ResourceConsumptionServiceMetrics method update.

public void update(DefaultResourceConsumptionService.ConsumptionEvaluationResult evaluationResult) {
    CompositeResourceConsumption systemConsumption = evaluationResult.getSystemConsumption();
    Set<Pair<String, String>> touchedApps = new HashSet<>();
    Set<String> touchedGroups = new HashSet<>();
    systemConsumption.getContributors().values().forEach(tierConsumption -> updateTier((CompositeResourceConsumption) tierConsumption, touchedApps, touchedGroups));
    // Remove no longer referenced items
    copyAndRemove(metricsByCapacityGroupAndApp.keySet(), touchedApps).forEach(removed -> metricsByCapacityGroupAndApp.remove(removed).reset());
    copyAndRemove(capacityGroupLimits.keySet(), touchedGroups).forEach(removed -> capacityGroupLimits.remove(removed).reset());
    updateTimestamp = registry.clock().wallTime();
}

Example 3 with CompositeResourceConsumption

use of com.netflix.titus.master.service.management.CompositeResourceConsumption in project titus-control-plane by Netflix.

the class ResourceConsumptionEvaluatorTest method batchJobWithMultipleTasks.

@SuppressWarnings("unchecked")
@Test
public void batchJobWithMultipleTasks() {
    when(applicationSlaManagementService.getApplicationSLAs()).thenReturn(asList(ConsumptionModelGenerator.DEFAULT_SLA, ConsumptionModelGenerator.CRITICAL_SLA_1, ConsumptionModelGenerator.NOT_USED_SLA));
    // Job with defined capacity group SLA
    Job<BatchJobExt> goodCapacityJob = newBatchJob("goodCapacityJob", jd -> jd.toBuilder().withExtensions(jd.getExtensions().toBuilder().withSize(2).build()).withCapacityGroup(ConsumptionModelGenerator.CRITICAL_SLA_1.getAppName()).build()).getLeft();
    List<Task> goodCapacityTasks = jobComponentStub.getJobOperations().getTasks(goodCapacityJob.getId());
    // Job without appName defined
    Job<BatchJobExt> noAppNameJob = newBatchJob("badCapacityJob", jd -> jd.toBuilder().withApplicationName("").withExtensions(jd.getExtensions().toBuilder().withSize(2).build()).withCapacityGroup(ConsumptionModelGenerator.DEFAULT_SLA.getAppName()).build()).getLeft();
    List<Task> noAppNameTasks = jobComponentStub.getJobOperations().getTasks(noAppNameJob.getId());
    // Job with capacity group for which SLA is not defined
    Job<BatchJobExt> badCapacityJob = newBatchJob("badCapacityJob", jd -> jd.toBuilder().withExtensions(jd.getExtensions().toBuilder().withSize(2).build()).withCapacityGroup("missingCapacityGroup").build()).getLeft();
    List<Task> badCapacityTasks = jobComponentStub.getJobOperations().getTasks(badCapacityJob.getId());
    // Evaluate
    ResourceConsumptionEvaluator evaluator = new ResourceConsumptionEvaluator(applicationSlaManagementService, v3JobOperations);
    Set<String> undefined = evaluator.getUndefinedCapacityGroups();
    assertThat(undefined).contains("missingCapacityGroup");
    CompositeResourceConsumption systemConsumption = evaluator.getSystemConsumption();
    Map<String, ResourceConsumption> tierConsumptions = systemConsumption.getContributors();
    assertThat(tierConsumptions).containsKeys(Tier.Critical.name(), Tier.Flex.name());
    // Critical capacity group
    CompositeResourceConsumption criticalConsumption = (CompositeResourceConsumption) findConsumption(systemConsumption, Tier.Critical.name(), ConsumptionModelGenerator.CRITICAL_SLA_1.getAppName()).get();
    assertThat(criticalConsumption.getCurrentConsumption()).isEqualTo(expectedCurrentConsumptionForBatchJob(goodCapacityJob, goodCapacityTasks));
    assertThat(criticalConsumption.getMaxConsumption()).isEqualTo(expectedMaxConsumptionForBatchJob(goodCapacityJob));
    assertThat(criticalConsumption.getAllowedConsumption()).isEqualTo(ConsumptionModelGenerator.capacityGroupLimit(ConsumptionModelGenerator.CRITICAL_SLA_1));
    assertThat(criticalConsumption.isAboveLimit()).isTrue();
    // Default capacity group
    CompositeResourceConsumption defaultConsumption = (CompositeResourceConsumption) findConsumption(systemConsumption, Tier.Flex.name(), ConsumptionModelGenerator.DEFAULT_SLA.getAppName()).get();
    assertThat(defaultConsumption.getCurrentConsumption()).isEqualTo(ResourceDimensions.add(expectedCurrentConsumptionForBatchJob(noAppNameJob, noAppNameTasks), expectedCurrentConsumptionForBatchJob(badCapacityJob, badCapacityTasks)));
    assertThat(defaultConsumption.getMaxConsumption()).isEqualTo(ResourceDimensions.add(expectedMaxConsumptionForBatchJob(noAppNameJob), expectedMaxConsumptionForBatchJob(badCapacityJob)));
    assertThat(defaultConsumption.getAllowedConsumption()).isEqualTo(ConsumptionModelGenerator.capacityGroupLimit(ConsumptionModelGenerator.DEFAULT_SLA));
    assertThat(defaultConsumption.isAboveLimit()).isFalse();
    // Not used capacity group
    CompositeResourceConsumption notUsedConsumption = (CompositeResourceConsumption) findConsumption(systemConsumption, Tier.Critical.name(), ConsumptionModelGenerator.NOT_USED_SLA.getAppName()).get();
    assertThat(notUsedConsumption.getCurrentConsumption()).isEqualTo(ResourceDimension.empty());
    assertThat(notUsedConsumption.getAllowedConsumption()).isEqualTo(ConsumptionModelGenerator.capacityGroupLimit(ConsumptionModelGenerator.NOT_USED_SLA));
    assertThat(notUsedConsumption.isAboveLimit()).isFalse();
}

Example 4 with CompositeResourceConsumption

use of com.netflix.titus.master.service.management.CompositeResourceConsumption in project titus-control-plane by Netflix.

the class ResourceConsumptionLogTest method testAllocationEventLogging.

@Test
public void testAllocationEventLogging() throws Exception {
    CompositeResourceConsumption consumption = new CompositeResourceConsumption("myCapacityGroup", ResourceConsumption.ConsumptionLevel.CapacityGroup, // actual
    new ResourceDimension(1, 0, 1, 10, 10, 0), // max
    new ResourceDimension(2, 0, 2, 20, 20, 0), // limit
    new ResourceDimension(3, 0, 3, 30, 30, 0), Collections.singletonMap("attrKey", "attrValue"), Collections.emptyMap(), false);
    CapacityGroupAllocationEvent event = new CapacityGroupAllocationEvent("myCapacityGroup", System.currentTimeMillis(), consumption);
    String result = ResourceConsumptionLog.doLog(event);
    String expected = "Resource consumption change: group=myCapacityGroup [below limit] actual=[cpu=1.0, memoryMB=1, diskMB=10, networkMbs=10, gpu=0, opportunisticCpu=0], max=[cpu=2.0, memoryMB=2, diskMB=20, networkMbs=20, gpu=0, opportunisticCpu=0], limit=[cpu=3.0, memoryMB=3, diskMB=30, networkMbs=30, gpu=0, opportunisticCpu=0], attrs={attrKey=attrValue}";
    assertThat(result).isEqualTo(expected);
}

Example 5 with CompositeResourceConsumption

use of com.netflix.titus.master.service.management.CompositeResourceConsumption in project titus-control-plane by Netflix.

the class DefaultResourceConsumptionService method notifyAboutResourceConsumptionChange.

private void notifyAboutResourceConsumptionChange(ConsumptionEvaluationResult oldEvaluation) {
    Map<String, ResourceConsumption> newCapacityGroupConsumptions = ResourceConsumptions.groupBy(latestEvaluation.getSystemConsumption(), ConsumptionLevel.CapacityGroup);
    Map<String, ResourceConsumption> oldCapacityGroupConsumptions = oldEvaluation == null ? Collections.emptyMap() : ResourceConsumptions.groupBy(oldEvaluation.getSystemConsumption(), ConsumptionLevel.CapacityGroup);
    long now = worker.now();
    newCapacityGroupConsumptions.values().forEach(newConsumption -> {
        ResourceConsumption previous = oldCapacityGroupConsumptions.get(newConsumption.getConsumerName());
        if (previous == null || !previous.equals(newConsumption)) {
            publishEvent(new CapacityGroupAllocationEvent(newConsumption.getConsumerName(), now, (CompositeResourceConsumption) newConsumption));
        }
    });
}