Examples with RequestUtilization com.hubspot.singularity.RequestUtilization used on opensource projects

Example 1 with RequestUtilization

use of com.hubspot.singularity.RequestUtilization in project Singularity by HubSpot.

the class SingularityMesosOfferScheduler method getMaxProbableUsageForSlave.

private MaxProbableUsage getMaxProbableUsageForSlave(List<SingularityTaskId> activeTaskIds, Map<String, RequestUtilization> requestUtilizations, String sanitizedHostname) {
    double cpu = 0;
    double memBytes = 0;
    double diskBytes = 0;
    for (SingularityTaskId taskId : activeTaskIds) {
        if (taskId.getSanitizedHost().equals(sanitizedHostname)) {
            if (requestUtilizations.containsKey(taskId.getRequestId())) {
                RequestUtilization utilization = requestUtilizations.get(taskId.getRequestId());
                // To account for cpu bursts, tend towards max usage if the app is consistently over-utilizing cpu, tend towards avg if it is over-utilized in short bursts
                cpu += (utilization.getMaxCpuUsed() - utilization.getAvgCpuUsed()) * utilization.getCpuBurstRating() + utilization.getAvgCpuUsed();
                memBytes += utilization.getMaxMemBytesUsed();
                diskBytes += utilization.getMaxDiskBytesUsed();
            } else {
                Optional<SingularityTask> maybeTask = taskManager.getTask(taskId);
                if (maybeTask.isPresent()) {
                    Resources resources = maybeTask.get().getTaskRequest().getPendingTask().getResources().or(maybeTask.get().getTaskRequest().getDeploy().getResources()).or(defaultResources);
                    cpu += resources.getCpus();
                    memBytes += resources.getDiskMb() * SingularitySlaveUsage.BYTES_PER_MEGABYTE;
                    diskBytes += resources.getDiskMb() * SingularitySlaveUsage.BYTES_PER_MEGABYTE;
                }
            }
        }
    }
    return new MaxProbableUsage(cpu, memBytes, diskBytes);
}

Example 2 with RequestUtilization

use of com.hubspot.singularity.RequestUtilization in project Singularity by HubSpot.

the class SingularityMesosOfferScheduler method checkOffers.

public Collection<SingularityOfferHolder> checkOffers(final Collection<Offer> offers) {
    for (SingularityPendingTaskId taskId : taskManager.getPendingTasksMarkedForDeletion()) {
        lock.runWithRequestLock(() -> taskManager.deletePendingTask(taskId), taskId.getRequestId(), String.format("%s#%s", getClass().getSimpleName(), "checkOffers -> pendingTaskDeletes"));
    }
    scheduler.checkForDecomissions();
    scheduler.drainPendingQueue();
    if (offers.isEmpty()) {
        LOG.debug("No offers to check");
        return Collections.emptyList();
    }
    final List<SingularityTaskRequestHolder> sortedTaskRequestHolders = getSortedDueTaskRequests();
    final int numDueTasks = sortedTaskRequestHolders.size();
    final Map<String, SingularityOfferHolder> offerHolders = offers.stream().collect(Collectors.groupingBy((o) -> o.getAgentId().getValue())).entrySet().stream().filter((e) -> e.getValue().size() > 0).map((e) -> {
        List<Offer> offersList = e.getValue();
        String slaveId = e.getKey();
        return new SingularityOfferHolder(offersList, numDueTasks, slaveAndRackHelper.getRackIdOrDefault(offersList.get(0)), slaveId, offersList.get(0).getHostname(), slaveAndRackHelper.getTextAttributes(offersList.get(0)), slaveAndRackHelper.getReservedSlaveAttributes(offersList.get(0)));
    }).collect(Collectors.toMap(SingularityOfferHolder::getSlaveId, Function.identity()));
    if (sortedTaskRequestHolders.isEmpty()) {
        return offerHolders.values();
    }
    final AtomicInteger tasksScheduled = new AtomicInteger(0);
    Map<String, RequestUtilization> requestUtilizations = usageManager.getRequestUtilizations();
    List<SingularityTaskId> activeTaskIds = taskManager.getActiveTaskIds();
    final Map<String, SingularitySlaveUsageWithCalculatedScores> currentSlaveUsagesBySlaveId = usageManager.getCurrentSlaveUsages(offerHolders.values().stream().map(SingularityOfferHolder::getSlaveId).collect(Collectors.toList())).parallelStream().collect(Collectors.toMap(SingularitySlaveUsageWithId::getSlaveId, (usageWithId) -> new SingularitySlaveUsageWithCalculatedScores(usageWithId, configuration.getMesosConfiguration().getScoringStrategy(), configuration.getMesosConfiguration().getScoreUsingSystemLoad(), getMaxProbableUsageForSlave(activeTaskIds, requestUtilizations, offerHolders.get(usageWithId.getSlaveId()).getSanitizedHost()))));
    LOG.trace("Found slave usages {}", currentSlaveUsagesBySlaveId);
    Map<String, Integer> tasksPerOfferHost = new ConcurrentHashMap<>();
    for (SingularityTaskRequestHolder taskRequestHolder : sortedTaskRequestHolders) {
        lock.runWithRequestLock(() -> {
            Map<String, Double> scorePerOffer = new ConcurrentHashMap<>();
            List<SingularityTaskId> activeTaskIdsForRequest = leaderCache.getActiveTaskIdsForRequest(taskRequestHolder.getTaskRequest().getRequest().getId());
            List<CompletableFuture<Void>> scoringFutures = new ArrayList<>();
            AtomicReference<Throwable> scoringException = new AtomicReference<>(null);
            for (SingularityOfferHolder offerHolder : offerHolders.values()) {
                if (!isOfferFull(offerHolder)) {
                    scoringFutures.add(offerScoringSemaphore.call(() -> CompletableFuture.runAsync(() -> {
                        try {
                            double score = calculateScore(offerHolder, currentSlaveUsagesBySlaveId, tasksPerOfferHost, taskRequestHolder, activeTaskIdsForRequest);
                            if (score != 0) {
                                scorePerOffer.put(offerHolder.getSlaveId(), score);
                            }
                        } catch (Throwable t) {
                            LOG.error("Uncaught exception while scoring offers", t);
                            scoringException.set(t);
                        }
                    }, offerScoringExecutor)));
                }
            }
            CompletableFutures.allOf(scoringFutures).join();
            if (scoringException.get() != null) {
                LOG.warn("Exception caught in offer scoring futures, semaphore info: (concurrentRequests: {}, queueSize: {})", offerScoringSemaphore.getConcurrentRequests(), offerScoringSemaphore.getQueueSize());
                // This will be caught by either the LeaderOnlyPoller or resourceOffers uncaught exception code, causing an abort
                throw new RuntimeException(scoringException.get());
            }
            if (!scorePerOffer.isEmpty()) {
                SingularityOfferHolder bestOffer = offerHolders.get(Collections.max(scorePerOffer.entrySet(), Map.Entry.comparingByValue()).getKey());
                LOG.info("Best offer {}/1 is on {}", scorePerOffer.get(bestOffer.getSlaveId()), bestOffer.getSanitizedHost());
                SingularityMesosTaskHolder taskHolder = acceptTask(bestOffer, tasksPerOfferHost, taskRequestHolder);
                tasksScheduled.getAndIncrement();
                bestOffer.addMatchedTask(taskHolder);
                updateSlaveUsageScores(taskRequestHolder, currentSlaveUsagesBySlaveId, bestOffer.getSlaveId(), requestUtilizations);
            }
        }, taskRequestHolder.getTaskRequest().getRequest().getId(), String.format("%s#%s", getClass().getSimpleName(), "checkOffers"));
    }
    LOG.info("{} tasks scheduled, {} tasks remaining after examining {} offers", tasksScheduled, numDueTasks - tasksScheduled.get(), offers.size());
    return offerHolders.values();
}

Example 3 with RequestUtilization

use of com.hubspot.singularity.RequestUtilization in project Singularity by HubSpot.

the class SingularityMesosOfferScheduler method updateSlaveUsageScores.

private void updateSlaveUsageScores(SingularityTaskRequestHolder taskHolder, Map<String, SingularitySlaveUsageWithCalculatedScores> currentSlaveUsagesBySlaveId, String slaveId, Map<String, RequestUtilization> requestUtilizations) {
    Optional<SingularitySlaveUsageWithCalculatedScores> maybeUsage = Optional.fromNullable(currentSlaveUsagesBySlaveId.get(slaveId));
    if (maybeUsage.isPresent() && !maybeUsage.get().isMissingUsageData()) {
        SingularitySlaveUsageWithCalculatedScores usage = maybeUsage.get();
        usage.addEstimatedCpuReserved(taskHolder.getTotalResources().getCpus());
        usage.addEstimatedMemoryReserved(taskHolder.getTotalResources().getMemoryMb());
        usage.addEstimatedDiskReserved(taskHolder.getTotalResources().getDiskMb());
        if (requestUtilizations.containsKey(taskHolder.getTaskRequest().getRequest().getId())) {
            RequestUtilization requestUtilization = requestUtilizations.get(taskHolder.getTaskRequest().getRequest().getId());
            usage.addEstimatedCpuUsage(requestUtilization.getMaxCpuUsed());
            usage.addEstimatedMemoryBytesUsage(requestUtilization.getMaxMemBytesUsed());
            usage.addEstimatedDiskBytesUsage(requestUtilization.getMaxDiskBytesUsed());
        }
        usage.setScores(configuration.getMesosConfiguration().getScoringStrategy());
    }
}

Example 4 with RequestUtilization

use of com.hubspot.singularity.RequestUtilization in project Singularity by HubSpot.

the class SingularityUsagePoller method updateRequestUtilization.

private void updateRequestUtilization(Map<String, RequestUtilization> utilizationPerRequestId, List<SingularityTaskUsage> pastTaskUsages, SingularityTaskUsage latestUsage, SingularityTaskId task, double memoryMbReservedForTask, double cpuReservedForTask, double diskMbReservedForTask) {
    String requestId = task.getRequestId();
    RequestUtilization requestUtilization = utilizationPerRequestId.getOrDefault(requestId, new RequestUtilization(requestId, task.getDeployId()));
    long curMaxMemBytesUsed = 0;
    long curMinMemBytesUsed = Long.MAX_VALUE;
    double curMaxCpuUsed = 0;
    double curMinCpuUsed = Double.MAX_VALUE;
    long curMaxDiskBytesUsed = 0;
    long curMinDiskBytesUsed = Long.MAX_VALUE;
    if (utilizationPerRequestId.containsKey(requestId)) {
        curMaxMemBytesUsed = requestUtilization.getMaxMemBytesUsed();
        curMinMemBytesUsed = requestUtilization.getMinMemBytesUsed();
        curMaxCpuUsed = requestUtilization.getMaxCpuUsed();
        curMinCpuUsed = requestUtilization.getMinCpuUsed();
        curMaxDiskBytesUsed = requestUtilization.getMaxDiskBytesUsed();
        curMinDiskBytesUsed = requestUtilization.getMinDiskBytesUsed();
    }
    List<SingularityTaskUsage> pastTaskUsagesCopy = copyUsages(pastTaskUsages, latestUsage, task);
    pastTaskUsagesCopy.sort(Comparator.comparingDouble(SingularityTaskUsage::getTimestamp));
    int numTasks = pastTaskUsagesCopy.size() - 1;
    int numCpuOverages = 0;
    for (int i = 0; i < numTasks; i++) {
        SingularityTaskUsage olderUsage = pastTaskUsagesCopy.get(i);
        SingularityTaskUsage newerUsage = pastTaskUsagesCopy.get(i + 1);
        double cpusUsed = (newerUsage.getCpuSeconds() - olderUsage.getCpuSeconds()) / (newerUsage.getTimestamp() - olderUsage.getTimestamp());
        curMaxCpuUsed = Math.max(cpusUsed, curMaxCpuUsed);
        curMinCpuUsed = Math.min(cpusUsed, curMinCpuUsed);
        curMaxMemBytesUsed = Math.max(newerUsage.getMemoryTotalBytes(), curMaxMemBytesUsed);
        curMinMemBytesUsed = Math.min(newerUsage.getMemoryTotalBytes(), curMinMemBytesUsed);
        curMaxDiskBytesUsed = Math.max(newerUsage.getDiskTotalBytes(), curMaxDiskBytesUsed);
        curMinDiskBytesUsed = Math.min(newerUsage.getDiskTotalBytes(), curMinDiskBytesUsed);
        if (cpusUsed > cpuReservedForTask) {
            numCpuOverages++;
        }
        requestUtilization.addCpuUsed(cpusUsed).addMemBytesUsed(newerUsage.getMemoryTotalBytes()).addDiskBytesUsed(newerUsage.getDiskTotalBytes()).incrementTaskCount();
    }
    double cpuBurstRating = pastTaskUsagesCopy.size() > 0 ? numCpuOverages / (double) pastTaskUsagesCopy.size() : 1;
    requestUtilization.addMemBytesReserved((long) (memoryMbReservedForTask * SingularitySlaveUsage.BYTES_PER_MEGABYTE * numTasks)).addCpuReserved(cpuReservedForTask * numTasks).addDiskBytesReserved((long) diskMbReservedForTask * SingularitySlaveUsage.BYTES_PER_MEGABYTE * numTasks).setMaxCpuUsed(curMaxCpuUsed).setMinCpuUsed(curMinCpuUsed).setMaxMemBytesUsed(curMaxMemBytesUsed).setMinMemBytesUsed(curMinMemBytesUsed).setMaxDiskBytesUsed(curMaxDiskBytesUsed).setMinDiskBytesUsed(curMinDiskBytesUsed).setCpuBurstRating(cpuBurstRating);
    utilizationPerRequestId.put(requestId, requestUtilization);
}

Example 5 with RequestUtilization

use of com.hubspot.singularity.RequestUtilization in project Singularity by HubSpot.

the class SingularityUsagePoller method getClusterUtilization.

private SingularityClusterUtilization getClusterUtilization(Map<String, RequestUtilization> utilizationPerRequestId, long totalMemBytesUsed, long totalMemBytesAvailable, double totalCpuUsed, double totalCpuAvailable, long totalDiskBytesUsed, long totalDiskBytesAvailable, long now) {
    int numRequestsWithUnderUtilizedCpu = 0;
    int numRequestsWithOverUtilizedCpu = 0;
    int numRequestsWithUnderUtilizedMemBytes = 0;
    int numRequestsWithUnderUtilizedDiskBytes = 0;
    double totalUnderUtilizedCpu = 0;
    double totalOverUtilizedCpu = 0;
    long totalUnderUtilizedMemBytes = 0;
    long totalUnderUtilizedDiskBytes = 0;
    double maxUnderUtilizedCpu = 0;
    double maxOverUtilizedCpu = 0;
    long maxUnderUtilizedMemBytes = 0;
    long maxUnderUtilizedDiskBytes = 0;
    String maxUnderUtilizedCpuRequestId = null;
    String maxOverUtilizedCpuRequestId = null;
    String maxUnderUtilizedMemBytesRequestId = null;
    String maxUnderUtilizedDiskBytesRequestId = null;
    double minUnderUtilizedCpu = Double.MAX_VALUE;
    double minOverUtilizedCpu = Double.MAX_VALUE;
    long minUnderUtilizedMemBytes = Long.MAX_VALUE;
    long minUnderUtilizedDiskBytes = Long.MAX_VALUE;
    for (RequestUtilization utilization : utilizationPerRequestId.values()) {
        Optional<SingularityDeploy> maybeDeploy = deployManager.getDeploy(utilization.getRequestId(), utilization.getDeployId());
        if (maybeDeploy.isPresent() && maybeDeploy.get().getResources().isPresent()) {
            String requestId = utilization.getRequestId();
            long memoryBytesReserved = (long) (maybeDeploy.get().getResources().get().getMemoryMb() * SingularitySlaveUsage.BYTES_PER_MEGABYTE);
            double cpuReserved = maybeDeploy.get().getResources().get().getCpus();
            long diskBytesReserved = (long) maybeDeploy.get().getResources().get().getDiskMb() * SingularitySlaveUsage.BYTES_PER_MEGABYTE;
            double unusedCpu = cpuReserved - utilization.getAvgCpuUsed();
            long unusedMemBytes = (long) (memoryBytesReserved - utilization.getAvgMemBytesUsed());
            long unusedDiskBytes = (long) (diskBytesReserved - utilization.getAvgDiskBytesUsed());
            if (unusedCpu > 0) {
                numRequestsWithUnderUtilizedCpu++;
                totalUnderUtilizedCpu += unusedCpu;
                if (unusedCpu > maxUnderUtilizedCpu) {
                    maxUnderUtilizedCpu = unusedCpu;
                    maxUnderUtilizedCpuRequestId = requestId;
                }
                minUnderUtilizedCpu = Math.min(unusedCpu, minUnderUtilizedCpu);
            } else if (unusedCpu < 0) {
                double overusedCpu = Math.abs(unusedCpu);
                numRequestsWithOverUtilizedCpu++;
                totalOverUtilizedCpu += overusedCpu;
                if (overusedCpu > maxOverUtilizedCpu) {
                    maxOverUtilizedCpu = overusedCpu;
                    maxOverUtilizedCpuRequestId = requestId;
                }
                minOverUtilizedCpu = Math.min(overusedCpu, minOverUtilizedCpu);
            }
            if (unusedMemBytes > 0) {
                numRequestsWithUnderUtilizedMemBytes++;
                totalUnderUtilizedMemBytes += unusedMemBytes;
                if (unusedMemBytes > maxUnderUtilizedMemBytes) {
                    maxUnderUtilizedMemBytes = unusedMemBytes;
                    maxUnderUtilizedMemBytesRequestId = requestId;
                }
                minUnderUtilizedMemBytes = Math.min(unusedMemBytes, minUnderUtilizedMemBytes);
            }
            if (unusedDiskBytes > 0) {
                numRequestsWithUnderUtilizedDiskBytes++;
                totalUnderUtilizedDiskBytes += unusedDiskBytes;
                if (unusedDiskBytes > maxUnderUtilizedDiskBytes) {
                    maxUnderUtilizedDiskBytes = unusedDiskBytes;
                    maxUnderUtilizedDiskBytesRequestId = requestId;
                }
                minUnderUtilizedDiskBytes = Math.min(unusedDiskBytes, minUnderUtilizedMemBytes);
            }
        }
    }
    double avgUnderUtilizedCpu = numRequestsWithUnderUtilizedCpu != 0 ? totalUnderUtilizedCpu / numRequestsWithUnderUtilizedCpu : 0;
    double avgOverUtilizedCpu = numRequestsWithOverUtilizedCpu != 0 ? totalOverUtilizedCpu / numRequestsWithOverUtilizedCpu : 0;
    long avgUnderUtilizedMemBytes = numRequestsWithUnderUtilizedMemBytes != 0 ? totalUnderUtilizedMemBytes / numRequestsWithUnderUtilizedMemBytes : 0;
    long avgUnderUtilizedDiskBytes = numRequestsWithUnderUtilizedDiskBytes != 0 ? totalUnderUtilizedDiskBytes / numRequestsWithUnderUtilizedDiskBytes : 0;
    return new SingularityClusterUtilization(new ArrayList<>(utilizationPerRequestId.values()), numRequestsWithUnderUtilizedCpu, numRequestsWithOverUtilizedCpu, numRequestsWithUnderUtilizedMemBytes, numRequestsWithUnderUtilizedDiskBytes, totalUnderUtilizedCpu, totalOverUtilizedCpu, totalUnderUtilizedMemBytes, totalUnderUtilizedDiskBytes, avgUnderUtilizedCpu, avgOverUtilizedCpu, avgUnderUtilizedMemBytes, avgUnderUtilizedDiskBytes, maxUnderUtilizedCpu, maxOverUtilizedCpu, maxUnderUtilizedMemBytes, maxUnderUtilizedDiskBytes, maxUnderUtilizedCpuRequestId, maxOverUtilizedCpuRequestId, maxUnderUtilizedMemBytesRequestId, maxUnderUtilizedDiskBytesRequestId, getMin(minUnderUtilizedCpu), getMin(minOverUtilizedCpu), getMin(minUnderUtilizedMemBytes), getMin(minUnderUtilizedDiskBytes), totalMemBytesUsed, totalMemBytesAvailable, totalDiskBytesUsed, totalDiskBytesAvailable, totalCpuUsed, totalCpuAvailable, now);
}