Search in sources :

Example 6 with RequestUtilization

use of com.hubspot.singularity.RequestUtilization in project Singularity by HubSpot.

the class SingularityUsagePoller method runActionOnPoll.

@Override
public void runActionOnPoll() {
    Map<String, RequestUtilization> utilizationPerRequestId = new ConcurrentHashMap<>();
    final long now = System.currentTimeMillis();
    AtomicLong totalMemBytesUsed = new AtomicLong(0);
    AtomicLong totalMemBytesAvailable = new AtomicLong(0);
    AtomicDouble totalCpuUsed = new AtomicDouble(0.00);
    AtomicDouble totalCpuAvailable = new AtomicDouble(0.00);
    AtomicLong totalDiskBytesUsed = new AtomicLong(0);
    AtomicLong totalDiskBytesAvailable = new AtomicLong(0);
    Map<SingularitySlaveUsage, List<TaskIdWithUsage>> overLoadedHosts = new ConcurrentHashMap<>();
    List<CompletableFuture<Void>> usageFutures = new ArrayList<>();
    usageHelper.getSlavesToTrackUsageFor().forEach((slave) -> {
        usageFutures.add(usageCollectionSemaphore.call(() -> CompletableFuture.runAsync(() -> {
            collectSlaveUage(slave, now, utilizationPerRequestId, overLoadedHosts, totalMemBytesUsed, totalMemBytesAvailable, totalCpuUsed, totalCpuAvailable, totalDiskBytesUsed, totalDiskBytesAvailable);
        })));
    });
    CompletableFutures.allOf(usageFutures).join();
    usageManager.saveClusterUtilization(getClusterUtilization(utilizationPerRequestId, totalMemBytesUsed.get(), totalMemBytesAvailable.get(), totalCpuUsed.get(), totalCpuAvailable.get(), totalDiskBytesUsed.get(), totalDiskBytesAvailable.get(), now));
    if (configuration.isShuffleTasksForOverloadedSlaves()) {
        shuffleTasksOnOverloadedHosts(overLoadedHosts);
    }
}
Also used : AtomicDouble(com.google.common.util.concurrent.AtomicDouble) SingularitySlaveUsage(com.hubspot.singularity.SingularitySlaveUsage) RequestUtilization(com.hubspot.singularity.RequestUtilization) ArrayList(java.util.ArrayList) AtomicLong(java.util.concurrent.atomic.AtomicLong) CompletableFuture(java.util.concurrent.CompletableFuture) List(java.util.List) ArrayList(java.util.ArrayList) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap)

Aggregations

RequestUtilization (com.hubspot.singularity.RequestUtilization)6 Resources (com.hubspot.mesos.Resources)2 SingularitySlaveUsage (com.hubspot.singularity.SingularitySlaveUsage)2 SingularityTask (com.hubspot.singularity.SingularityTask)2 SingularityTaskId (com.hubspot.singularity.SingularityTaskId)2 MaxProbableUsage (com.hubspot.singularity.mesos.SingularitySlaveUsageWithCalculatedScores.MaxProbableUsage)2 ArrayList (java.util.ArrayList)2 List (java.util.List)2 CompletableFuture (java.util.concurrent.CompletableFuture)2 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)2 VisibleForTesting (com.google.common.annotations.VisibleForTesting)1 Optional (com.google.common.base.Optional)1 AtomicDouble (com.google.common.util.concurrent.AtomicDouble)1 ThreadFactoryBuilder (com.google.common.util.concurrent.ThreadFactoryBuilder)1 Inject (com.google.inject.Inject)1 RequestType (com.hubspot.singularity.RequestType)1 SingularityClusterUtilization (com.hubspot.singularity.SingularityClusterUtilization)1 SingularityDeploy (com.hubspot.singularity.SingularityDeploy)1 SingularityDeployStatistics (com.hubspot.singularity.SingularityDeployStatistics)1 SingularityPendingTaskId (com.hubspot.singularity.SingularityPendingTaskId)1