Search in sources :

Example 1 with SingularityPendingTaskId

use of com.hubspot.singularity.SingularityPendingTaskId in project Singularity by HubSpot.

the class SingularityWebCache method cachePendingTasks.

public void cachePendingTasks(List<SingularityPendingTask> pendingTasks) {
    pendingMissMeter.mark();
    Map<SingularityPendingTaskId, SingularityPendingTask> newPendingTasks = new HashMap<>(pendingTasks.size());
    for (SingularityPendingTask pendingTask : pendingTasks) {
        newPendingTasks.put(pendingTask.getPendingTaskId(), pendingTask);
    }
    cachedPendingTasks = newPendingTasks;
    lastPendingTaskCache = System.currentTimeMillis();
}
Also used : HashMap(java.util.HashMap) SingularityPendingTask(com.hubspot.singularity.SingularityPendingTask) SingularityPendingTaskId(com.hubspot.singularity.SingularityPendingTaskId)

Example 2 with SingularityPendingTaskId

use of com.hubspot.singularity.SingularityPendingTaskId in project Singularity by HubSpot.

the class StateManager method getNumTasks.

private Map<String, Long> getNumTasks(List<SingularityRequestWithState> requests) {
    final CounterMap<String> numTasks = new CounterMap<>(requests.size());
    for (SingularityTaskId taskId : taskManager.getActiveTaskIds()) {
        numTasks.incr(taskId.getRequestId());
    }
    for (SingularityPendingTaskId pendingTaskId : taskManager.getPendingTaskIds()) {
        numTasks.incr(pendingTaskId.getRequestId());
    }
    for (SingularityTaskId cleaningTaskId : taskManager.getCleanupTaskIds()) {
        Optional<SingularityRequestWithState> request = requestManager.getRequest(cleaningTaskId.getRequestId());
        if (request.isPresent() && request.get().getRequest().isScheduled()) {
            continue;
        }
        numTasks.decr(cleaningTaskId.getRequestId());
    }
    return numTasks.toCountMap();
}
Also used : CounterMap(com.hubspot.mesos.CounterMap) SingularityPendingTaskId(com.hubspot.singularity.SingularityPendingTaskId) SingularityRequestWithState(com.hubspot.singularity.SingularityRequestWithState) SingularityTaskId(com.hubspot.singularity.SingularityTaskId)

Example 3 with SingularityPendingTaskId

use of com.hubspot.singularity.SingularityPendingTaskId in project Singularity by HubSpot.

the class SingularityMesosOfferScheduler method checkOffers.

public Collection<SingularityOfferHolder> checkOffers(final Collection<Offer> offers) {
    for (SingularityPendingTaskId taskId : taskManager.getPendingTasksMarkedForDeletion()) {
        lock.runWithRequestLock(() -> taskManager.deletePendingTask(taskId), taskId.getRequestId(), String.format("%s#%s", getClass().getSimpleName(), "checkOffers -> pendingTaskDeletes"));
    }
    scheduler.checkForDecomissions();
    scheduler.drainPendingQueue();
    if (offers.isEmpty()) {
        LOG.debug("No offers to check");
        return Collections.emptyList();
    }
    final List<SingularityTaskRequestHolder> sortedTaskRequestHolders = getSortedDueTaskRequests();
    final int numDueTasks = sortedTaskRequestHolders.size();
    final Map<String, SingularityOfferHolder> offerHolders = offers.stream().collect(Collectors.groupingBy((o) -> o.getAgentId().getValue())).entrySet().stream().filter((e) -> e.getValue().size() > 0).map((e) -> {
        List<Offer> offersList = e.getValue();
        String slaveId = e.getKey();
        return new SingularityOfferHolder(offersList, numDueTasks, slaveAndRackHelper.getRackIdOrDefault(offersList.get(0)), slaveId, offersList.get(0).getHostname(), slaveAndRackHelper.getTextAttributes(offersList.get(0)), slaveAndRackHelper.getReservedSlaveAttributes(offersList.get(0)));
    }).collect(Collectors.toMap(SingularityOfferHolder::getSlaveId, Function.identity()));
    if (sortedTaskRequestHolders.isEmpty()) {
        return offerHolders.values();
    }
    final AtomicInteger tasksScheduled = new AtomicInteger(0);
    Map<String, RequestUtilization> requestUtilizations = usageManager.getRequestUtilizations();
    List<SingularityTaskId> activeTaskIds = taskManager.getActiveTaskIds();
    final Map<String, SingularitySlaveUsageWithCalculatedScores> currentSlaveUsagesBySlaveId = usageManager.getCurrentSlaveUsages(offerHolders.values().stream().map(SingularityOfferHolder::getSlaveId).collect(Collectors.toList())).parallelStream().collect(Collectors.toMap(SingularitySlaveUsageWithId::getSlaveId, (usageWithId) -> new SingularitySlaveUsageWithCalculatedScores(usageWithId, configuration.getMesosConfiguration().getScoringStrategy(), configuration.getMesosConfiguration().getScoreUsingSystemLoad(), getMaxProbableUsageForSlave(activeTaskIds, requestUtilizations, offerHolders.get(usageWithId.getSlaveId()).getSanitizedHost()))));
    LOG.trace("Found slave usages {}", currentSlaveUsagesBySlaveId);
    Map<String, Integer> tasksPerOfferHost = new ConcurrentHashMap<>();
    for (SingularityTaskRequestHolder taskRequestHolder : sortedTaskRequestHolders) {
        lock.runWithRequestLock(() -> {
            Map<String, Double> scorePerOffer = new ConcurrentHashMap<>();
            List<SingularityTaskId> activeTaskIdsForRequest = leaderCache.getActiveTaskIdsForRequest(taskRequestHolder.getTaskRequest().getRequest().getId());
            List<CompletableFuture<Void>> scoringFutures = new ArrayList<>();
            AtomicReference<Throwable> scoringException = new AtomicReference<>(null);
            for (SingularityOfferHolder offerHolder : offerHolders.values()) {
                if (!isOfferFull(offerHolder)) {
                    scoringFutures.add(offerScoringSemaphore.call(() -> CompletableFuture.runAsync(() -> {
                        try {
                            double score = calculateScore(offerHolder, currentSlaveUsagesBySlaveId, tasksPerOfferHost, taskRequestHolder, activeTaskIdsForRequest);
                            if (score != 0) {
                                scorePerOffer.put(offerHolder.getSlaveId(), score);
                            }
                        } catch (Throwable t) {
                            LOG.error("Uncaught exception while scoring offers", t);
                            scoringException.set(t);
                        }
                    }, offerScoringExecutor)));
                }
            }
            CompletableFutures.allOf(scoringFutures).join();
            if (scoringException.get() != null) {
                LOG.warn("Exception caught in offer scoring futures, semaphore info: (concurrentRequests: {}, queueSize: {})", offerScoringSemaphore.getConcurrentRequests(), offerScoringSemaphore.getQueueSize());
                // This will be caught by either the LeaderOnlyPoller or resourceOffers uncaught exception code, causing an abort
                throw new RuntimeException(scoringException.get());
            }
            if (!scorePerOffer.isEmpty()) {
                SingularityOfferHolder bestOffer = offerHolders.get(Collections.max(scorePerOffer.entrySet(), Map.Entry.comparingByValue()).getKey());
                LOG.info("Best offer {}/1 is on {}", scorePerOffer.get(bestOffer.getSlaveId()), bestOffer.getSanitizedHost());
                SingularityMesosTaskHolder taskHolder = acceptTask(bestOffer, tasksPerOfferHost, taskRequestHolder);
                tasksScheduled.getAndIncrement();
                bestOffer.addMatchedTask(taskHolder);
                updateSlaveUsageScores(taskRequestHolder, currentSlaveUsagesBySlaveId, bestOffer.getSlaveId(), requestUtilizations);
            }
        }, taskRequestHolder.getTaskRequest().getRequest().getId(), String.format("%s#%s", getClass().getSimpleName(), "checkOffers"));
    }
    LOG.info("{} tasks scheduled, {} tasks remaining after examining {} offers", tasksScheduled, numDueTasks - tasksScheduled.get(), offers.size());
    return offerHolders.values();
}
Also used : ThreadFactoryBuilder(com.google.common.util.concurrent.ThreadFactoryBuilder) SingularityTask(com.hubspot.singularity.SingularityTask) DeployManager(com.hubspot.singularity.data.DeployManager) Inject(com.google.inject.Inject) LoggerFactory(org.slf4j.LoggerFactory) CompletableFuture(java.util.concurrent.CompletableFuture) Offer(org.apache.mesos.v1.Protos.Offer) SingularityScheduler(com.hubspot.singularity.scheduler.SingularityScheduler) Singleton(javax.inject.Singleton) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) ArrayList(java.util.ArrayList) SingularityDeployStatistics(com.hubspot.singularity.SingularityDeployStatistics) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Optional(com.google.common.base.Optional) Map(java.util.Map) SingularitySlaveUsage(com.hubspot.singularity.SingularitySlaveUsage) Resources(com.hubspot.mesos.Resources) RequestType(com.hubspot.singularity.RequestType) TaskManager(com.hubspot.singularity.data.TaskManager) ExecutorService(java.util.concurrent.ExecutorService) SingularityPendingTaskId(com.hubspot.singularity.SingularityPendingTaskId) SingularityTaskId(com.hubspot.singularity.SingularityTaskId) SingularityConfiguration(com.hubspot.singularity.config.SingularityConfiguration) ResourceUsageType(com.hubspot.singularity.SingularitySlaveUsage.ResourceUsageType) Logger(org.slf4j.Logger) MaxProbableUsage(com.hubspot.singularity.mesos.SingularitySlaveUsageWithCalculatedScores.MaxProbableUsage) CustomExecutorConfiguration(com.hubspot.singularity.config.CustomExecutorConfiguration) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) CompletableFutures(com.hubspot.singularity.async.CompletableFutures) AsyncSemaphore(com.hubspot.singularity.async.AsyncSemaphore) Collectors(java.util.stream.Collectors) SlaveMatchState(com.hubspot.singularity.SlaveMatchState) Executors(java.util.concurrent.Executors) SingularityMesosTaskHolder(com.hubspot.singularity.helpers.SingularityMesosTaskHolder) TimeUnit(java.util.concurrent.TimeUnit) UsageManager(com.hubspot.singularity.data.UsageManager) SingularityTaskRequest(com.hubspot.singularity.SingularityTaskRequest) List(java.util.List) MesosUtils(com.hubspot.singularity.helpers.MesosUtils) RequestUtilization(com.hubspot.singularity.RequestUtilization) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Collections(java.util.Collections) SingularitySlaveUsageWithId(com.hubspot.singularity.SingularitySlaveUsageWithId) MesosConfiguration(com.hubspot.singularity.config.MesosConfiguration) SingularityLeaderCache(com.hubspot.singularity.scheduler.SingularityLeaderCache) RequestUtilization(com.hubspot.singularity.RequestUtilization) ArrayList(java.util.ArrayList) CompletableFuture(java.util.concurrent.CompletableFuture) ArrayList(java.util.ArrayList) List(java.util.List) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) SingularityTaskId(com.hubspot.singularity.SingularityTaskId) SingularityPendingTaskId(com.hubspot.singularity.SingularityPendingTaskId) SingularityMesosTaskHolder(com.hubspot.singularity.helpers.SingularityMesosTaskHolder) AtomicReference(java.util.concurrent.atomic.AtomicReference) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) AtomicInteger(java.util.concurrent.atomic.AtomicInteger)

Example 4 with SingularityPendingTaskId

use of com.hubspot.singularity.SingularityPendingTaskId in project Singularity by HubSpot.

the class SingularityCmdLineArgsMigration method checkPendingTasks.

private void checkPendingTasks() {
    try {
        if (curator.checkExists().forPath(TASK_PENDING_PATH) == null) {
            return;
        }
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
    try {
        for (SingularityPendingTaskId pendingTaskId : taskManager.getPendingTaskIds()) {
            Optional<String> cmdLineArgs = getCmdLineArgs(pendingTaskId);
            SingularityCreateResult result = taskManager.savePendingTask(new SingularityPendingTaskBuilder().setPendingTaskId(pendingTaskId).setCmdLineArgsList(getCmdLineArgs(cmdLineArgs)).build());
            LOG.info("Saving {} ({}) {}", pendingTaskId, cmdLineArgs, result);
        }
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}
Also used : SingularityPendingTaskId(com.hubspot.singularity.SingularityPendingTaskId) SingularityCreateResult(com.hubspot.singularity.SingularityCreateResult) SingularityPendingTaskBuilder(com.hubspot.singularity.SingularityPendingTaskBuilder)

Example 5 with SingularityPendingTaskId

use of com.hubspot.singularity.SingularityPendingTaskId in project Singularity by HubSpot.

the class SingularityPendingTaskIdMigration method applyMigration.

@Override
public void applyMigration() {
    final long start = System.currentTimeMillis();
    try {
        if (curator.checkExists().forPath(PENDING_TASKS_ROOT) == null) {
            return;
        }
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
    try {
        for (String pendingTaskId : curator.getChildren().forPath(PENDING_TASKS_ROOT)) {
            SingularityPendingTaskId newPendingTaskId = createFrom(pendingTaskId, start);
            if (!newPendingTaskId.toString().equals(pendingTaskId)) {
                LOG.info("Migrating {} to {}", pendingTaskId, newPendingTaskId);
                Optional<String> cmdLineArgs = getCmdLineArgs(pendingTaskId);
                taskManager.savePendingTask(new SingularityPendingTaskBuilder().setPendingTaskId(newPendingTaskId).setCmdLineArgsList(cmdLineArgs.isPresent() ? Optional.of(Collections.singletonList(cmdLineArgs.get())) : Optional.<List<String>>absent()).build());
                curator.delete().forPath(ZKPaths.makePath(PENDING_TASKS_ROOT, pendingTaskId));
            }
        }
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}
Also used : SingularityPendingTaskId(com.hubspot.singularity.SingularityPendingTaskId) List(java.util.List) InvalidSingularityTaskIdException(com.hubspot.singularity.InvalidSingularityTaskIdException) SingularityPendingTaskBuilder(com.hubspot.singularity.SingularityPendingTaskBuilder)

Aggregations

SingularityPendingTaskId (com.hubspot.singularity.SingularityPendingTaskId)36 SingularityRequest (com.hubspot.singularity.SingularityRequest)16 Test (org.junit.Test)16 SingularityTaskId (com.hubspot.singularity.SingularityTaskId)15 SingularityRequestBuilder (com.hubspot.singularity.SingularityRequestBuilder)9 SingularityPendingTask (com.hubspot.singularity.SingularityPendingTask)8 SingularityRunNowRequest (com.hubspot.singularity.api.SingularityRunNowRequest)8 SingularityPendingRequest (com.hubspot.singularity.SingularityPendingRequest)7 SingularityPendingTaskBuilder (com.hubspot.singularity.SingularityPendingTaskBuilder)7 SingularityDeploy (com.hubspot.singularity.SingularityDeploy)6 SingularityTask (com.hubspot.singularity.SingularityTask)6 SingularityRequestWithState (com.hubspot.singularity.SingularityRequestWithState)5 SingularityDeployKey (com.hubspot.singularity.SingularityDeployKey)4 PendingType (com.hubspot.singularity.SingularityPendingRequest.PendingType)4 SingularityRequestDeployState (com.hubspot.singularity.SingularityRequestDeployState)4 Optional (com.google.common.base.Optional)3 Inject (com.google.inject.Inject)3 RequestType (com.hubspot.singularity.RequestType)3 SingularityDeployBuilder (com.hubspot.singularity.SingularityDeployBuilder)3 ArrayList (java.util.ArrayList)3