Examples with SingularityTaskRequest com.hubspot.singularity.SingularityTaskRequest used on opensource projects

Example 16 with SingularityTaskRequest

use of com.hubspot.singularity.SingularityTaskRequest in project Singularity by HubSpot.

the class TaskRequestManager method getTaskRequests.

public List<SingularityTaskRequest> getTaskRequests(List<SingularityPendingTask> tasks) {
    final Multimap<String, SingularityPendingTask> requestIdToPendingTaskId = ArrayListMultimap.create(tasks.size(), 1);
    for (SingularityPendingTask task : tasks) {
        requestIdToPendingTaskId.put(task.getPendingTaskId().getRequestId(), task);
    }
    final List<SingularityRequestWithState> matchingRequests = requestManager.getRequests(requestIdToPendingTaskId.keySet());
    final Map<SingularityPendingTask, SingularityDeployKey> deployKeys = SingularityDeployKey.fromPendingTasks(requestIdToPendingTaskId.values());
    final Map<SingularityDeployKey, SingularityDeploy> matchingDeploys = deployManager.getDeploysForKeys(Sets.newHashSet(deployKeys.values()));
    final List<SingularityTaskRequest> taskRequests = Lists.newArrayListWithCapacity(matchingRequests.size());
    for (SingularityRequestWithState request : matchingRequests) {
        Optional<SingularityPendingDeploy> maybePendingDeploy = deployManager.getPendingDeploy(request.getRequest().getId());
        for (SingularityPendingTask task : requestIdToPendingTaskId.get(request.getRequest().getId())) {
            SingularityDeploy foundDeploy = matchingDeploys.get(deployKeys.get(task));
            if (foundDeploy == null) {
                LOG.warn("Couldn't find a matching deploy for pending task {}", task);
                continue;
            }
            if (!request.getState().isRunnable()) {
                LOG.warn("Request was in state {} for pending task {}", request.getState(), task);
                continue;
            }
            Optional<SingularityRequest> updatedRequest = maybePendingDeploy.isPresent() && maybePendingDeploy.get().getDeployMarker().getDeployId().equals(task.getPendingTaskId().getDeployId()) ? maybePendingDeploy.get().getUpdatedRequest() : Optional.<SingularityRequest>absent();
            taskRequests.add(new SingularityTaskRequest(updatedRequest.or(request.getRequest()), foundDeploy, task));
        }
    }
    return taskRequests;
}

Example 17 with SingularityTaskRequest

use of com.hubspot.singularity.SingularityTaskRequest in project Singularity by HubSpot.

the class SingularityMesosTaskPrioritizer method getSortedDueTasks.

public List<SingularityTaskRequest> getSortedDueTasks(List<SingularityTaskRequest> dueTasks) {
    long now = System.currentTimeMillis();
    List<SingularityTaskRequestWithPriority> taskRequestWithPriorities = new ArrayList<>();
    for (SingularityTaskRequest taskRequest : dueTasks) {
        taskRequestWithPriorities.add(new SingularityTaskRequestWithPriority(taskRequest, getWeightedPriority(taskRequest, now)));
    }
    Collections.sort(taskRequestWithPriorities, SingularityTaskRequestWithPriority.weightedPriorityComparator());
    List<SingularityTaskRequest> taskRequests = new ArrayList<>();
    for (SingularityTaskRequestWithPriority taskRequestWithPriority : taskRequestWithPriorities) {
        taskRequests.add(taskRequestWithPriority.getTaskRequest());
    }
    return taskRequests;
}

Example 18 with SingularityTaskRequest

use of com.hubspot.singularity.SingularityTaskRequest in project Singularity by HubSpot.

the class SingularityTaskSizeOptimizer method getSizeOptimizedTask.

SingularityTask getSizeOptimizedTask(SingularityMesosTaskHolder taskHolder) {
    if (configuration.isStoreAllMesosTaskInfoForDebugging()) {
        return taskHolder.getTask();
    }
    SingularityTask task = taskHolder.getTask();
    TaskInfo.Builder mesosTask = taskHolder.getMesosTask().toBuilder();
    mesosTask.clearData();
    List<MesosOfferObject> offers = task.getOffers().stream().map(MesosOfferObject::sizeOptimized).collect(Collectors.toList());
    SingularityTaskRequest taskRequest = task.getTaskRequest();
    if (task.getTaskRequest().getDeploy().getExecutorData().isPresent()) {
        SingularityDeployBuilder deploy = task.getTaskRequest().getDeploy().toBuilder();
        deploy.setExecutorData(Optional.absent());
        taskRequest = new SingularityTaskRequest(task.getTaskRequest().getRequest(), deploy.build(), task.getTaskRequest().getPendingTask());
    }
    return new SingularityTask(taskRequest, task.getTaskId(), offers, mesosProtosUtils.taskFromProtos(mesosTask.build()), task.getRackId());
}

Example 19 with SingularityTaskRequest

use of com.hubspot.singularity.SingularityTaskRequest in project Singularity by HubSpot.

the class SingularityMesosOfferScheduler method score.

private double score(SingularityOfferHolder offerHolder, Map<String, Integer> tasksPerOffer, SingularityTaskRequestHolder taskRequestHolder, Optional<SingularitySlaveUsageWithCalculatedScores> maybeSlaveUsage, List<SingularityTaskId> activeTaskIdsForRequest) {
    final SingularityTaskRequest taskRequest = taskRequestHolder.getTaskRequest();
    final SingularityPendingTaskId pendingTaskId = taskRequest.getPendingTask().getPendingTaskId();
    if (tooManyTasksPerOfferHostForRequest(tasksPerOffer, offerHolder.getHostname(), taskRequestHolder.getTaskRequest())) {
        LOG.debug("Skipping task request for request id {}, too many tasks already scheduled using offer {}", taskRequest.getRequest().getId(), offerHolder.getHostname());
        return 0;
    }
    if (isTooManyInstancesForRequest(taskRequest, activeTaskIdsForRequest)) {
        LOG.debug("Skipping pending task {}, too many instances already running", pendingTaskId);
        return 0;
    }
    if (LOG.isTraceEnabled()) {
        LOG.trace("Attempting to match task {} resources {} with required role '{}' ({} for task + {} for executor) with remaining offer resources {}", pendingTaskId, taskRequestHolder.getTotalResources(), taskRequest.getRequest().getRequiredRole().or("*"), taskRequestHolder.getTaskResources(), taskRequestHolder.getExecutorResources(), MesosUtils.formatForLogging(offerHolder.getCurrentResources()));
    }
    final boolean matchesResources = MesosUtils.doesOfferMatchResources(taskRequest.getRequest().getRequiredRole(), taskRequestHolder.getTotalResources(), offerHolder.getCurrentResources(), taskRequestHolder.getRequestedPorts());
    if (!matchesResources) {
        return 0;
    }
    final SlaveMatchState slaveMatchState = slaveAndRackManager.doesOfferMatch(offerHolder, taskRequest, activeTaskIdsForRequest);
    if (slaveMatchState.isMatchAllowed()) {
        return score(offerHolder.getHostname(), taskRequest, maybeSlaveUsage);
    } else if (LOG.isTraceEnabled()) {
        LOG.trace("Ignoring offer on host {} with roles {} on {} for task {}; matched resources: {}, slave match state: {}", offerHolder.getHostname(), offerHolder.getRoles(), offerHolder.getHostname(), pendingTaskId, matchesResources, slaveMatchState);
    }
    return 0;
}

Example 20 with SingularityTaskRequest

use of com.hubspot.singularity.SingularityTaskRequest in project Singularity by HubSpot.

the class SingularityMesosOfferScheduler method acceptTask.

private SingularityMesosTaskHolder acceptTask(SingularityOfferHolder offerHolder, Map<String, Integer> tasksPerOffer, SingularityTaskRequestHolder taskRequestHolder) {
    final SingularityTaskRequest taskRequest = taskRequestHolder.getTaskRequest();
    final SingularityMesosTaskHolder taskHolder = mesosTaskBuilder.buildTask(offerHolder, offerHolder.getCurrentResources(), taskRequest, taskRequestHolder.getTaskResources(), taskRequestHolder.getExecutorResources());
    final SingularityTask zkTask = taskSizeOptimizer.getSizeOptimizedTask(taskHolder);
    LOG.trace("Accepted and built task {}", zkTask);
    LOG.info("Launching task {} slot on slave {} ({})", taskHolder.getTask().getTaskId(), offerHolder.getSlaveId(), offerHolder.getHostname());
    taskManager.createTaskAndDeletePendingTask(zkTask);
    addRequestToMapByOfferHost(tasksPerOffer, offerHolder.getHostname(), taskRequest.getRequest().getId());
    return taskHolder;
}