Examples with SingularityTask com.hubspot.singularity.SingularityTask used on opensource projects

Example 91 with SingularityTask

use of com.hubspot.singularity.SingularityTask in project Singularity by HubSpot.

the class SingularitySlaveAndRackManager method doesOfferMatch.

SlaveMatchState doesOfferMatch(SingularityOfferHolder offerHolder, SingularityTaskRequest taskRequest, List<SingularityTaskId> activeTaskIdsForRequest) {
    final String host = offerHolder.getHostname();
    final String rackId = offerHolder.getRackId();
    final String slaveId = offerHolder.getSlaveId();
    final MachineState currentSlaveState = slaveManager.getSlave(slaveId).get().getCurrentState().getState();
    if (currentSlaveState == MachineState.FROZEN) {
        return SlaveMatchState.SLAVE_FROZEN;
    }
    if (currentSlaveState.isDecommissioning()) {
        return SlaveMatchState.SLAVE_DECOMMISSIONING;
    }
    final MachineState currentRackState = rackManager.getRack(rackId).get().getCurrentState().getState();
    if (currentRackState == MachineState.FROZEN) {
        return SlaveMatchState.RACK_FROZEN;
    }
    if (currentRackState.isDecommissioning()) {
        return SlaveMatchState.RACK_DECOMMISSIONING;
    }
    if (!taskRequest.getRequest().getRackAffinity().or(Collections.emptyList()).isEmpty()) {
        if (!taskRequest.getRequest().getRackAffinity().get().contains(rackId)) {
            LOG.trace("Task {} requires a rack in {} (current rack {})", taskRequest.getPendingTask().getPendingTaskId(), taskRequest.getRequest().getRackAffinity().get(), rackId);
            return SlaveMatchState.RACK_AFFINITY_NOT_MATCHING;
        }
    }
    if (!isSlaveAttributesMatch(offerHolder, taskRequest)) {
        return SlaveMatchState.SLAVE_ATTRIBUTES_DO_NOT_MATCH;
    }
    final SlavePlacement slavePlacement = taskRequest.getRequest().getSlavePlacement().or(configuration.getDefaultSlavePlacement());
    if (!taskRequest.getRequest().isRackSensitive() && slavePlacement == SlavePlacement.GREEDY) {
        // todo: account for this or let this behavior continue?
        return SlaveMatchState.NOT_RACK_OR_SLAVE_PARTICULAR;
    }
    final int numDesiredInstances = taskRequest.getRequest().getInstancesSafe();
    boolean allowBounceToSameHost = isAllowBounceToSameHost(taskRequest.getRequest());
    Multiset<String> countPerRack = HashMultiset.create(slaveManager.getNumActive());
    double numOnSlave = 0;
    double numCleaningOnSlave = 0;
    double numFromSameBounceOnSlave = 0;
    double numOtherDeploysOnSlave = 0;
    boolean taskLaunchedFromBounceWithActionId = taskRequest.getPendingTask().getPendingTaskId().getPendingType() == PendingType.BOUNCE && taskRequest.getPendingTask().getActionId().isPresent();
    final String sanitizedHost = offerHolder.getSanitizedHost();
    final String sanitizedRackId = offerHolder.getSanitizedRackId();
    Collection<SingularityTaskId> cleaningTasks = leaderCache.getCleanupTaskIds();
    for (SingularityTaskId taskId : activeTaskIdsForRequest) {
        if (!cleaningTasks.contains(taskId) && taskRequest.getDeploy().getId().equals(taskId.getDeployId())) {
            countPerRack.add(taskId.getSanitizedRackId());
        }
        if (!taskId.getSanitizedHost().equals(sanitizedHost)) {
            continue;
        }
        if (taskRequest.getDeploy().getId().equals(taskId.getDeployId())) {
            if (cleaningTasks.contains(taskId)) {
                numCleaningOnSlave++;
            } else {
                numOnSlave++;
            }
            if (taskLaunchedFromBounceWithActionId) {
                Optional<SingularityTask> maybeTask = taskManager.getTask(taskId);
                boolean errorInTaskData = false;
                if (maybeTask.isPresent()) {
                    SingularityPendingTask pendingTask = maybeTask.get().getTaskRequest().getPendingTask();
                    if (pendingTask.getPendingTaskId().getPendingType() == PendingType.BOUNCE) {
                        if (pendingTask.getActionId().isPresent()) {
                            if (pendingTask.getActionId().get().equals(taskRequest.getPendingTask().getActionId().get())) {
                                numFromSameBounceOnSlave++;
                            }
                        } else {
                            // No actionId present on bounce, fall back to more restrictive placement strategy
                            errorInTaskData = true;
                        }
                    }
                } else {
                    // Could not find appropriate task data, fall back to more restrictive placement strategy
                    errorInTaskData = true;
                }
                if (errorInTaskData) {
                    allowBounceToSameHost = false;
                }
            }
        } else {
            numOtherDeploysOnSlave++;
        }
    }
    if (taskRequest.getRequest().isRackSensitive()) {
        final boolean isRackOk = isRackOk(countPerRack, sanitizedRackId, numDesiredInstances, taskRequest.getRequest().getId(), slaveId, host, numCleaningOnSlave, leaderCache);
        if (!isRackOk) {
            return SlaveMatchState.RACK_SATURATED;
        }
    }
    switch(slavePlacement) {
        case SEPARATE:
        case SEPARATE_BY_DEPLOY:
        case SPREAD_ALL_SLAVES:
            if (allowBounceToSameHost && taskLaunchedFromBounceWithActionId) {
                if (numFromSameBounceOnSlave > 0) {
                    LOG.trace("Rejecting SEPARATE task {} from slave {} ({}) due to numFromSameBounceOnSlave {}", taskRequest.getRequest().getId(), slaveId, host, numFromSameBounceOnSlave);
                    return SlaveMatchState.SLAVE_SATURATED;
                }
            } else {
                if (numOnSlave > 0 || numCleaningOnSlave > 0) {
                    LOG.trace("Rejecting {} task {} from slave {} ({}) due to numOnSlave {} numCleaningOnSlave {}", slavePlacement.name(), taskRequest.getRequest().getId(), slaveId, host, numOnSlave, numCleaningOnSlave);
                    return SlaveMatchState.SLAVE_SATURATED;
                }
            }
            break;
        case SEPARATE_BY_REQUEST:
            if (numOnSlave > 0 || numCleaningOnSlave > 0 || numOtherDeploysOnSlave > 0) {
                LOG.trace("Rejecting SEPARATE_BY_REQUEST task {} from slave {} ({}) due to numOnSlave {} numCleaningOnSlave {} numOtherDeploysOnSlave {}", taskRequest.getRequest().getId(), slaveId, host, numOnSlave, numCleaningOnSlave, numOtherDeploysOnSlave);
                return SlaveMatchState.SLAVE_SATURATED;
            }
            break;
        case OPTIMISTIC:
            // If no tasks are active for this request yet, we can fall back to greedy.
            if (activeTaskIdsForRequest.size() > 0) {
                Collection<SingularityPendingTaskId> pendingTasksForRequestClusterwide = leaderCache.getPendingTaskIdsForRequest(taskRequest.getRequest().getId());
                Set<String> currentHostsForRequest = activeTaskIdsForRequest.stream().map(SingularityTaskId::getSanitizedHost).collect(Collectors.toSet());
                final double numPerSlave = activeTaskIdsForRequest.size() / (double) currentHostsForRequest.size();
                final double leniencyCoefficient = configuration.getPlacementLeniency();
                final double threshold = numPerSlave * (1 + (pendingTasksForRequestClusterwide.size() * leniencyCoefficient));
                final boolean isSlaveOk = numOnSlave <= threshold;
                if (!isSlaveOk) {
                    LOG.trace("Rejecting OPTIMISTIC task {} from slave {} ({}) because numOnSlave {} violates threshold {} (based on active tasks for request {}, current hosts for request {}, pending tasks for request {})", taskRequest.getRequest().getId(), slaveId, host, numOnSlave, threshold, activeTaskIdsForRequest.size(), currentHostsForRequest.size(), pendingTasksForRequestClusterwide.size());
                    return SlaveMatchState.SLAVE_SATURATED;
                }
            }
            break;
        case GREEDY:
    }
    return SlaveMatchState.OK;
}

Example 92 with SingularityTask

use of com.hubspot.singularity.SingularityTask in project Singularity by HubSpot.

the class SingularityStartup method enqueueHealthAndNewTaskChecks.

private void enqueueHealthAndNewTaskChecks() {
    final long start = System.currentTimeMillis();
    final List<SingularityTask> activeTasks = taskManager.getActiveTasks();
    final Map<SingularityTaskId, SingularityTask> activeTaskMap = Maps.uniqueIndex(activeTasks, SingularityTaskIdHolder.getTaskIdFunction());
    final Map<SingularityTaskId, List<SingularityTaskHistoryUpdate>> taskUpdates = taskManager.getTaskHistoryUpdates(activeTaskMap.keySet());
    final Map<SingularityDeployKey, SingularityPendingDeploy> pendingDeploys = Maps.uniqueIndex(deployManager.getPendingDeploys(), SingularityDeployKey.FROM_PENDING_TO_DEPLOY_KEY);
    final Map<String, SingularityRequestWithState> idToRequest = Maps.uniqueIndex(requestManager.getRequests(), SingularityRequestWithState.REQUEST_STATE_TO_REQUEST_ID);
    requestManager.getActiveRequests();
    int enqueuedNewTaskChecks = 0;
    int enqueuedHealthchecks = 0;
    for (Map.Entry<SingularityTaskId, SingularityTask> entry : activeTaskMap.entrySet()) {
        SingularityTaskId taskId = entry.getKey();
        SingularityTask task = entry.getValue();
        SimplifiedTaskState simplifiedTaskState = SingularityTaskHistoryUpdate.getCurrentState(taskUpdates.get(taskId));
        if (simplifiedTaskState != SimplifiedTaskState.DONE) {
            SingularityDeployKey deployKey = new SingularityDeployKey(taskId.getRequestId(), taskId.getDeployId());
            Optional<SingularityPendingDeploy> pendingDeploy = Optional.fromNullable(pendingDeploys.get(deployKey));
            Optional<SingularityRequestWithState> request = Optional.fromNullable(idToRequest.get(taskId.getRequestId()));
            if (!pendingDeploy.isPresent()) {
                newTaskChecker.enqueueNewTaskCheck(task, request, healthchecker);
                enqueuedNewTaskChecks++;
            }
            if (simplifiedTaskState == SimplifiedTaskState.RUNNING) {
                if (healthchecker.enqueueHealthcheck(task, pendingDeploy, request)) {
                    enqueuedHealthchecks++;
                }
            }
        }
    }
    LOG.info("Enqueued {} health checks and {} new task checks (out of {} active tasks) in {}", enqueuedHealthchecks, enqueuedNewTaskChecks, activeTasks.size(), JavaUtils.duration(start));
}

Example 93 with SingularityTask

use of com.hubspot.singularity.SingularityTask in project Singularity by HubSpot.

the class SingularityTaskSizeOptimizer method getSizeOptimizedTask.

SingularityTask getSizeOptimizedTask(SingularityMesosTaskHolder taskHolder) {
    if (configuration.isStoreAllMesosTaskInfoForDebugging()) {
        return taskHolder.getTask();
    }
    SingularityTask task = taskHolder.getTask();
    TaskInfo.Builder mesosTask = taskHolder.getMesosTask().toBuilder();
    mesosTask.clearData();
    List<MesosOfferObject> offers = task.getOffers().stream().map(MesosOfferObject::sizeOptimized).collect(Collectors.toList());
    SingularityTaskRequest taskRequest = task.getTaskRequest();
    if (task.getTaskRequest().getDeploy().getExecutorData().isPresent()) {
        SingularityDeployBuilder deploy = task.getTaskRequest().getDeploy().toBuilder();
        deploy.setExecutorData(Optional.absent());
        taskRequest = new SingularityTaskRequest(task.getTaskRequest().getRequest(), deploy.build(), task.getTaskRequest().getPendingTask());
    }
    return new SingularityTask(taskRequest, task.getTaskId(), offers, mesosProtosUtils.taskFromProtos(mesosTask.build()), task.getRackId());
}

Example 94 with SingularityTask

use of com.hubspot.singularity.SingularityTask in project Singularity by HubSpot.

the class SingularityMesosOfferScheduler method acceptTask.

private SingularityMesosTaskHolder acceptTask(SingularityOfferHolder offerHolder, Map<String, Integer> tasksPerOffer, SingularityTaskRequestHolder taskRequestHolder) {
    final SingularityTaskRequest taskRequest = taskRequestHolder.getTaskRequest();
    final SingularityMesosTaskHolder taskHolder = mesosTaskBuilder.buildTask(offerHolder, offerHolder.getCurrentResources(), taskRequest, taskRequestHolder.getTaskResources(), taskRequestHolder.getExecutorResources());
    final SingularityTask zkTask = taskSizeOptimizer.getSizeOptimizedTask(taskHolder);
    LOG.trace("Accepted and built task {}", zkTask);
    LOG.info("Launching task {} slot on slave {} ({})", taskHolder.getTask().getTaskId(), offerHolder.getSlaveId(), offerHolder.getHostname());
    taskManager.createTaskAndDeletePendingTask(zkTask);
    addRequestToMapByOfferHost(tasksPerOffer, offerHolder.getHostname(), taskRequest.getRequest().getId());
    return taskHolder;
}