Examples with SingularityPendingDeploy com.hubspot.singularity.SingularityPendingDeploy used on opensource projects

Example 11 with SingularityPendingDeploy

use of com.hubspot.singularity.SingularityPendingDeploy in project Singularity by HubSpot.

the class TaskRequestManager method getTaskRequests.

public List<SingularityTaskRequest> getTaskRequests(List<SingularityPendingTask> tasks) {
    final Multimap<String, SingularityPendingTask> requestIdToPendingTaskId = ArrayListMultimap.create(tasks.size(), 1);
    for (SingularityPendingTask task : tasks) {
        requestIdToPendingTaskId.put(task.getPendingTaskId().getRequestId(), task);
    }
    final List<SingularityRequestWithState> matchingRequests = requestManager.getRequests(requestIdToPendingTaskId.keySet());
    final Map<SingularityPendingTask, SingularityDeployKey> deployKeys = SingularityDeployKey.fromPendingTasks(requestIdToPendingTaskId.values());
    final Map<SingularityDeployKey, SingularityDeploy> matchingDeploys = deployManager.getDeploysForKeys(Sets.newHashSet(deployKeys.values()));
    final List<SingularityTaskRequest> taskRequests = Lists.newArrayListWithCapacity(matchingRequests.size());
    for (SingularityRequestWithState request : matchingRequests) {
        Optional<SingularityPendingDeploy> maybePendingDeploy = deployManager.getPendingDeploy(request.getRequest().getId());
        for (SingularityPendingTask task : requestIdToPendingTaskId.get(request.getRequest().getId())) {
            SingularityDeploy foundDeploy = matchingDeploys.get(deployKeys.get(task));
            if (foundDeploy == null) {
                LOG.warn("Couldn't find a matching deploy for pending task {}", task);
                continue;
            }
            if (!request.getState().isRunnable()) {
                LOG.warn("Request was in state {} for pending task {}", request.getState(), task);
                continue;
            }
            Optional<SingularityRequest> updatedRequest = maybePendingDeploy.isPresent() && maybePendingDeploy.get().getDeployMarker().getDeployId().equals(task.getPendingTaskId().getDeployId()) ? maybePendingDeploy.get().getUpdatedRequest() : Optional.<SingularityRequest>absent();
            taskRequests.add(new SingularityTaskRequest(updatedRequest.or(request.getRequest()), foundDeploy, task));
        }
    }
    return taskRequests;
}

Example 12 with SingularityPendingDeploy

use of com.hubspot.singularity.SingularityPendingDeploy in project Singularity by HubSpot.

the class RequestHelper method getTaskIdsByStatusForRequest.

private Optional<SingularityTaskIdsByStatus> getTaskIdsByStatusForRequest(SingularityRequestWithState requestWithState) {
    String requestId = requestWithState.getRequest().getId();
    Optional<SingularityPendingDeploy> pendingDeploy = deployManager.getPendingDeploy(requestId);
    List<SingularityTaskId> cleaningTaskIds = taskManager.getCleanupTaskIds().stream().filter((t) -> t.getRequestId().equals(requestId)).collect(Collectors.toList());
    List<SingularityPendingTaskId> pendingTaskIds = taskManager.getPendingTaskIdsForRequest(requestId);
    List<SingularityTaskId> activeTaskIds = taskManager.getActiveTaskIdsForRequest(requestId);
    activeTaskIds.removeAll(cleaningTaskIds);
    List<SingularityTaskId> healthyTaskIds = new ArrayList<>();
    List<SingularityTaskId> notYetHealthyTaskIds = new ArrayList<>();
    Map<String, List<SingularityTaskId>> taskIdsByDeployId = activeTaskIds.stream().collect(Collectors.groupingBy(SingularityTaskId::getDeployId));
    for (Map.Entry<String, List<SingularityTaskId>> entry : taskIdsByDeployId.entrySet()) {
        Optional<SingularityDeploy> deploy = deployManager.getDeploy(requestId, entry.getKey());
        List<SingularityTaskId> healthyTasksIdsForDeploy = deployHealthHelper.getHealthyTasks(requestWithState.getRequest(), deploy, entry.getValue(), pendingDeploy.isPresent() && pendingDeploy.get().getDeployMarker().getDeployId().equals(entry.getKey()));
        for (SingularityTaskId taskId : entry.getValue()) {
            if (healthyTasksIdsForDeploy.contains(taskId)) {
                healthyTaskIds.add(taskId);
            } else {
                notYetHealthyTaskIds.add(taskId);
            }
        }
    }
    return Optional.of(new SingularityTaskIdsByStatus(healthyTaskIds, notYetHealthyTaskIds, pendingTaskIds, cleaningTaskIds));
}

Example 13 with SingularityPendingDeploy

use of com.hubspot.singularity.SingularityPendingDeploy in project Singularity by HubSpot.

the class SingularityStartup method enqueueHealthAndNewTaskChecks.

private void enqueueHealthAndNewTaskChecks() {
    final long start = System.currentTimeMillis();
    final List<SingularityTask> activeTasks = taskManager.getActiveTasks();
    final Map<SingularityTaskId, SingularityTask> activeTaskMap = Maps.uniqueIndex(activeTasks, SingularityTaskIdHolder.getTaskIdFunction());
    final Map<SingularityTaskId, List<SingularityTaskHistoryUpdate>> taskUpdates = taskManager.getTaskHistoryUpdates(activeTaskMap.keySet());
    final Map<SingularityDeployKey, SingularityPendingDeploy> pendingDeploys = Maps.uniqueIndex(deployManager.getPendingDeploys(), SingularityDeployKey.FROM_PENDING_TO_DEPLOY_KEY);
    final Map<String, SingularityRequestWithState> idToRequest = Maps.uniqueIndex(requestManager.getRequests(), SingularityRequestWithState.REQUEST_STATE_TO_REQUEST_ID);
    requestManager.getActiveRequests();
    int enqueuedNewTaskChecks = 0;
    int enqueuedHealthchecks = 0;
    for (Map.Entry<SingularityTaskId, SingularityTask> entry : activeTaskMap.entrySet()) {
        SingularityTaskId taskId = entry.getKey();
        SingularityTask task = entry.getValue();
        SimplifiedTaskState simplifiedTaskState = SingularityTaskHistoryUpdate.getCurrentState(taskUpdates.get(taskId));
        if (simplifiedTaskState != SimplifiedTaskState.DONE) {
            SingularityDeployKey deployKey = new SingularityDeployKey(taskId.getRequestId(), taskId.getDeployId());
            Optional<SingularityPendingDeploy> pendingDeploy = Optional.fromNullable(pendingDeploys.get(deployKey));
            Optional<SingularityRequestWithState> request = Optional.fromNullable(idToRequest.get(taskId.getRequestId()));
            if (!pendingDeploy.isPresent()) {
                newTaskChecker.enqueueNewTaskCheck(task, request, healthchecker);
                enqueuedNewTaskChecks++;
            }
            if (simplifiedTaskState == SimplifiedTaskState.RUNNING) {
                if (healthchecker.enqueueHealthcheck(task, pendingDeploy, request)) {
                    enqueuedHealthchecks++;
                }
            }
        }
    }
    LOG.info("Enqueued {} health checks and {} new task checks (out of {} active tasks) in {}", enqueuedHealthchecks, enqueuedNewTaskChecks, activeTasks.size(), JavaUtils.duration(start));
}

Example 14 with SingularityPendingDeploy

use of com.hubspot.singularity.SingularityPendingDeploy in project Singularity by HubSpot.

the class SingularityTaskHistoryPersister method runActionOnPoll.

@Override
public void runActionOnPoll() {
    LOG.info("Checking inactive task ids for task history persistence");
    final long start = System.currentTimeMillis();
    final List<SingularityTaskId> allTaskIds = taskManager.getAllTaskIds();
    final Set<SingularityTaskId> activeTaskIds = Sets.newHashSet(taskManager.getActiveTaskIds());
    final Set<SingularityTaskId> lbCleaningTaskIds = Sets.newHashSet(taskManager.getLBCleanupTasks());
    final List<SingularityPendingDeploy> pendingDeploys = deployManager.getPendingDeploys();
    int numTotal = 0;
    int numTransferred = 0;
    final Multimap<String, SingularityTaskId> eligibleTaskIdByRequestId = TreeMultimap.create(Ordering.natural(), SingularityTaskId.STARTED_AT_COMPARATOR_DESC);
    for (SingularityTaskId taskId : allTaskIds) {
        if (activeTaskIds.contains(taskId) || lbCleaningTaskIds.contains(taskId) || isPartOfPendingDeploy(pendingDeploys, taskId)) {
            continue;
        }
        eligibleTaskIdByRequestId.put(taskId.getRequestId(), taskId);
    }
    for (Map.Entry<String, Collection<SingularityTaskId>> entry : eligibleTaskIdByRequestId.asMap().entrySet()) {
        int i = 0;
        for (SingularityTaskId taskId : entry.getValue()) {
            final long age = start - taskId.getStartedAt();
            if (age < configuration.getTaskPersistAfterStartupBufferMillis()) {
                LOG.debug("Not persisting {}, it has started up too recently {} (buffer: {}) - this prevents race conditions with ZK tx", taskId, JavaUtils.durationFromMillis(age), JavaUtils.durationFromMillis(configuration.getTaskPersistAfterStartupBufferMillis()));
                continue;
            }
            if (moveToHistoryOrCheckForPurge(taskId, i++)) {
                numTransferred++;
            }
            numTotal++;
        }
    }
    LOG.info("Transferred {} out of {} inactive task ids (total {}) in {}", numTransferred, numTotal, allTaskIds.size(), JavaUtils.duration(start));
}