Examples with SingularityRequestDeployState com.hubspot.singularity.SingularityRequestDeployState used on opensource projects

Example 6 with SingularityRequestDeployState

use of com.hubspot.singularity.SingularityRequestDeployState in project Singularity by HubSpot.

the class SingularityScheduler method handlePendingRequestsForDeployKey.

private void handlePendingRequestsForDeployKey(AtomicInteger obsoleteRequests, AtomicInteger heldForScheduledActiveTask, AtomicInteger totalNewScheduledTasks, SingularityDeployKey deployKey, List<SingularityPendingRequest> pendingRequestsForDeploy) {
    final String requestId = deployKey.getRequestId();
    final Optional<SingularityRequestWithState> maybeRequest = requestManager.getRequest(requestId);
    final SingularityDeployStatistics deployStatistics = getDeployStatistics(deployKey.getRequestId(), deployKey.getDeployId());
    if (!isRequestActive(maybeRequest)) {
        LOG.debug("Pending request {} was obsolete (request {})", requestId, SingularityRequestWithState.getRequestState(maybeRequest));
        obsoleteRequests.getAndIncrement();
        for (SingularityPendingRequest pendingRequest : pendingRequestsForDeploy) {
            requestManager.deletePendingRequest(pendingRequest);
        }
        return;
    }
    SingularityRequestWithState request = maybeRequest.get();
    Optional<SingularityRequestDeployState> maybeRequestDeployState = deployManager.getRequestDeployState(requestId);
    Optional<SingularityPendingDeploy> maybePendingDeploy = deployManager.getPendingDeploy(requestId);
    List<SingularityTaskId> matchingTaskIds = getMatchingTaskIds(request.getRequest(), deployKey);
    List<SingularityPendingRequest> effectivePendingRequests = new ArrayList<>();
    // Things that are closest to now (ie smaller timestamps) should come first in the queue
    pendingRequestsForDeploy.sort(Comparator.comparingLong(SingularityPendingRequest::getTimestamp));
    int scheduledTasks = 0;
    for (SingularityPendingRequest pendingRequest : pendingRequestsForDeploy) {
        final SingularityRequest updatedRequest = updatedRequest(maybePendingDeploy, pendingRequest, request);
        if (!shouldScheduleTasks(updatedRequest, pendingRequest, maybePendingDeploy, maybeRequestDeployState)) {
            LOG.debug("Pending request {} was obsolete (request {})", pendingRequest, SingularityRequestWithState.getRequestState(maybeRequest));
            obsoleteRequests.getAndIncrement();
            requestManager.deletePendingRequest(pendingRequest);
            continue;
        }
        int missingInstances = getNumMissingInstances(matchingTaskIds, updatedRequest, pendingRequest, maybePendingDeploy);
        if (missingInstances == 0 && !matchingTaskIds.isEmpty() && updatedRequest.isScheduled() && pendingRequest.getPendingType() == PendingType.NEW_DEPLOY) {
            LOG.trace("Holding pending request {} because it is scheduled and has an active task", pendingRequest);
            heldForScheduledActiveTask.getAndIncrement();
            continue;
        }
        if (effectivePendingRequests.isEmpty()) {
            effectivePendingRequests.add(pendingRequest);
            RequestState requestState = checkCooldown(request.getState(), request.getRequest(), deployStatistics);
            scheduledTasks += scheduleTasks(request.getRequest(), requestState, deployStatistics, pendingRequest, matchingTaskIds, maybePendingDeploy);
            requestManager.deletePendingRequest(pendingRequest);
        } else if (pendingRequest.getPendingType() == PendingType.IMMEDIATE) {
            effectivePendingRequests.add(pendingRequest);
            RequestState requestState = checkCooldown(request.getState(), request.getRequest(), deployStatistics);
            scheduledTasks += scheduleTasks(request.getRequest(), requestState, deployStatistics, pendingRequest, matchingTaskIds, maybePendingDeploy);
            requestManager.deletePendingRequest(pendingRequest);
        } else if (pendingRequest.getPendingType() == PendingType.ONEOFF) {
            effectivePendingRequests.add(pendingRequest);
            RequestState requestState = checkCooldown(request.getState(), request.getRequest(), deployStatistics);
            scheduledTasks += scheduleTasks(request.getRequest(), requestState, deployStatistics, pendingRequest, matchingTaskIds, maybePendingDeploy);
            requestManager.deletePendingRequest(pendingRequest);
        } else if (updatedRequest.isScheduled() && (pendingRequest.getPendingType() == PendingType.NEW_DEPLOY || pendingRequest.getPendingType() == PendingType.TASK_DONE)) {
            // If we are here, there is already an immediate of run of the scheduled task launched. Drop anything that would
            // leave a second instance of the request in the pending queue.
            requestManager.deletePendingRequest(pendingRequest);
        }
    // Any other subsequent requests are not honored until after the pending queue is cleared.
    }
    totalNewScheduledTasks.getAndAdd(scheduledTasks);
}

Example 7 with SingularityRequestDeployState

use of com.hubspot.singularity.SingularityRequestDeployState in project Singularity by HubSpot.

the class SingularityScheduler method checkForStaleScheduledTasks.

private List<SingularityTaskRequest> checkForStaleScheduledTasks(List<SingularityPendingTask> pendingTasks, List<SingularityTaskRequest> taskRequests) {
    final Set<String> foundPendingTaskId = Sets.newHashSetWithExpectedSize(taskRequests.size());
    final Set<String> requestIds = Sets.newHashSetWithExpectedSize(taskRequests.size());
    for (SingularityTaskRequest taskRequest : taskRequests) {
        foundPendingTaskId.add(taskRequest.getPendingTask().getPendingTaskId().getId());
        requestIds.add(taskRequest.getRequest().getId());
    }
    for (SingularityPendingTask pendingTask : pendingTasks) {
        if (!foundPendingTaskId.contains(pendingTask.getPendingTaskId().getId())) {
            LOG.info("Removing stale pending task {}", pendingTask.getPendingTaskId());
            taskManager.deletePendingTask(pendingTask.getPendingTaskId());
        }
    }
    // TODO this check isn't necessary if we keep track better during deploys
    final Map<String, SingularityRequestDeployState> deployStates = deployManager.getRequestDeployStatesByRequestIds(requestIds);
    final List<SingularityTaskRequest> taskRequestsWithValidDeploys = Lists.newArrayListWithCapacity(taskRequests.size());
    for (SingularityTaskRequest taskRequest : taskRequests) {
        SingularityRequestDeployState requestDeployState = deployStates.get(taskRequest.getRequest().getId());
        if (!matchesDeploy(requestDeployState, taskRequest) && !(taskRequest.getRequest().getRequestType() == RequestType.RUN_ONCE)) {
            LOG.info("Removing stale pending task {} because the deployId did not match active/pending deploys {}", taskRequest.getPendingTask().getPendingTaskId(), requestDeployState);
            taskManager.deletePendingTask(taskRequest.getPendingTask().getPendingTaskId());
        } else {
            taskRequestsWithValidDeploys.add(taskRequest);
        }
    }
    return taskRequestsWithValidDeploys;
}

Example 8 with SingularityRequestDeployState

use of com.hubspot.singularity.SingularityRequestDeployState in project Singularity by HubSpot.

the class SingularityCleaner method shouldKillTask.

private boolean shouldKillTask(SingularityTaskCleanup taskCleanup, List<SingularityTaskId> activeTaskIds, Set<SingularityTaskId> cleaningTasks, Multiset<SingularityDeployKey> incrementalCleaningTasks) {
    final Optional<SingularityRequestWithState> requestWithState = requestManager.getRequest(taskCleanup.getTaskId().getRequestId());
    if (!requestWithState.isPresent()) {
        LOG.debug("Killing a task {} immediately because the request was missing", taskCleanup);
        return true;
    }
    final SingularityRequest request = requestWithState.get().getRequest();
    if (taskCleanup.getRunBeforeKillId().isPresent()) {
        List<SingularityTaskShellCommandUpdate> shellCommandUpdates = taskManager.getTaskShellCommandUpdates(taskCleanup.getRunBeforeKillId().get());
        boolean finished = false;
        for (SingularityTaskShellCommandUpdate update : shellCommandUpdates) {
            if (update.getUpdateType().isFinished()) {
                finished = true;
                break;
            }
        }
        if (!finished) {
            LOG.debug("Waiting for pre-kill shell command {} to finish before killing task", taskCleanup.getRunBeforeKillId());
            return false;
        }
    }
    if (taskCleanup.getCleanupType().shouldKillTaskInstantly(request)) {
        LOG.debug("Killing a task {} immediately because of its cleanup type", taskCleanup);
        return true;
    }
    // If pausing, must be a long-running task to kill here
    if (requestWithState.get().getState() == RequestState.PAUSED && (!(taskCleanup.getCleanupType() == TaskCleanupType.PAUSING) || request.isLongRunning())) {
        LOG.debug("Killing a task {} immediately because the request was paused", taskCleanup);
        return true;
    }
    if (!request.isLongRunning()) {
        final long timeSinceCleanup = System.currentTimeMillis() - taskCleanup.getTimestamp();
        final long maxWaitTime = request.getKillOldNonLongRunningTasksAfterMillis().or(killNonLongRunningTasksInCleanupAfterMillis);
        final boolean tooOld = (maxWaitTime < 1) || (timeSinceCleanup > maxWaitTime);
        if (!tooOld) {
            LOG.trace("Not killing a non-longRunning task {}, running time since cleanup {} (max wait time is {})", taskCleanup, timeSinceCleanup, maxWaitTime);
        } else {
            LOG.debug("Killing a non-longRunning task {} - running time since cleanup {} exceeded max wait time {}", taskCleanup, timeSinceCleanup, maxWaitTime);
        }
        return tooOld;
    }
    final String requestId = request.getId();
    final Optional<SingularityRequestDeployState> deployState = deployManager.getRequestDeployState(requestId);
    if (taskCleanup.getCleanupType() == TaskCleanupType.DECOMISSIONING && deployState.get().getPendingDeploy().isPresent() && deployState.get().getPendingDeploy().get().getDeployId().equals(taskCleanup.getTaskId().getDeployId())) {
        final long timeSinceCleanup = System.currentTimeMillis() - taskCleanup.getTimestamp();
        final long maxWaitTime = configuration.getPendingDeployHoldTaskDuringDecommissionMillis();
        final boolean tooOld = (maxWaitTime < 1) || (timeSinceCleanup > maxWaitTime);
        if (!tooOld) {
            LOG.trace("Not killing {} - part of pending deploy - running time since cleanup {} (max wait time is {})", taskCleanup, timeSinceCleanup, maxWaitTime);
            return false;
        } else {
            LOG.debug("Killing {} - part of pending deploy but running time since cleanup {} exceeded max wait time {}", taskCleanup, timeSinceCleanup, maxWaitTime);
            return true;
        }
    }
    if (!deployState.isPresent() || !deployState.get().getActiveDeploy().isPresent()) {
        LOG.debug("Killing a task {} immediately because there is no active deploy state {}", taskCleanup, deployState);
        return true;
    }
    final String activeDeployId = deployState.get().getActiveDeploy().get().getDeployId();
    final String matchingTasksDeployId = taskCleanup.getCleanupType() == TaskCleanupType.INCREMENTAL_DEPLOY_CANCELLED || taskCleanup.getCleanupType() == TaskCleanupType.INCREMENTAL_DEPLOY_FAILED ? activeDeployId : taskCleanup.getTaskId().getDeployId();
    // check to see if there are enough active tasks out there that have been active for long enough that we can safely shut this task down.
    final List<SingularityTaskId> matchingTasks = new ArrayList<>();
    for (SingularityTaskId taskId : activeTaskIds) {
        if (!taskId.getRequestId().equals(requestId) || !taskId.getDeployId().equals(matchingTasksDeployId)) {
            continue;
        }
        if (cleaningTasks.contains(taskId)) {
            continue;
        }
        matchingTasks.add(taskId);
    }
    // For an incremental bounce or incremental deploy cleanup, shut down old tasks as new ones are started
    final SingularityDeployKey key = SingularityDeployKey.fromTaskId(taskCleanup.getTaskId());
    if (taskCleanup.getCleanupType() == TaskCleanupType.INCREMENTAL_BOUNCE) {
        return shouldKillIncrementalBounceTask(request, taskCleanup, matchingTasksDeployId, matchingTasks, key, incrementalCleaningTasks);
    } else if (isIncrementalDeployCleanup(taskCleanup)) {
        return shouldKillIncrementalDeployCleanupTask(request, taskCleanup, matchingTasksDeployId, matchingTasks, key, incrementalCleaningTasks);
    } else {
        if (matchingTasks.size() < request.getInstancesSafe()) {
            LOG.trace("Not killing a task {} yet, only {} matching out of a required {}", taskCleanup, matchingTasks.size(), request.getInstancesSafe());
            return false;
        }
    }
    final Optional<SingularityDeploy> deploy = deployManager.getDeploy(requestId, activeDeployId);
    final DeployHealth deployHealth = deployHealthHelper.getDeployHealth(requestWithState.get().getRequest(), deploy, matchingTasks, false);
    switch(deployHealth) {
        case HEALTHY:
            for (SingularityTaskId taskId : matchingTasks) {
                DeployHealth lbHealth = getLbHealth(request, taskId);
                if (lbHealth != DeployHealth.HEALTHY) {
                    LOG.trace("Not killing a task {}, waiting for new replacement tasks to be added to LB (current state: {})", taskCleanup, lbHealth);
                    return false;
                }
            }
            LOG.debug("Killing a task {}, all replacement tasks are healthy", taskCleanup);
            return true;
        case WAITING:
        case UNHEALTHY:
        default:
            LOG.trace("Not killing a task {}, waiting for new replacement tasks to be healthy (current state: {})", taskCleanup, deployHealth);
            return false;
    }
}

Example 9 with SingularityRequestDeployState

use of com.hubspot.singularity.SingularityRequestDeployState in project Singularity by HubSpot.

the class SingularityCooldownChecker method shouldExitCooldown.

private boolean shouldExitCooldown(SingularityRequestWithState cooldownRequest) {
    Optional<SingularityRequestDeployState> maybeDeployState = deployManager.getRequestDeployState(cooldownRequest.getRequest().getId());
    if (!maybeDeployState.isPresent() || !maybeDeployState.get().getActiveDeploy().isPresent()) {
        LOG.trace("{} had no deployState / activeDeploy {}, exiting cooldown", cooldownRequest.getRequest().getId(), maybeDeployState);
        return true;
    }
    Optional<SingularityDeployStatistics> maybeDeployStatistics = deployManager.getDeployStatistics(cooldownRequest.getRequest().getId(), maybeDeployState.get().getActiveDeploy().get().getDeployId());
    if (!maybeDeployStatistics.isPresent()) {
        LOG.trace("{} had no deploy statistics, exiting cooldown", new SingularityDeployKey(cooldownRequest.getRequest().getId(), maybeDeployState.get().getActiveDeploy().get().getDeployId()));
        return true;
    }
    Optional<Long> lastFinishAt = maybeDeployStatistics.get().getLastFinishAt();
    if (!lastFinishAt.isPresent()) {
        LOG.trace("{} had no last finish, exiting cooldown", new SingularityDeployKey(cooldownRequest.getRequest().getId(), maybeDeployState.get().getActiveDeploy().get().getDeployId()));
        return true;
    }
    if (cooldown.hasCooldownExpired(cooldownRequest.getRequest(), maybeDeployStatistics.get(), Optional.<Integer>absent(), Optional.<Long>absent())) {
        return true;
    }
    return false;
}

Example 10 with SingularityRequestDeployState

use of com.hubspot.singularity.SingularityRequestDeployState in project Singularity by HubSpot.

the class SingularitySchedulerTestBase method finishDeploy.

protected void finishDeploy(SingularityDeployMarker marker, SingularityDeploy deploy) {
    deployManager.deletePendingDeploy(marker.getRequestId());
    deployManager.saveDeployResult(marker, Optional.of(deploy), new SingularityDeployResult(DeployState.SUCCEEDED));
    deployManager.saveNewRequestDeployState(new SingularityRequestDeployState(marker.getRequestId(), Optional.of(marker), Optional.<SingularityDeployMarker>absent()));
}