Examples with SingularityRequestWithState com.hubspot.singularity.SingularityRequestWithState used on opensource projects

Example 21 with SingularityRequestWithState

use of com.hubspot.singularity.SingularityRequestWithState in project Singularity by HubSpot.

the class RequestResource method getTaskByRunId.

@GET
@Path("/request/{requestId}/run/{runId}")
@ApiOperation("Retrieve an active task by runId")
public Optional<SingularityTaskId> getTaskByRunId(@Auth SingularityUser user, @PathParam("requestId") String requestId, @PathParam("runId") String runId) {
    SingularityRequestWithState requestWithState = fetchRequestWithState(requestId, user);
    authorizationHelper.checkForAuthorization(requestWithState.getRequest(), user, SingularityAuthorizationScope.READ);
    return taskManager.getTaskByRunId(requestId, runId);
}

Example 22 with SingularityRequestWithState

use of com.hubspot.singularity.SingularityRequestWithState in project Singularity by HubSpot.

the class SingularityCleaner method processRequestCleanup.

private void processRequestCleanup(long start, AtomicInteger numTasksKilled, AtomicInteger numScheduledTasksRemoved, SingularityRequestCleanup requestCleanup) {
    final List<SingularityTaskId> activeTaskIds = taskManager.getActiveTaskIdsForRequest(requestCleanup.getRequestId());
    final List<SingularityPendingTask> pendingTasks = taskManager.getPendingTasksForRequest(requestCleanup.getRequestId());
    final String requestId = requestCleanup.getRequestId();
    final Optional<SingularityRequestWithState> requestWithState = requestManager.getRequest(requestId);
    boolean killActiveTasks = requestCleanup.getKillTasks().or(configuration.isDefaultValueForKillTasksOfPausedRequests());
    boolean killScheduledTasks = true;
    switch(requestCleanup.getCleanupType()) {
        case PAUSING:
            if (SingularityRequestWithState.isActive(requestWithState)) {
                if (isObsolete(start, requestCleanup.getTimestamp())) {
                    killScheduledTasks = false;
                    killActiveTasks = false;
                    LOG.info("Ignoring {}, because {} is {}", requestCleanup, requestCleanup.getRequestId(), requestWithState.get().getState());
                } else {
                    LOG.debug("Waiting on {} (it will expire after {}), because {} is {}", requestCleanup, JavaUtils.durationFromMillis(getObsoleteExpirationTime()), requestCleanup.getRequestId(), requestWithState.get().getState());
                    return;
                }
            } else {
                if (pause(requestCleanup, activeTaskIds) == TaskCleanupType.PAUSING) {
                    killActiveTasks = false;
                }
            }
            break;
        case DELETING:
            if (!Iterables.isEmpty(activeTaskIds)) {
                killActiveTasks = false;
                killScheduledTasks = false;
                delete(requestCleanup, activeTaskIds);
            } else {
                Optional<SingularityRequestHistory> maybeHistory = requestHistoryHelper.getLastHistory(requestId);
                if (maybeHistory.isPresent()) {
                    if (maybeHistory.get().getRequest().isLoadBalanced() && configuration.isDeleteRemovedRequestsFromLoadBalancer() && requestCleanup.getRemoveFromLoadBalancer().or(true)) {
                        createLbCleanupRequest(requestId, activeTaskIds);
                    }
                    requestManager.markDeleted(maybeHistory.get().getRequest(), start, requestCleanup.getUser(), requestCleanup.getMessage());
                }
                cleanupRequestData(requestCleanup);
            }
            break;
        case BOUNCE:
        case INCREMENTAL_BOUNCE:
            killActiveTasks = false;
            killScheduledTasks = false;
            bounce(requestCleanup, activeTaskIds);
            break;
    }
    if (killActiveTasks) {
        for (SingularityTaskId matchingTaskId : activeTaskIds) {
            LOG.debug("Killing task {} due to {}", matchingTaskId, requestCleanup);
            scheduler.killAndRecord(matchingTaskId, requestCleanup.getCleanupType(), Optional.absent());
            numTasksKilled.getAndIncrement();
        }
    } else {
        LOG.info("Active tasks for {} not killed", requestCleanup);
    }
    if (killScheduledTasks) {
        for (SingularityPendingTask matchingTask : Iterables.filter(pendingTasks, SingularityPendingTask.matchingRequest(requestId))) {
            LOG.debug("Deleting scheduled task {} due to {}", matchingTask, requestCleanup);
            taskManager.deletePendingTask(matchingTask.getPendingTaskId());
            numScheduledTasksRemoved.getAndIncrement();
        }
    }
    requestManager.deleteCleanRequest(requestId, requestCleanup.getCleanupType());
}

Example 23 with SingularityRequestWithState

use of com.hubspot.singularity.SingularityRequestWithState in project Singularity by HubSpot.

the class SingularityCleaner method shouldKillTask.

private boolean shouldKillTask(SingularityTaskCleanup taskCleanup, List<SingularityTaskId> activeTaskIds, Set<SingularityTaskId> cleaningTasks, Multiset<SingularityDeployKey> incrementalCleaningTasks) {
    final Optional<SingularityRequestWithState> requestWithState = requestManager.getRequest(taskCleanup.getTaskId().getRequestId());
    if (!requestWithState.isPresent()) {
        LOG.debug("Killing a task {} immediately because the request was missing", taskCleanup);
        return true;
    }
    final SingularityRequest request = requestWithState.get().getRequest();
    if (taskCleanup.getRunBeforeKillId().isPresent()) {
        List<SingularityTaskShellCommandUpdate> shellCommandUpdates = taskManager.getTaskShellCommandUpdates(taskCleanup.getRunBeforeKillId().get());
        boolean finished = false;
        for (SingularityTaskShellCommandUpdate update : shellCommandUpdates) {
            if (update.getUpdateType().isFinished()) {
                finished = true;
                break;
            }
        }
        if (!finished) {
            LOG.debug("Waiting for pre-kill shell command {} to finish before killing task", taskCleanup.getRunBeforeKillId());
            return false;
        }
    }
    if (taskCleanup.getCleanupType().shouldKillTaskInstantly(request)) {
        LOG.debug("Killing a task {} immediately because of its cleanup type", taskCleanup);
        return true;
    }
    // If pausing, must be a long-running task to kill here
    if (requestWithState.get().getState() == RequestState.PAUSED && (!(taskCleanup.getCleanupType() == TaskCleanupType.PAUSING) || request.isLongRunning())) {
        LOG.debug("Killing a task {} immediately because the request was paused", taskCleanup);
        return true;
    }
    if (!request.isLongRunning()) {
        final long timeSinceCleanup = System.currentTimeMillis() - taskCleanup.getTimestamp();
        final long maxWaitTime = request.getKillOldNonLongRunningTasksAfterMillis().or(killNonLongRunningTasksInCleanupAfterMillis);
        final boolean tooOld = (maxWaitTime < 1) || (timeSinceCleanup > maxWaitTime);
        if (!tooOld) {
            LOG.trace("Not killing a non-longRunning task {}, running time since cleanup {} (max wait time is {})", taskCleanup, timeSinceCleanup, maxWaitTime);
        } else {
            LOG.debug("Killing a non-longRunning task {} - running time since cleanup {} exceeded max wait time {}", taskCleanup, timeSinceCleanup, maxWaitTime);
        }
        return tooOld;
    }
    final String requestId = request.getId();
    final Optional<SingularityRequestDeployState> deployState = deployManager.getRequestDeployState(requestId);
    if (taskCleanup.getCleanupType() == TaskCleanupType.DECOMISSIONING && deployState.get().getPendingDeploy().isPresent() && deployState.get().getPendingDeploy().get().getDeployId().equals(taskCleanup.getTaskId().getDeployId())) {
        final long timeSinceCleanup = System.currentTimeMillis() - taskCleanup.getTimestamp();
        final long maxWaitTime = configuration.getPendingDeployHoldTaskDuringDecommissionMillis();
        final boolean tooOld = (maxWaitTime < 1) || (timeSinceCleanup > maxWaitTime);
        if (!tooOld) {
            LOG.trace("Not killing {} - part of pending deploy - running time since cleanup {} (max wait time is {})", taskCleanup, timeSinceCleanup, maxWaitTime);
            return false;
        } else {
            LOG.debug("Killing {} - part of pending deploy but running time since cleanup {} exceeded max wait time {}", taskCleanup, timeSinceCleanup, maxWaitTime);
            return true;
        }
    }
    if (!deployState.isPresent() || !deployState.get().getActiveDeploy().isPresent()) {
        LOG.debug("Killing a task {} immediately because there is no active deploy state {}", taskCleanup, deployState);
        return true;
    }
    final String activeDeployId = deployState.get().getActiveDeploy().get().getDeployId();
    final String matchingTasksDeployId = taskCleanup.getCleanupType() == TaskCleanupType.INCREMENTAL_DEPLOY_CANCELLED || taskCleanup.getCleanupType() == TaskCleanupType.INCREMENTAL_DEPLOY_FAILED ? activeDeployId : taskCleanup.getTaskId().getDeployId();
    // check to see if there are enough active tasks out there that have been active for long enough that we can safely shut this task down.
    final List<SingularityTaskId> matchingTasks = new ArrayList<>();
    for (SingularityTaskId taskId : activeTaskIds) {
        if (!taskId.getRequestId().equals(requestId) || !taskId.getDeployId().equals(matchingTasksDeployId)) {
            continue;
        }
        if (cleaningTasks.contains(taskId)) {
            continue;
        }
        matchingTasks.add(taskId);
    }
    // For an incremental bounce or incremental deploy cleanup, shut down old tasks as new ones are started
    final SingularityDeployKey key = SingularityDeployKey.fromTaskId(taskCleanup.getTaskId());
    if (taskCleanup.getCleanupType() == TaskCleanupType.INCREMENTAL_BOUNCE) {
        return shouldKillIncrementalBounceTask(request, taskCleanup, matchingTasksDeployId, matchingTasks, key, incrementalCleaningTasks);
    } else if (isIncrementalDeployCleanup(taskCleanup)) {
        return shouldKillIncrementalDeployCleanupTask(request, taskCleanup, matchingTasksDeployId, matchingTasks, key, incrementalCleaningTasks);
    } else {
        if (matchingTasks.size() < request.getInstancesSafe()) {
            LOG.trace("Not killing a task {} yet, only {} matching out of a required {}", taskCleanup, matchingTasks.size(), request.getInstancesSafe());
            return false;
        }
    }
    final Optional<SingularityDeploy> deploy = deployManager.getDeploy(requestId, activeDeployId);
    final DeployHealth deployHealth = deployHealthHelper.getDeployHealth(requestWithState.get().getRequest(), deploy, matchingTasks, false);
    switch(deployHealth) {
        case HEALTHY:
            for (SingularityTaskId taskId : matchingTasks) {
                DeployHealth lbHealth = getLbHealth(request, taskId);
                if (lbHealth != DeployHealth.HEALTHY) {
                    LOG.trace("Not killing a task {}, waiting for new replacement tasks to be added to LB (current state: {})", taskCleanup, lbHealth);
                    return false;
                }
            }
            LOG.debug("Killing a task {}, all replacement tasks are healthy", taskCleanup);
            return true;
        case WAITING:
        case UNHEALTHY:
        default:
            LOG.trace("Not killing a task {}, waiting for new replacement tasks to be healthy (current state: {})", taskCleanup, deployHealth);
            return false;
    }
}

Example 24 with SingularityRequestWithState

use of com.hubspot.singularity.SingularityRequestWithState in project Singularity by HubSpot.

the class StateManager method generateState.

public SingularityState generateState(boolean includeRequestIds) {
    final int launchingTasks = taskManager.getNumLaunchingTasks();
    final int activeTasks = taskManager.getNumActiveTasks() - launchingTasks;
    final int scheduledTasks = taskManager.getNumScheduledTasks();
    final int cleaningTasks = taskManager.getNumCleanupTasks();
    final int lbCleanupTasks = taskManager.getNumLbCleanupTasks();
    final int lbCleanupRequests = requestManager.getNumLbCleanupRequests();
    final SingularityScheduledTasksInfo scheduledTasksInfo = SingularityScheduledTasksInfo.getInfo(taskManager.getPendingTasks(), singularityConfiguration.getDeltaAfterWhichTasksAreLateMillis());
    final List<String> overProvisionedRequestIds = new ArrayList<>();
    final Set<String> possiblyUnderProvisionedRequestIds = new HashSet<>();
    final List<SingularityRequestWithState> requests = requestManager.getRequests();
    final Map<String, Long> numInstances = getNumTasks(requests);
    int numActiveRequests = 0;
    int numPausedRequests = 0;
    int cooldownRequests = 0;
    int numFinishedRequests = 0;
    for (SingularityRequestWithState requestWithState : requests) {
        switch(requestWithState.getState()) {
            case DEPLOYING_TO_UNPAUSE:
            case ACTIVE:
                numActiveRequests++;
                break;
            case FINISHED:
                numFinishedRequests++;
                break;
            case PAUSED:
                numPausedRequests++;
                break;
            case SYSTEM_COOLDOWN:
                cooldownRequests++;
                break;
            case DELETED:
                break;
        }
        updatePossiblyUnderProvisionedAndOverProvisionedIds(requestWithState, numInstances, overProvisionedRequestIds, possiblyUnderProvisionedRequestIds);
    }
    filterForPendingRequests(possiblyUnderProvisionedRequestIds);
    final List<String> underProvisionedRequestIds = getUnderProvisionedRequestIds(possiblyUnderProvisionedRequestIds);
    final int pendingRequests = requestManager.getSizeOfPendingQueue();
    final int cleaningRequests = requestManager.getSizeOfCleanupQueue();
    List<SingularityRack> racks = rackManager.getObjects();
    int activeRacks = 0;
    int deadRacks = 0;
    int decommissioningRacks = 0;
    int unknownRacks = 0;
    for (SingularityRack rack : racks) {
        switch(rack.getCurrentState().getState()) {
            case ACTIVE:
                activeRacks++;
                break;
            case DEAD:
                deadRacks++;
                break;
            case MISSING_ON_STARTUP:
                unknownRacks++;
                break;
            case DECOMMISSIONED:
            case STARTING_DECOMMISSION:
            case DECOMMISSIONING:
                decommissioningRacks++;
                break;
            default:
                unknownRacks++;
                break;
        }
    }
    List<SingularitySlave> slaves = slaveManager.getObjects();
    int activeSlaves = 0;
    int deadSlaves = 0;
    int decommissioningSlaves = 0;
    int unknownSlaves = 0;
    for (SingularitySlave slave : slaves) {
        switch(slave.getCurrentState().getState()) {
            case ACTIVE:
                activeSlaves++;
                break;
            case DEAD:
                deadSlaves++;
                break;
            case MISSING_ON_STARTUP:
                unknownSlaves++;
                break;
            case DECOMMISSIONED:
            case STARTING_DECOMMISSION:
            case DECOMMISSIONING:
                decommissioningSlaves++;
                break;
            default:
                unknownSlaves++;
                break;
        }
    }
    final List<SingularityHostState> states = getHostStates();
    int numDeploys = 0;
    long oldestDeploy = 0;
    long oldestDeployStep = 0;
    List<SingularityDeployMarker> activeDeploys = new ArrayList<>();
    final long now = System.currentTimeMillis();
    for (SingularityPendingDeploy pendingDeploy : deployManager.getPendingDeploys()) {
        activeDeploys.add(pendingDeploy.getDeployMarker());
        if (pendingDeploy.getDeployProgress().isPresent() && !pendingDeploy.getDeployProgress().get().isStepComplete()) {
            long deployStepDelta = now - pendingDeploy.getDeployProgress().get().getTimestamp();
            if (deployStepDelta > oldestDeployStep) {
                oldestDeployStep = deployStepDelta;
            }
        }
        long delta = now - pendingDeploy.getDeployMarker().getTimestamp();
        if (delta > oldestDeploy) {
            oldestDeploy = delta;
        }
        numDeploys++;
    }
    final Optional<Boolean> authDatastoreHealthy = authDatastore.isHealthy();
    final Optional<Double> minimumPriorityLevel = getMinimumPriorityLevel();
    return new SingularityState(activeTasks, launchingTasks, numActiveRequests, cooldownRequests, numPausedRequests, scheduledTasks, pendingRequests, lbCleanupTasks, lbCleanupRequests, cleaningRequests, activeSlaves, deadSlaves, decommissioningSlaves, activeRacks, deadRacks, decommissioningRacks, cleaningTasks, states, oldestDeploy, numDeploys, oldestDeployStep, activeDeploys, scheduledTasksInfo.getNumLateTasks(), scheduledTasksInfo.getNumFutureTasks(), scheduledTasksInfo.getMaxTaskLag(), System.currentTimeMillis(), includeRequestIds ? overProvisionedRequestIds : null, includeRequestIds ? underProvisionedRequestIds : null, overProvisionedRequestIds.size(), underProvisionedRequestIds.size(), numFinishedRequests, unknownRacks, unknownSlaves, authDatastoreHealthy, minimumPriorityLevel, statusUpdateDeltaAvg.get());
}

Example 25 with SingularityRequestWithState

use of com.hubspot.singularity.SingularityRequestWithState in project Singularity by HubSpot.

the class TaskRequestManager method getTaskRequests.

public List<SingularityTaskRequest> getTaskRequests(List<SingularityPendingTask> tasks) {
    final Multimap<String, SingularityPendingTask> requestIdToPendingTaskId = ArrayListMultimap.create(tasks.size(), 1);
    for (SingularityPendingTask task : tasks) {
        requestIdToPendingTaskId.put(task.getPendingTaskId().getRequestId(), task);
    }
    final List<SingularityRequestWithState> matchingRequests = requestManager.getRequests(requestIdToPendingTaskId.keySet());
    final Map<SingularityPendingTask, SingularityDeployKey> deployKeys = SingularityDeployKey.fromPendingTasks(requestIdToPendingTaskId.values());
    final Map<SingularityDeployKey, SingularityDeploy> matchingDeploys = deployManager.getDeploysForKeys(Sets.newHashSet(deployKeys.values()));
    final List<SingularityTaskRequest> taskRequests = Lists.newArrayListWithCapacity(matchingRequests.size());
    for (SingularityRequestWithState request : matchingRequests) {
        Optional<SingularityPendingDeploy> maybePendingDeploy = deployManager.getPendingDeploy(request.getRequest().getId());
        for (SingularityPendingTask task : requestIdToPendingTaskId.get(request.getRequest().getId())) {
            SingularityDeploy foundDeploy = matchingDeploys.get(deployKeys.get(task));
            if (foundDeploy == null) {
                LOG.warn("Couldn't find a matching deploy for pending task {}", task);
                continue;
            }
            if (!request.getState().isRunnable()) {
                LOG.warn("Request was in state {} for pending task {}", request.getState(), task);
                continue;
            }
            Optional<SingularityRequest> updatedRequest = maybePendingDeploy.isPresent() && maybePendingDeploy.get().getDeployMarker().getDeployId().equals(task.getPendingTaskId().getDeployId()) ? maybePendingDeploy.get().getUpdatedRequest() : Optional.<SingularityRequest>absent();
            taskRequests.add(new SingularityTaskRequest(updatedRequest.or(request.getRequest()), foundDeploy, task));
        }
    }
    return taskRequests;
}