Examples with RequestState com.hubspot.singularity.RequestState used on opensource projects

Example 1 with RequestState

use of com.hubspot.singularity.RequestState in project Singularity by HubSpot.

the class SingularityScheduler method handleCompletedTaskWithStatistics.

private Optional<PendingType> handleCompletedTaskWithStatistics(Optional<SingularityTask> task, SingularityTaskId taskId, long timestamp, ExtendedTaskState state, SingularityDeployStatistics deployStatistics, SingularityCreateResult taskHistoryUpdateCreateResult, Protos.TaskStatus status) {
    final Optional<SingularityRequestWithState> maybeRequestWithState = requestManager.getRequest(taskId.getRequestId());
    final Optional<SingularityPendingDeploy> maybePendingDeploy = deployManager.getPendingDeploy(taskId.getRequestId());
    if (!isRequestActive(maybeRequestWithState)) {
        LOG.warn("Not scheduling a new task, {} is {}", taskId.getRequestId(), SingularityRequestWithState.getRequestState(maybeRequestWithState));
        return Optional.absent();
    }
    RequestState requestState = maybeRequestWithState.get().getState();
    final SingularityRequest request = maybePendingDeploy.isPresent() ? maybePendingDeploy.get().getUpdatedRequest().or(maybeRequestWithState.get().getRequest()) : maybeRequestWithState.get().getRequest();
    final Optional<SingularityRequestDeployState> requestDeployState = deployManager.getRequestDeployState(request.getId());
    if (!isDeployInUse(requestDeployState, taskId.getDeployId(), true)) {
        LOG.debug("Task {} completed, but it didn't match active deploy state {} - ignoring", taskId.getId(), requestDeployState);
        return Optional.absent();
    }
    if (taskHistoryUpdateCreateResult == SingularityCreateResult.CREATED && requestState != RequestState.SYSTEM_COOLDOWN) {
        mailer.queueTaskCompletedMail(task, taskId, request, state);
    } else if (requestState == RequestState.SYSTEM_COOLDOWN) {
        LOG.debug("Not sending a task completed email because task {} is in SYSTEM_COOLDOWN", taskId);
    } else {
        LOG.debug("Not sending a task completed email for task {} because Singularity already processed this update", taskId);
    }
    if (!status.hasReason() || !status.getReason().equals(Reason.REASON_INVALID_OFFERS)) {
        if (!state.isSuccess() && taskHistoryUpdateCreateResult == SingularityCreateResult.CREATED && cooldown.shouldEnterCooldown(request, taskId, requestState, deployStatistics, timestamp)) {
            LOG.info("Request {} is entering cooldown due to task {}", request.getId(), taskId);
            requestState = RequestState.SYSTEM_COOLDOWN;
            requestManager.cooldown(request, System.currentTimeMillis());
            mailer.sendRequestInCooldownMail(request);
        }
    } else {
        LOG.debug("Not triggering cooldown due to TASK_LOST from invalid offers for request {}", request.getId());
    }
    PendingType pendingType = PendingType.TASK_DONE;
    Optional<List<String>> cmdLineArgsList = Optional.absent();
    if (!state.isSuccess() && shouldRetryImmediately(request, deployStatistics, task)) {
        LOG.debug("Retrying {} because {}", request.getId(), state);
        pendingType = PendingType.RETRY;
        if (task.isPresent()) {
            cmdLineArgsList = task.get().getTaskRequest().getPendingTask().getCmdLineArgsList();
        }
    } else if (!request.isAlwaysRunning()) {
        return Optional.absent();
    }
    if (state.isSuccess() && requestState == RequestState.SYSTEM_COOLDOWN) {
        // TODO send not cooldown anymore email
        LOG.info("Request {} succeeded a task, removing from cooldown", request.getId());
        requestManager.exitCooldown(request, System.currentTimeMillis(), Optional.<String>absent(), Optional.<String>absent());
    }
    SingularityPendingRequest pendingRequest = new SingularityPendingRequest(request.getId(), requestDeployState.get().getActiveDeploy().get().getDeployId(), System.currentTimeMillis(), Optional.absent(), pendingType, cmdLineArgsList, Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent());
    requestManager.addToPendingQueue(pendingRequest);
    return Optional.of(pendingType);
}

Example 2 with RequestState

use of com.hubspot.singularity.RequestState in project Singularity by HubSpot.

the class RequestResource method submitRequest.

private void submitRequest(SingularityRequest request, Optional<SingularityRequestWithState> oldRequestWithState, Optional<RequestHistoryType> historyType, Optional<Boolean> skipHealthchecks, Optional<String> message, Optional<SingularityBounceRequest> maybeBounceRequest, SingularityUser user) {
    checkNotNullBadRequest(request.getId(), "Request must have an id");
    checkConflict(!requestManager.cleanupRequestExists(request.getId()), "Request %s is currently cleaning. Try again after a few moments", request.getId());
    Optional<SingularityPendingDeploy> maybePendingDeploy = deployManager.getPendingDeploy(request.getId());
    checkConflict(!(maybePendingDeploy.isPresent() && maybePendingDeploy.get().getUpdatedRequest().isPresent()), "Request %s has a pending deploy that may change the request data. Try again when the deploy has finished", request.getId());
    Optional<SingularityRequest> oldRequest = oldRequestWithState.isPresent() ? Optional.of(oldRequestWithState.get().getRequest()) : Optional.<SingularityRequest>absent();
    if (oldRequest.isPresent()) {
        authorizationHelper.checkForAuthorization(oldRequest.get(), user, SingularityAuthorizationScope.WRITE);
        authorizationHelper.checkForAuthorizedChanges(request, oldRequest.get(), user);
        validator.checkActionEnabled(SingularityAction.UPDATE_REQUEST);
    } else {
        validator.checkActionEnabled(SingularityAction.CREATE_REQUEST);
    }
    if (request.getSlavePlacement().isPresent() && request.getSlavePlacement().get() == SlavePlacement.SPREAD_ALL_SLAVES) {
        checkBadRequest(validator.isSpreadAllSlavesEnabled(), "You must enabled spread to all slaves in order to use the SPREAD_ALL_SLAVES request type");
        int currentActiveSlaveCount = slaveManager.getNumObjectsAtState(MachineState.ACTIVE);
        request = request.toBuilder().setInstances(Optional.of(currentActiveSlaveCount)).build();
    }
    if (!oldRequest.isPresent() || !(oldRequest.get().getInstancesSafe() == request.getInstancesSafe())) {
        validator.checkScale(request, Optional.<Integer>absent());
    }
    authorizationHelper.checkForAuthorization(request, user, SingularityAuthorizationScope.WRITE);
    RequestState requestState = RequestState.ACTIVE;
    if (oldRequestWithState.isPresent()) {
        requestState = oldRequestWithState.get().getState();
    }
    requestHelper.updateRequest(request, oldRequest, requestState, historyType, user.getEmail(), skipHealthchecks, message, maybeBounceRequest);
}

Example 3 with RequestState

use of com.hubspot.singularity.RequestState in project Singularity by HubSpot.

the class SingularityScheduler method handlePendingRequestsForDeployKey.

private void handlePendingRequestsForDeployKey(AtomicInteger obsoleteRequests, AtomicInteger heldForScheduledActiveTask, AtomicInteger totalNewScheduledTasks, SingularityDeployKey deployKey, List<SingularityPendingRequest> pendingRequestsForDeploy) {
    final String requestId = deployKey.getRequestId();
    final Optional<SingularityRequestWithState> maybeRequest = requestManager.getRequest(requestId);
    final SingularityDeployStatistics deployStatistics = getDeployStatistics(deployKey.getRequestId(), deployKey.getDeployId());
    if (!isRequestActive(maybeRequest)) {
        LOG.debug("Pending request {} was obsolete (request {})", requestId, SingularityRequestWithState.getRequestState(maybeRequest));
        obsoleteRequests.getAndIncrement();
        for (SingularityPendingRequest pendingRequest : pendingRequestsForDeploy) {
            requestManager.deletePendingRequest(pendingRequest);
        }
        return;
    }
    SingularityRequestWithState request = maybeRequest.get();
    Optional<SingularityRequestDeployState> maybeRequestDeployState = deployManager.getRequestDeployState(requestId);
    Optional<SingularityPendingDeploy> maybePendingDeploy = deployManager.getPendingDeploy(requestId);
    List<SingularityTaskId> matchingTaskIds = getMatchingTaskIds(request.getRequest(), deployKey);
    List<SingularityPendingRequest> effectivePendingRequests = new ArrayList<>();
    // Things that are closest to now (ie smaller timestamps) should come first in the queue
    pendingRequestsForDeploy.sort(Comparator.comparingLong(SingularityPendingRequest::getTimestamp));
    int scheduledTasks = 0;
    for (SingularityPendingRequest pendingRequest : pendingRequestsForDeploy) {
        final SingularityRequest updatedRequest = updatedRequest(maybePendingDeploy, pendingRequest, request);
        if (!shouldScheduleTasks(updatedRequest, pendingRequest, maybePendingDeploy, maybeRequestDeployState)) {
            LOG.debug("Pending request {} was obsolete (request {})", pendingRequest, SingularityRequestWithState.getRequestState(maybeRequest));
            obsoleteRequests.getAndIncrement();
            requestManager.deletePendingRequest(pendingRequest);
            continue;
        }
        int missingInstances = getNumMissingInstances(matchingTaskIds, updatedRequest, pendingRequest, maybePendingDeploy);
        if (missingInstances == 0 && !matchingTaskIds.isEmpty() && updatedRequest.isScheduled() && pendingRequest.getPendingType() == PendingType.NEW_DEPLOY) {
            LOG.trace("Holding pending request {} because it is scheduled and has an active task", pendingRequest);
            heldForScheduledActiveTask.getAndIncrement();
            continue;
        }
        if (effectivePendingRequests.isEmpty()) {
            effectivePendingRequests.add(pendingRequest);
            RequestState requestState = checkCooldown(request.getState(), request.getRequest(), deployStatistics);
            scheduledTasks += scheduleTasks(request.getRequest(), requestState, deployStatistics, pendingRequest, matchingTaskIds, maybePendingDeploy);
            requestManager.deletePendingRequest(pendingRequest);
        } else if (pendingRequest.getPendingType() == PendingType.IMMEDIATE) {
            effectivePendingRequests.add(pendingRequest);
            RequestState requestState = checkCooldown(request.getState(), request.getRequest(), deployStatistics);
            scheduledTasks += scheduleTasks(request.getRequest(), requestState, deployStatistics, pendingRequest, matchingTaskIds, maybePendingDeploy);
            requestManager.deletePendingRequest(pendingRequest);
        } else if (pendingRequest.getPendingType() == PendingType.ONEOFF) {
            effectivePendingRequests.add(pendingRequest);
            RequestState requestState = checkCooldown(request.getState(), request.getRequest(), deployStatistics);
            scheduledTasks += scheduleTasks(request.getRequest(), requestState, deployStatistics, pendingRequest, matchingTaskIds, maybePendingDeploy);
            requestManager.deletePendingRequest(pendingRequest);
        } else if (updatedRequest.isScheduled() && (pendingRequest.getPendingType() == PendingType.NEW_DEPLOY || pendingRequest.getPendingType() == PendingType.TASK_DONE)) {
            // If we are here, there is already an immediate of run of the scheduled task launched. Drop anything that would
            // leave a second instance of the request in the pending queue.
            requestManager.deletePendingRequest(pendingRequest);
        }
    // Any other subsequent requests are not honored until after the pending queue is cleared.
    }
    totalNewScheduledTasks.getAndAdd(scheduledTasks);
}

Example 4 with RequestState

use of com.hubspot.singularity.RequestState in project Singularity by HubSpot.

the class SingularityDeploysTest method testUnpauseOnDeploy.

@Test
public void testUnpauseOnDeploy() {
    initRequest();
    initFirstDeploy();
    requestManager.pause(request, System.currentTimeMillis(), Optional.<String>absent(), Optional.<String>absent());
    boolean exception = false;
    try {
        deploy("d2");
    } catch (Exception e) {
        exception = true;
    }
    Assert.assertTrue(exception);
    deploy("d3", Optional.of(true));
    Assert.assertTrue(requestManager.getRequest(requestId).get().getState() == RequestState.DEPLOYING_TO_UNPAUSE);
    scheduler.drainPendingQueue();
    sms.resourceOffers(Arrays.asList(createOffer(20, 20000, 50000, "slave1", "host1")));
    statusUpdate(taskManager.getActiveTasks().get(0), TaskState.TASK_FAILED);
    deployChecker.checkDeploys();
    Assert.assertTrue(requestManager.getRequest(requestId).get().getState() == RequestState.PAUSED);
    Assert.assertTrue(taskManager.getActiveTaskIds().isEmpty());
    Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty());
    Assert.assertTrue(requestManager.getPendingRequests().isEmpty());
    deploy("d4", Optional.of(true));
    Assert.assertTrue(requestManager.getRequest(requestId).get().getState() == RequestState.DEPLOYING_TO_UNPAUSE);
    scheduler.drainPendingQueue();
    sms.resourceOffers(Arrays.asList(createOffer(20, 20000, 50000, "slave1", "host1")));
    statusUpdate(taskManager.getActiveTasks().get(0), TaskState.TASK_RUNNING);
    deployChecker.checkDeploys();
    RequestState requestState = requestManager.getRequest(requestId).get().getState();
    Assert.assertTrue(requestState == RequestState.ACTIVE);
}