Examples with SingularityRequestWithState com.hubspot.singularity.SingularityRequestWithState used on opensource projects

Example 6 with SingularityRequestWithState

use of com.hubspot.singularity.SingularityRequestWithState in project Singularity by HubSpot.

the class SingularityCooldownChecker method checkCooldowns.

public void checkCooldowns() {
    final long start = System.currentTimeMillis();
    final List<SingularityRequestWithState> cooldownRequests = Lists.newArrayList(requestManager.getCooldownRequests(false));
    if (cooldownRequests.isEmpty()) {
        LOG.trace("No cooldown requests");
        return;
    }
    AtomicInteger exitedCooldown = new AtomicInteger(0);
    cooldownRequests.parallelStream().forEach((cooldownRequest) -> {
        lock.runWithRequestLock(() -> {
            if (checkCooldown(cooldownRequest)) {
                exitedCooldown.getAndIncrement();
            }
        }, cooldownRequest.getRequest().getId(), getClass().getSimpleName());
    });
    LOG.info("{} out of {} cooldown requests exited cooldown in {}", exitedCooldown.get(), cooldownRequests.size(), JavaUtils.duration(start));
}

Example 7 with SingularityRequestWithState

use of com.hubspot.singularity.SingularityRequestWithState in project Singularity by HubSpot.

the class SingularityDeployChecker method checkDeploy.

private void checkDeploy(final SingularityPendingDeploy pendingDeploy, final List<SingularityDeployMarker> cancelDeploys, final Map<SingularityPendingDeploy, SingularityDeployKey> pendingDeployToKey, final Map<SingularityDeployKey, SingularityDeploy> deployKeyToDeploy, List<SingularityUpdatePendingDeployRequest> updateRequests) {
    final SingularityDeployKey deployKey = pendingDeployToKey.get(pendingDeploy);
    final Optional<SingularityDeploy> deploy = Optional.fromNullable(deployKeyToDeploy.get(deployKey));
    Optional<SingularityRequestWithState> maybeRequestWithState = requestManager.getRequest(pendingDeploy.getDeployMarker().getRequestId());
    if (!(maybeRequestWithState.isPresent() && maybeRequestWithState.get().getState() == RequestState.FINISHED) && !(configuration.isAllowDeployOfPausedRequests() && maybeRequestWithState.isPresent() && maybeRequestWithState.get().getState() == RequestState.PAUSED) && !SingularityRequestWithState.isActive(maybeRequestWithState)) {
        LOG.warn("Deploy {} request was {}, removing deploy", pendingDeploy, SingularityRequestWithState.getRequestState(maybeRequestWithState));
        if (shouldCancelLoadBalancer(pendingDeploy)) {
            cancelLoadBalancer(pendingDeploy, SingularityDeployFailure.deployRemoved());
        }
        failPendingDeployDueToState(pendingDeploy, maybeRequestWithState, deploy);
        return;
    }
    final SingularityDeployMarker pendingDeployMarker = pendingDeploy.getDeployMarker();
    final Optional<SingularityDeployMarker> cancelRequest = findCancel(cancelDeploys, pendingDeployMarker);
    final Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest = findUpdateRequest(updateRequests, pendingDeploy);
    final SingularityRequestWithState requestWithState = maybeRequestWithState.get();
    final SingularityRequest request = pendingDeploy.getUpdatedRequest().or(requestWithState.getRequest());
    final List<SingularityTaskId> requestTasks = taskManager.getTaskIdsForRequest(request.getId());
    final List<SingularityTaskId> activeTasks = taskManager.filterActiveTaskIds(requestTasks);
    final List<SingularityTaskId> inactiveDeployMatchingTasks = new ArrayList<>(requestTasks.size());
    for (SingularityTaskId taskId : requestTasks) {
        if (taskId.getDeployId().equals(pendingDeployMarker.getDeployId()) && !activeTasks.contains(taskId)) {
            inactiveDeployMatchingTasks.add(taskId);
        }
    }
    final List<SingularityTaskId> deployMatchingTasks = new ArrayList<>(activeTasks.size());
    final List<SingularityTaskId> allOtherMatchingTasks = new ArrayList<>(activeTasks.size());
    for (SingularityTaskId taskId : activeTasks) {
        if (taskId.getDeployId().equals(pendingDeployMarker.getDeployId())) {
            deployMatchingTasks.add(taskId);
        } else {
            allOtherMatchingTasks.add(taskId);
        }
    }
    SingularityDeployResult deployResult = getDeployResult(request, requestWithState.getState(), cancelRequest, pendingDeploy, updatePendingDeployRequest, deploy, deployMatchingTasks, allOtherMatchingTasks, inactiveDeployMatchingTasks);
    LOG.info("Deploy {} had result {} after {}", pendingDeployMarker, deployResult, JavaUtils.durationFromMillis(System.currentTimeMillis() - pendingDeployMarker.getTimestamp()));
    if (deployResult.getDeployState() == DeployState.SUCCEEDED) {
        if (saveNewDeployState(pendingDeployMarker, Optional.of(pendingDeployMarker))) {
            if (!(request.getRequestType() == RequestType.RUN_ONCE)) {
                deleteObsoletePendingTasks(pendingDeploy);
            }
            finishDeploy(requestWithState, deploy, pendingDeploy, allOtherMatchingTasks, deployResult);
            return;
        } else {
            LOG.warn("Failing deploy {} because it failed to save deploy state", pendingDeployMarker);
            deployResult = new SingularityDeployResult(DeployState.FAILED_INTERNAL_STATE, Optional.of(String.format("Deploy had state %s but failed to persist it correctly", deployResult.getDeployState())), deployResult.getLbUpdate(), SingularityDeployFailure.failedToSave(), deployResult.getTimestamp());
        }
    } else if (!deployResult.getDeployState().isDeployFinished()) {
        return;
    }
    // success case is handled, handle failure cases:
    saveNewDeployState(pendingDeployMarker, Optional.<SingularityDeployMarker>absent());
    finishDeploy(requestWithState, deploy, pendingDeploy, deployMatchingTasks, deployResult);
}

Example 8 with SingularityRequestWithState

use of com.hubspot.singularity.SingularityRequestWithState in project Singularity by HubSpot.

the class SingularityMailPoller method checkToSendTaskFinishedMail.

private void checkToSendTaskFinishedMail(SingularityTaskId taskId) {
    Optional<SingularityRequestWithState> requestWithState = requestManager.getRequest(taskId.getRequestId());
    Optional<SingularityTaskHistory> taskHistory = taskManager.getTaskHistory(taskId);
    ShouldSendMailState shouldSendState = shouldSendTaskFinishedMail(taskId, requestWithState, taskHistory);
    if (shouldSendState == ShouldSendMailState.WAIT) {
        return;
    }
    try {
        mailer.sendTaskCompletedMail(taskHistory.get(), requestWithState.get().getRequest());
    } catch (Throwable t) {
        LOG.error("While trying to send task completed mail for {}", taskId, t);
    } finally {
        SingularityDeleteResult result = taskManager.deleteFinishedTaskMailQueue(taskId);
        LOG.debug("Task {} mail sent with status {} (delete result {})", taskId, shouldSendState, result);
    }
}

Example 9 with SingularityRequestWithState

use of com.hubspot.singularity.SingularityRequestWithState in project Singularity by HubSpot.

the class SingularityScheduler method handleCompletedTaskWithStatistics.

private Optional<PendingType> handleCompletedTaskWithStatistics(Optional<SingularityTask> task, SingularityTaskId taskId, long timestamp, ExtendedTaskState state, SingularityDeployStatistics deployStatistics, SingularityCreateResult taskHistoryUpdateCreateResult, Protos.TaskStatus status) {
    final Optional<SingularityRequestWithState> maybeRequestWithState = requestManager.getRequest(taskId.getRequestId());
    final Optional<SingularityPendingDeploy> maybePendingDeploy = deployManager.getPendingDeploy(taskId.getRequestId());
    if (!isRequestActive(maybeRequestWithState)) {
        LOG.warn("Not scheduling a new task, {} is {}", taskId.getRequestId(), SingularityRequestWithState.getRequestState(maybeRequestWithState));
        return Optional.absent();
    }
    RequestState requestState = maybeRequestWithState.get().getState();
    final SingularityRequest request = maybePendingDeploy.isPresent() ? maybePendingDeploy.get().getUpdatedRequest().or(maybeRequestWithState.get().getRequest()) : maybeRequestWithState.get().getRequest();
    final Optional<SingularityRequestDeployState> requestDeployState = deployManager.getRequestDeployState(request.getId());
    if (!isDeployInUse(requestDeployState, taskId.getDeployId(), true)) {
        LOG.debug("Task {} completed, but it didn't match active deploy state {} - ignoring", taskId.getId(), requestDeployState);
        return Optional.absent();
    }
    if (taskHistoryUpdateCreateResult == SingularityCreateResult.CREATED && requestState != RequestState.SYSTEM_COOLDOWN) {
        mailer.queueTaskCompletedMail(task, taskId, request, state);
    } else if (requestState == RequestState.SYSTEM_COOLDOWN) {
        LOG.debug("Not sending a task completed email because task {} is in SYSTEM_COOLDOWN", taskId);
    } else {
        LOG.debug("Not sending a task completed email for task {} because Singularity already processed this update", taskId);
    }
    if (!status.hasReason() || !status.getReason().equals(Reason.REASON_INVALID_OFFERS)) {
        if (!state.isSuccess() && taskHistoryUpdateCreateResult == SingularityCreateResult.CREATED && cooldown.shouldEnterCooldown(request, taskId, requestState, deployStatistics, timestamp)) {
            LOG.info("Request {} is entering cooldown due to task {}", request.getId(), taskId);
            requestState = RequestState.SYSTEM_COOLDOWN;
            requestManager.cooldown(request, System.currentTimeMillis());
            mailer.sendRequestInCooldownMail(request);
        }
    } else {
        LOG.debug("Not triggering cooldown due to TASK_LOST from invalid offers for request {}", request.getId());
    }
    PendingType pendingType = PendingType.TASK_DONE;
    Optional<List<String>> cmdLineArgsList = Optional.absent();
    if (!state.isSuccess() && shouldRetryImmediately(request, deployStatistics, task)) {
        LOG.debug("Retrying {} because {}", request.getId(), state);
        pendingType = PendingType.RETRY;
        if (task.isPresent()) {
            cmdLineArgsList = task.get().getTaskRequest().getPendingTask().getCmdLineArgsList();
        }
    } else if (!request.isAlwaysRunning()) {
        return Optional.absent();
    }
    if (state.isSuccess() && requestState == RequestState.SYSTEM_COOLDOWN) {
        // TODO send not cooldown anymore email
        LOG.info("Request {} succeeded a task, removing from cooldown", request.getId());
        requestManager.exitCooldown(request, System.currentTimeMillis(), Optional.<String>absent(), Optional.<String>absent());
    }
    SingularityPendingRequest pendingRequest = new SingularityPendingRequest(request.getId(), requestDeployState.get().getActiveDeploy().get().getDeployId(), System.currentTimeMillis(), Optional.absent(), pendingType, cmdLineArgsList, Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent());
    requestManager.addToPendingQueue(pendingRequest);
    return Optional.of(pendingType);
}

Example 10 with SingularityRequestWithState

use of com.hubspot.singularity.SingularityRequestWithState in project Singularity by HubSpot.

the class RequestResource method bounce.

public SingularityRequestParent bounce(String requestId, Optional<SingularityBounceRequest> bounceRequest, SingularityUser user) {
    SingularityRequestWithState requestWithState = fetchRequestWithState(requestId, user);
    authorizationHelper.checkForAuthorization(requestWithState.getRequest(), user, SingularityAuthorizationScope.WRITE);
    validator.checkActionEnabled(SingularityAction.BOUNCE_REQUEST);
    checkBadRequest(requestWithState.getRequest().isLongRunning(), "Can not bounce a %s request (%s)", requestWithState.getRequest().getRequestType(), requestWithState);
    checkConflict(requestWithState.getState() != RequestState.PAUSED, "Request %s is paused. Unable to bounce (it must be manually unpaused first)", requestWithState.getRequest().getId());
    final boolean isIncrementalBounce = bounceRequest.isPresent() && bounceRequest.get().getIncremental().or(false);
    validator.checkResourcesForBounce(requestWithState.getRequest(), isIncrementalBounce);
    validator.checkRequestForPriorityFreeze(requestWithState.getRequest());
    final Optional<Boolean> skipHealthchecks = bounceRequest.isPresent() ? bounceRequest.get().getSkipHealthchecks() : Optional.<Boolean>absent();
    Optional<String> message = Optional.absent();
    Optional<String> actionId = Optional.absent();
    Optional<SingularityShellCommand> runBeforeKill = Optional.absent();
    if (bounceRequest.isPresent()) {
        actionId = bounceRequest.get().getActionId();
        message = bounceRequest.get().getMessage();
        if (bounceRequest.get().getRunShellCommandBeforeKill().isPresent()) {
            validator.checkValidShellCommand(bounceRequest.get().getRunShellCommandBeforeKill().get());
            runBeforeKill = bounceRequest.get().getRunShellCommandBeforeKill();
        }
    }
    if (!actionId.isPresent()) {
        actionId = Optional.of(UUID.randomUUID().toString());
    }
    final String deployId = getAndCheckDeployId(requestId);
    checkConflict(!(requestManager.markAsBouncing(requestId) == SingularityCreateResult.EXISTED), "%s is already bouncing", requestId);
    requestManager.createCleanupRequest(new SingularityRequestCleanup(user.getEmail(), isIncrementalBounce ? RequestCleanupType.INCREMENTAL_BOUNCE : RequestCleanupType.BOUNCE, System.currentTimeMillis(), Optional.absent(), Optional.absent(), requestId, Optional.of(deployId), skipHealthchecks, message, actionId, runBeforeKill));
    requestManager.bounce(requestWithState.getRequest(), System.currentTimeMillis(), Optional.of(user.getId()), message);
    final SingularityBounceRequest validatedBounceRequest = validator.checkBounceRequest(bounceRequest.or(SingularityBounceRequest.defaultRequest()));
    requestManager.saveExpiringObject(new SingularityExpiringBounce(requestId, deployId, Optional.of(user.getId()), System.currentTimeMillis(), validatedBounceRequest, actionId.get()));
    return fillEntireRequest(requestWithState);
}