Examples with SingularityRequest com.hubspot.singularity.SingularityRequest used on opensource projects

Example 36 with SingularityRequest

use of com.hubspot.singularity.SingularityRequest in project Singularity by HubSpot.

the class ValidatorTest method whenRunNowIfNoRunIdSetItWillGenerateAnId.

@Test
public void whenRunNowIfNoRunIdSetItWillGenerateAnId() {
    String deployID = "deploy";
    Optional<String> userEmail = Optional.absent();
    SingularityRequest request = new SingularityRequestBuilder("request2", RequestType.ON_DEMAND).build();
    Optional<SingularityRunNowRequest> runNowRequest = Optional.of(runNowRequest());
    List<SingularityTaskId> activeTasks = Collections.emptyList();
    List<SingularityPendingTaskId> pendingTasks = Collections.emptyList();
    SingularityPendingRequest pendingRequest = validator.checkRunNowRequest(deployID, userEmail, request, runNowRequest, activeTasks, pendingTasks);
    Assert.assertTrue(pendingRequest.getRunId().isPresent());
}

Example 37 with SingularityRequest

use of com.hubspot.singularity.SingularityRequest in project Singularity by HubSpot.

the class SingularityDeployChecker method finishDeploy.

private void finishDeploy(SingularityRequestWithState requestWithState, Optional<SingularityDeploy> deploy, SingularityPendingDeploy pendingDeploy, Iterable<SingularityTaskId> tasksToKill, SingularityDeployResult deployResult) {
    SingularityRequest request = requestWithState.getRequest();
    if (!request.isOneOff() && !(request.getRequestType() == RequestType.RUN_ONCE)) {
        cleanupTasks(pendingDeploy, request, deployResult, tasksToKill);
    }
    if (deploy.isPresent() && deploy.get().getRunImmediately().isPresent()) {
        String requestId = deploy.get().getRequestId();
        String deployId = deploy.get().getId();
        SingularityRunNowRequest runNowRequest = deploy.get().getRunImmediately().get();
        List<SingularityTaskId> activeTasks = taskManager.getActiveTaskIdsForRequest(requestId);
        List<SingularityPendingTaskId> pendingTasks = taskManager.getPendingTaskIdsForRequest(requestId);
        SingularityPendingRequestBuilder builder = new SingularityPendingRequestBuilder().setRequestId(requestId).setDeployId(deployId).setTimestamp(deployResult.getTimestamp()).setUser(pendingDeploy.getDeployMarker().getUser()).setCmdLineArgsList(runNowRequest.getCommandLineArgs()).setRunId(runNowRequest.getRunId().or(Optional.of(UUID.randomUUID().toString()))).setSkipHealthchecks(runNowRequest.getSkipHealthchecks().or(request.getSkipHealthchecks())).setMessage(runNowRequest.getMessage().or(pendingDeploy.getDeployMarker().getMessage())).setResources(runNowRequest.getResources()).setRunAsUserOverride(runNowRequest.getRunAsUserOverride()).setEnvOverrides(runNowRequest.getEnvOverrides()).setExtraArtifacts(runNowRequest.getExtraArtifacts()).setRunAt(runNowRequest.getRunAt());
        PendingType pendingType = null;
        if (request.isScheduled()) {
            if (activeTasks.isEmpty()) {
                pendingType = PendingType.IMMEDIATE;
            } else {
                // Don't run scheduled task over a running task. Will be picked up on the next run.
                pendingType = PendingType.NEW_DEPLOY;
            }
        } else if (!request.isLongRunning()) {
            if (request.getInstances().isPresent() && (activeTasks.size() + pendingTasks.size() < request.getInstances().get())) {
                pendingType = PendingType.ONEOFF;
            } else {
                // Don't run one-off / on-demand task when already at instance count cap
                pendingType = PendingType.NEW_DEPLOY;
            }
        }
        if (pendingType != null) {
            builder.setPendingType(canceledOr(deployResult.getDeployState(), pendingType));
            requestManager.addToPendingQueue(builder.build());
        } else {
            LOG.warn("Could not determine pending type for deploy {}.", deployId);
        }
    } else if (!request.isDeployable() && !request.isOneOff()) {
        PendingType pendingType = canceledOr(deployResult.getDeployState(), PendingType.NEW_DEPLOY);
        requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), pendingType, deploy.isPresent() ? deploy.get().getSkipHealthchecksOnDeploy() : Optional.absent(), pendingDeploy.getDeployMarker().getMessage()));
    }
    if (deployResult.getDeployState() == DeployState.SUCCEEDED) {
        if (request.isDeployable() && !request.isOneOff()) {
            // remove the lock on bounces in case we deployed during a bounce
            requestManager.markBounceComplete(request.getId());
        }
        if (requestWithState.getState() == RequestState.FINISHED) {
            // A FINISHED request is moved to ACTIVE state so we can reevaluate the schedule
            requestManager.activate(request, RequestHistoryType.UPDATED, System.currentTimeMillis(), deploy.isPresent() ? deploy.get().getUser() : Optional.absent(), Optional.absent());
        }
    }
    deployManager.saveDeployResult(pendingDeploy.getDeployMarker(), deploy, deployResult);
    if (request.isDeployable() && (deployResult.getDeployState() == DeployState.CANCELED || deployResult.getDeployState() == DeployState.FAILED || deployResult.getDeployState() == DeployState.OVERDUE)) {
        Optional<SingularityRequestDeployState> maybeRequestDeployState = deployManager.getRequestDeployState(request.getId());
        if (maybeRequestDeployState.isPresent() && maybeRequestDeployState.get().getActiveDeploy().isPresent() && !(requestWithState.getState() == RequestState.PAUSED || requestWithState.getState() == RequestState.DEPLOYING_TO_UNPAUSE)) {
            requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), maybeRequestDeployState.get().getActiveDeploy().get().getDeployId(), deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), deployResult.getDeployState() == DeployState.CANCELED ? PendingType.DEPLOY_CANCELLED : PendingType.DEPLOY_FAILED, request.getSkipHealthchecks(), pendingDeploy.getDeployMarker().getMessage()));
        }
    }
    if (request.isDeployable() && deployResult.getDeployState() == DeployState.SUCCEEDED && pendingDeploy.getDeployProgress().isPresent() && requestWithState.getState() != RequestState.PAUSED) {
        if (pendingDeploy.getDeployProgress().get().getTargetActiveInstances() != request.getInstancesSafe()) {
            requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), PendingType.UPDATED_REQUEST, request.getSkipHealthchecks(), pendingDeploy.getDeployMarker().getMessage()));
        }
    }
    if (requestWithState.getState() == RequestState.DEPLOYING_TO_UNPAUSE) {
        if (deployResult.getDeployState() == DeployState.SUCCEEDED) {
            requestManager.activate(request, RequestHistoryType.DEPLOYED_TO_UNPAUSE, deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), Optional.<String>absent());
            requestManager.deleteExpiringObject(SingularityExpiringPause.class, request.getId());
        } else {
            requestManager.pause(request, deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), Optional.<String>absent());
        }
    }
    if (pendingDeploy.getUpdatedRequest().isPresent() && deployResult.getDeployState() == DeployState.SUCCEEDED) {
        requestManager.update(pendingDeploy.getUpdatedRequest().get(), System.currentTimeMillis(), pendingDeploy.getDeployMarker().getUser(), Optional.<String>absent());
        requestManager.deleteExpiringObject(SingularityExpiringScale.class, request.getId());
    }
    removePendingDeploy(pendingDeploy);
}

Example 38 with SingularityRequest

use of com.hubspot.singularity.SingularityRequest in project Singularity by HubSpot.

the class SingularityDeployChecker method checkDeploy.

private void checkDeploy(final SingularityPendingDeploy pendingDeploy, final List<SingularityDeployMarker> cancelDeploys, final Map<SingularityPendingDeploy, SingularityDeployKey> pendingDeployToKey, final Map<SingularityDeployKey, SingularityDeploy> deployKeyToDeploy, List<SingularityUpdatePendingDeployRequest> updateRequests) {
    final SingularityDeployKey deployKey = pendingDeployToKey.get(pendingDeploy);
    final Optional<SingularityDeploy> deploy = Optional.fromNullable(deployKeyToDeploy.get(deployKey));
    Optional<SingularityRequestWithState> maybeRequestWithState = requestManager.getRequest(pendingDeploy.getDeployMarker().getRequestId());
    if (!(maybeRequestWithState.isPresent() && maybeRequestWithState.get().getState() == RequestState.FINISHED) && !(configuration.isAllowDeployOfPausedRequests() && maybeRequestWithState.isPresent() && maybeRequestWithState.get().getState() == RequestState.PAUSED) && !SingularityRequestWithState.isActive(maybeRequestWithState)) {
        LOG.warn("Deploy {} request was {}, removing deploy", pendingDeploy, SingularityRequestWithState.getRequestState(maybeRequestWithState));
        if (shouldCancelLoadBalancer(pendingDeploy)) {
            cancelLoadBalancer(pendingDeploy, SingularityDeployFailure.deployRemoved());
        }
        failPendingDeployDueToState(pendingDeploy, maybeRequestWithState, deploy);
        return;
    }
    final SingularityDeployMarker pendingDeployMarker = pendingDeploy.getDeployMarker();
    final Optional<SingularityDeployMarker> cancelRequest = findCancel(cancelDeploys, pendingDeployMarker);
    final Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest = findUpdateRequest(updateRequests, pendingDeploy);
    final SingularityRequestWithState requestWithState = maybeRequestWithState.get();
    final SingularityRequest request = pendingDeploy.getUpdatedRequest().or(requestWithState.getRequest());
    final List<SingularityTaskId> requestTasks = taskManager.getTaskIdsForRequest(request.getId());
    final List<SingularityTaskId> activeTasks = taskManager.filterActiveTaskIds(requestTasks);
    final List<SingularityTaskId> inactiveDeployMatchingTasks = new ArrayList<>(requestTasks.size());
    for (SingularityTaskId taskId : requestTasks) {
        if (taskId.getDeployId().equals(pendingDeployMarker.getDeployId()) && !activeTasks.contains(taskId)) {
            inactiveDeployMatchingTasks.add(taskId);
        }
    }
    final List<SingularityTaskId> deployMatchingTasks = new ArrayList<>(activeTasks.size());
    final List<SingularityTaskId> allOtherMatchingTasks = new ArrayList<>(activeTasks.size());
    for (SingularityTaskId taskId : activeTasks) {
        if (taskId.getDeployId().equals(pendingDeployMarker.getDeployId())) {
            deployMatchingTasks.add(taskId);
        } else {
            allOtherMatchingTasks.add(taskId);
        }
    }
    SingularityDeployResult deployResult = getDeployResult(request, requestWithState.getState(), cancelRequest, pendingDeploy, updatePendingDeployRequest, deploy, deployMatchingTasks, allOtherMatchingTasks, inactiveDeployMatchingTasks);
    LOG.info("Deploy {} had result {} after {}", pendingDeployMarker, deployResult, JavaUtils.durationFromMillis(System.currentTimeMillis() - pendingDeployMarker.getTimestamp()));
    if (deployResult.getDeployState() == DeployState.SUCCEEDED) {
        if (saveNewDeployState(pendingDeployMarker, Optional.of(pendingDeployMarker))) {
            if (!(request.getRequestType() == RequestType.RUN_ONCE)) {
                deleteObsoletePendingTasks(pendingDeploy);
            }
            finishDeploy(requestWithState, deploy, pendingDeploy, allOtherMatchingTasks, deployResult);
            return;
        } else {
            LOG.warn("Failing deploy {} because it failed to save deploy state", pendingDeployMarker);
            deployResult = new SingularityDeployResult(DeployState.FAILED_INTERNAL_STATE, Optional.of(String.format("Deploy had state %s but failed to persist it correctly", deployResult.getDeployState())), deployResult.getLbUpdate(), SingularityDeployFailure.failedToSave(), deployResult.getTimestamp());
        }
    } else if (!deployResult.getDeployState().isDeployFinished()) {
        return;
    }
    // success case is handled, handle failure cases:
    saveNewDeployState(pendingDeployMarker, Optional.<SingularityDeployMarker>absent());
    finishDeploy(requestWithState, deploy, pendingDeploy, deployMatchingTasks, deployResult);
}

Example 39 with SingularityRequest

use of com.hubspot.singularity.SingularityRequest in project Singularity by HubSpot.

the class SingularityScheduler method handleCompletedTaskWithStatistics.

private Optional<PendingType> handleCompletedTaskWithStatistics(Optional<SingularityTask> task, SingularityTaskId taskId, long timestamp, ExtendedTaskState state, SingularityDeployStatistics deployStatistics, SingularityCreateResult taskHistoryUpdateCreateResult, Protos.TaskStatus status) {
    final Optional<SingularityRequestWithState> maybeRequestWithState = requestManager.getRequest(taskId.getRequestId());
    final Optional<SingularityPendingDeploy> maybePendingDeploy = deployManager.getPendingDeploy(taskId.getRequestId());
    if (!isRequestActive(maybeRequestWithState)) {
        LOG.warn("Not scheduling a new task, {} is {}", taskId.getRequestId(), SingularityRequestWithState.getRequestState(maybeRequestWithState));
        return Optional.absent();
    }
    RequestState requestState = maybeRequestWithState.get().getState();
    final SingularityRequest request = maybePendingDeploy.isPresent() ? maybePendingDeploy.get().getUpdatedRequest().or(maybeRequestWithState.get().getRequest()) : maybeRequestWithState.get().getRequest();
    final Optional<SingularityRequestDeployState> requestDeployState = deployManager.getRequestDeployState(request.getId());
    if (!isDeployInUse(requestDeployState, taskId.getDeployId(), true)) {
        LOG.debug("Task {} completed, but it didn't match active deploy state {} - ignoring", taskId.getId(), requestDeployState);
        return Optional.absent();
    }
    if (taskHistoryUpdateCreateResult == SingularityCreateResult.CREATED && requestState != RequestState.SYSTEM_COOLDOWN) {
        mailer.queueTaskCompletedMail(task, taskId, request, state);
    } else if (requestState == RequestState.SYSTEM_COOLDOWN) {
        LOG.debug("Not sending a task completed email because task {} is in SYSTEM_COOLDOWN", taskId);
    } else {
        LOG.debug("Not sending a task completed email for task {} because Singularity already processed this update", taskId);
    }
    if (!status.hasReason() || !status.getReason().equals(Reason.REASON_INVALID_OFFERS)) {
        if (!state.isSuccess() && taskHistoryUpdateCreateResult == SingularityCreateResult.CREATED && cooldown.shouldEnterCooldown(request, taskId, requestState, deployStatistics, timestamp)) {
            LOG.info("Request {} is entering cooldown due to task {}", request.getId(), taskId);
            requestState = RequestState.SYSTEM_COOLDOWN;
            requestManager.cooldown(request, System.currentTimeMillis());
            mailer.sendRequestInCooldownMail(request);
        }
    } else {
        LOG.debug("Not triggering cooldown due to TASK_LOST from invalid offers for request {}", request.getId());
    }
    PendingType pendingType = PendingType.TASK_DONE;
    Optional<List<String>> cmdLineArgsList = Optional.absent();
    if (!state.isSuccess() && shouldRetryImmediately(request, deployStatistics, task)) {
        LOG.debug("Retrying {} because {}", request.getId(), state);
        pendingType = PendingType.RETRY;
        if (task.isPresent()) {
            cmdLineArgsList = task.get().getTaskRequest().getPendingTask().getCmdLineArgsList();
        }
    } else if (!request.isAlwaysRunning()) {
        return Optional.absent();
    }
    if (state.isSuccess() && requestState == RequestState.SYSTEM_COOLDOWN) {
        // TODO send not cooldown anymore email
        LOG.info("Request {} succeeded a task, removing from cooldown", request.getId());
        requestManager.exitCooldown(request, System.currentTimeMillis(), Optional.<String>absent(), Optional.<String>absent());
    }
    SingularityPendingRequest pendingRequest = new SingularityPendingRequest(request.getId(), requestDeployState.get().getActiveDeploy().get().getDeployId(), System.currentTimeMillis(), Optional.absent(), pendingType, cmdLineArgsList, Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent());
    requestManager.addToPendingQueue(pendingRequest);
    return Optional.of(pendingType);
}

Example 40 with SingularityRequest

use of com.hubspot.singularity.SingularityRequest in project Singularity by HubSpot.

the class RequestResource method submitRequest.

private void submitRequest(SingularityRequest request, Optional<SingularityRequestWithState> oldRequestWithState, Optional<RequestHistoryType> historyType, Optional<Boolean> skipHealthchecks, Optional<String> message, Optional<SingularityBounceRequest> maybeBounceRequest, SingularityUser user) {
    checkNotNullBadRequest(request.getId(), "Request must have an id");
    checkConflict(!requestManager.cleanupRequestExists(request.getId()), "Request %s is currently cleaning. Try again after a few moments", request.getId());
    Optional<SingularityPendingDeploy> maybePendingDeploy = deployManager.getPendingDeploy(request.getId());
    checkConflict(!(maybePendingDeploy.isPresent() && maybePendingDeploy.get().getUpdatedRequest().isPresent()), "Request %s has a pending deploy that may change the request data. Try again when the deploy has finished", request.getId());
    Optional<SingularityRequest> oldRequest = oldRequestWithState.isPresent() ? Optional.of(oldRequestWithState.get().getRequest()) : Optional.<SingularityRequest>absent();
    if (oldRequest.isPresent()) {
        authorizationHelper.checkForAuthorization(oldRequest.get(), user, SingularityAuthorizationScope.WRITE);
        authorizationHelper.checkForAuthorizedChanges(request, oldRequest.get(), user);
        validator.checkActionEnabled(SingularityAction.UPDATE_REQUEST);
    } else {
        validator.checkActionEnabled(SingularityAction.CREATE_REQUEST);
    }
    if (request.getSlavePlacement().isPresent() && request.getSlavePlacement().get() == SlavePlacement.SPREAD_ALL_SLAVES) {
        checkBadRequest(validator.isSpreadAllSlavesEnabled(), "You must enabled spread to all slaves in order to use the SPREAD_ALL_SLAVES request type");
        int currentActiveSlaveCount = slaveManager.getNumObjectsAtState(MachineState.ACTIVE);
        request = request.toBuilder().setInstances(Optional.of(currentActiveSlaveCount)).build();
    }
    if (!oldRequest.isPresent() || !(oldRequest.get().getInstancesSafe() == request.getInstancesSafe())) {
        validator.checkScale(request, Optional.<Integer>absent());
    }
    authorizationHelper.checkForAuthorization(request, user, SingularityAuthorizationScope.WRITE);
    RequestState requestState = RequestState.ACTIVE;
    if (oldRequestWithState.isPresent()) {
        requestState = oldRequestWithState.get().getState();
    }
    requestHelper.updateRequest(request, oldRequest, requestState, historyType, user.getEmail(), skipHealthchecks, message, maybeBounceRequest);
}