Examples with PendingType com.hubspot.singularity.SingularityPendingRequest.PendingType used on opensource projects

Example 1 with PendingType

use of com.hubspot.singularity.SingularityPendingRequest.PendingType in project Singularity by HubSpot.

the class SingularityValidator method checkRunNowRequest.

public SingularityPendingRequest checkRunNowRequest(String deployId, Optional<String> userEmail, SingularityRequest request, Optional<SingularityRunNowRequest> maybeRunNowRequest, List<SingularityTaskId> activeTasks, List<SingularityPendingTaskId> pendingTasks) {
    SingularityRunNowRequest runNowRequest = fillRunNowRequest(maybeRunNowRequest);
    PendingType pendingType;
    if (request.isScheduled()) {
        pendingType = PendingType.IMMEDIATE;
        checkConflict(activeTasks.isEmpty(), "Cannot request immediate run of a scheduled job which is currently running (%s)", activeTasks);
    } else if (request.isOneOff()) {
        pendingType = PendingType.ONEOFF;
        if (request.getInstances().isPresent()) {
            checkRateLimited(activeTasks.size() + pendingTasks.size() < request.getInstances().get(), "No more than %s tasks allowed to run concurrently for request %s (%s active, %s pending)", request.getInstances().get(), request, activeTasks.size(), pendingTasks.size());
        }
    } else {
        throw badRequest("Can not request an immediate run of a non-scheduled / always running request (%s)", request);
    }
    if (runNowRequest.getRunAt().isPresent() && runNowRequest.getRunAt().get() > (System.currentTimeMillis() + TimeUnit.DAYS.toMillis(maxRunNowTaskLaunchDelay))) {
        throw badRequest("Task launch delay can be at most %d days from now.", maxRunNowTaskLaunchDelay);
    }
    return new SingularityPendingRequest(request.getId(), deployId, System.currentTimeMillis(), userEmail, pendingType, runNowRequest.getCommandLineArgs(), Optional.of(getRunId(runNowRequest.getRunId())), runNowRequest.getSkipHealthchecks(), runNowRequest.getMessage(), Optional.absent(), runNowRequest.getResources(), runNowRequest.getS3UploaderAdditionalFiles(), runNowRequest.getRunAsUserOverride(), runNowRequest.getEnvOverrides(), runNowRequest.getExtraArtifacts(), runNowRequest.getRunAt());
}

Example 2 with PendingType

use of com.hubspot.singularity.SingularityPendingRequest.PendingType in project Singularity by HubSpot.

the class SingularityDeployChecker method finishDeploy.

private void finishDeploy(SingularityRequestWithState requestWithState, Optional<SingularityDeploy> deploy, SingularityPendingDeploy pendingDeploy, Iterable<SingularityTaskId> tasksToKill, SingularityDeployResult deployResult) {
    SingularityRequest request = requestWithState.getRequest();
    if (!request.isOneOff() && !(request.getRequestType() == RequestType.RUN_ONCE)) {
        cleanupTasks(pendingDeploy, request, deployResult, tasksToKill);
    }
    if (deploy.isPresent() && deploy.get().getRunImmediately().isPresent()) {
        String requestId = deploy.get().getRequestId();
        String deployId = deploy.get().getId();
        SingularityRunNowRequest runNowRequest = deploy.get().getRunImmediately().get();
        List<SingularityTaskId> activeTasks = taskManager.getActiveTaskIdsForRequest(requestId);
        List<SingularityPendingTaskId> pendingTasks = taskManager.getPendingTaskIdsForRequest(requestId);
        SingularityPendingRequestBuilder builder = new SingularityPendingRequestBuilder().setRequestId(requestId).setDeployId(deployId).setTimestamp(deployResult.getTimestamp()).setUser(pendingDeploy.getDeployMarker().getUser()).setCmdLineArgsList(runNowRequest.getCommandLineArgs()).setRunId(runNowRequest.getRunId().or(Optional.of(UUID.randomUUID().toString()))).setSkipHealthchecks(runNowRequest.getSkipHealthchecks().or(request.getSkipHealthchecks())).setMessage(runNowRequest.getMessage().or(pendingDeploy.getDeployMarker().getMessage())).setResources(runNowRequest.getResources()).setRunAsUserOverride(runNowRequest.getRunAsUserOverride()).setEnvOverrides(runNowRequest.getEnvOverrides()).setExtraArtifacts(runNowRequest.getExtraArtifacts()).setRunAt(runNowRequest.getRunAt());
        PendingType pendingType = null;
        if (request.isScheduled()) {
            if (activeTasks.isEmpty()) {
                pendingType = PendingType.IMMEDIATE;
            } else {
                // Don't run scheduled task over a running task. Will be picked up on the next run.
                pendingType = PendingType.NEW_DEPLOY;
            }
        } else if (!request.isLongRunning()) {
            if (request.getInstances().isPresent() && (activeTasks.size() + pendingTasks.size() < request.getInstances().get())) {
                pendingType = PendingType.ONEOFF;
            } else {
                // Don't run one-off / on-demand task when already at instance count cap
                pendingType = PendingType.NEW_DEPLOY;
            }
        }
        if (pendingType != null) {
            builder.setPendingType(canceledOr(deployResult.getDeployState(), pendingType));
            requestManager.addToPendingQueue(builder.build());
        } else {
            LOG.warn("Could not determine pending type for deploy {}.", deployId);
        }
    } else if (!request.isDeployable() && !request.isOneOff()) {
        PendingType pendingType = canceledOr(deployResult.getDeployState(), PendingType.NEW_DEPLOY);
        requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), pendingType, deploy.isPresent() ? deploy.get().getSkipHealthchecksOnDeploy() : Optional.absent(), pendingDeploy.getDeployMarker().getMessage()));
    }
    if (deployResult.getDeployState() == DeployState.SUCCEEDED) {
        if (request.isDeployable() && !request.isOneOff()) {
            // remove the lock on bounces in case we deployed during a bounce
            requestManager.markBounceComplete(request.getId());
        }
        if (requestWithState.getState() == RequestState.FINISHED) {
            // A FINISHED request is moved to ACTIVE state so we can reevaluate the schedule
            requestManager.activate(request, RequestHistoryType.UPDATED, System.currentTimeMillis(), deploy.isPresent() ? deploy.get().getUser() : Optional.absent(), Optional.absent());
        }
    }
    deployManager.saveDeployResult(pendingDeploy.getDeployMarker(), deploy, deployResult);
    if (request.isDeployable() && (deployResult.getDeployState() == DeployState.CANCELED || deployResult.getDeployState() == DeployState.FAILED || deployResult.getDeployState() == DeployState.OVERDUE)) {
        Optional<SingularityRequestDeployState> maybeRequestDeployState = deployManager.getRequestDeployState(request.getId());
        if (maybeRequestDeployState.isPresent() && maybeRequestDeployState.get().getActiveDeploy().isPresent() && !(requestWithState.getState() == RequestState.PAUSED || requestWithState.getState() == RequestState.DEPLOYING_TO_UNPAUSE)) {
            requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), maybeRequestDeployState.get().getActiveDeploy().get().getDeployId(), deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), deployResult.getDeployState() == DeployState.CANCELED ? PendingType.DEPLOY_CANCELLED : PendingType.DEPLOY_FAILED, request.getSkipHealthchecks(), pendingDeploy.getDeployMarker().getMessage()));
        }
    }
    if (request.isDeployable() && deployResult.getDeployState() == DeployState.SUCCEEDED && pendingDeploy.getDeployProgress().isPresent() && requestWithState.getState() != RequestState.PAUSED) {
        if (pendingDeploy.getDeployProgress().get().getTargetActiveInstances() != request.getInstancesSafe()) {
            requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), PendingType.UPDATED_REQUEST, request.getSkipHealthchecks(), pendingDeploy.getDeployMarker().getMessage()));
        }
    }
    if (requestWithState.getState() == RequestState.DEPLOYING_TO_UNPAUSE) {
        if (deployResult.getDeployState() == DeployState.SUCCEEDED) {
            requestManager.activate(request, RequestHistoryType.DEPLOYED_TO_UNPAUSE, deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), Optional.<String>absent());
            requestManager.deleteExpiringObject(SingularityExpiringPause.class, request.getId());
        } else {
            requestManager.pause(request, deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), Optional.<String>absent());
        }
    }
    if (pendingDeploy.getUpdatedRequest().isPresent() && deployResult.getDeployState() == DeployState.SUCCEEDED) {
        requestManager.update(pendingDeploy.getUpdatedRequest().get(), System.currentTimeMillis(), pendingDeploy.getDeployMarker().getUser(), Optional.<String>absent());
        requestManager.deleteExpiringObject(SingularityExpiringScale.class, request.getId());
    }
    removePendingDeploy(pendingDeploy);
}

Example 3 with PendingType

use of com.hubspot.singularity.SingularityPendingRequest.PendingType in project Singularity by HubSpot.

the class SingularityScheduler method getNextRunAt.

private Optional<Long> getNextRunAt(SingularityRequest request, RequestState state, SingularityDeployStatistics deployStatistics, SingularityPendingRequest pendingRequest, Optional<SingularityPendingDeploy> maybePendingDeploy) {
    PendingType pendingType = pendingRequest.getPendingType();
    final long now = System.currentTimeMillis();
    long nextRunAt = now;
    if (request.isScheduled()) {
        if (pendingType == PendingType.IMMEDIATE || pendingType == PendingType.RETRY) {
            LOG.info("Scheduling requested immediate run of {}", request.getId());
        } else {
            try {
                Date nextRunAtDate = null;
                Date scheduleFrom = null;
                if (request.getScheduleTypeSafe() == ScheduleType.RFC5545) {
                    final RFC5545Schedule rfc5545Schedule = new RFC5545Schedule(request.getSchedule().get());
                    nextRunAtDate = rfc5545Schedule.getNextValidTime();
                    scheduleFrom = new Date(rfc5545Schedule.getStartDateTime().getMillis());
                } else {
                    scheduleFrom = new Date(now);
                    final CronExpression cronExpression = new CronExpression(request.getQuartzScheduleSafe());
                    if (request.getScheduleTimeZone().isPresent()) {
                        cronExpression.setTimeZone(TimeZone.getTimeZone(request.getScheduleTimeZone().get()));
                    }
                    nextRunAtDate = cronExpression.getNextValidTimeAfter(scheduleFrom);
                }
                if (nextRunAtDate == null) {
                    return Optional.absent();
                }
                LOG.trace("Calculating nextRunAtDate for {} (schedule: {}): {} (from: {})", request.getId(), request.getSchedule(), nextRunAtDate, scheduleFrom);
                // don't create a schedule that is overdue as this is used to indicate that singularity is not fulfilling requests.
                nextRunAt = Math.max(nextRunAtDate.getTime(), now);
                LOG.trace("Scheduling next run of {} (schedule: {}) at {} (from: {})", request.getId(), request.getSchedule(), nextRunAtDate, scheduleFrom);
            } catch (ParseException | InvalidRecurrenceRuleException pe) {
                throw Throwables.propagate(pe);
            }
        }
    }
    if (!request.isLongRunning() && pendingRequest.getRunAt().isPresent()) {
        nextRunAt = Math.max(nextRunAt, pendingRequest.getRunAt().get());
    }
    if (pendingType == PendingType.TASK_DONE && request.getWaitAtLeastMillisAfterTaskFinishesForReschedule().or(0L) > 0) {
        nextRunAt = Math.max(nextRunAt, now + request.getWaitAtLeastMillisAfterTaskFinishesForReschedule().get());
        LOG.trace("Adjusted next run of {} to {} (by {}) due to waitAtLeastMillisAfterTaskFinishesForReschedule", request.getId(), nextRunAt, JavaUtils.durationFromMillis(request.getWaitAtLeastMillisAfterTaskFinishesForReschedule().get()));
    }
    if (state == RequestState.SYSTEM_COOLDOWN && pendingType != PendingType.NEW_DEPLOY) {
        final long prevNextRunAt = nextRunAt;
        nextRunAt = Math.max(nextRunAt, now + TimeUnit.SECONDS.toMillis(configuration.getCooldownMinScheduleSeconds()));
        LOG.trace("Adjusted next run of {} to {} (from: {}) due to cooldown", request.getId(), nextRunAt, prevNextRunAt);
    }
    return Optional.of(nextRunAt);
}

Example 4 with PendingType

use of com.hubspot.singularity.SingularityPendingRequest.PendingType in project Singularity by HubSpot.

the class SingularityScheduler method handleCompletedTaskWithStatistics.

private Optional<PendingType> handleCompletedTaskWithStatistics(Optional<SingularityTask> task, SingularityTaskId taskId, long timestamp, ExtendedTaskState state, SingularityDeployStatistics deployStatistics, SingularityCreateResult taskHistoryUpdateCreateResult, Protos.TaskStatus status) {
    final Optional<SingularityRequestWithState> maybeRequestWithState = requestManager.getRequest(taskId.getRequestId());
    final Optional<SingularityPendingDeploy> maybePendingDeploy = deployManager.getPendingDeploy(taskId.getRequestId());
    if (!isRequestActive(maybeRequestWithState)) {
        LOG.warn("Not scheduling a new task, {} is {}", taskId.getRequestId(), SingularityRequestWithState.getRequestState(maybeRequestWithState));
        return Optional.absent();
    }
    RequestState requestState = maybeRequestWithState.get().getState();
    final SingularityRequest request = maybePendingDeploy.isPresent() ? maybePendingDeploy.get().getUpdatedRequest().or(maybeRequestWithState.get().getRequest()) : maybeRequestWithState.get().getRequest();
    final Optional<SingularityRequestDeployState> requestDeployState = deployManager.getRequestDeployState(request.getId());
    if (!isDeployInUse(requestDeployState, taskId.getDeployId(), true)) {
        LOG.debug("Task {} completed, but it didn't match active deploy state {} - ignoring", taskId.getId(), requestDeployState);
        return Optional.absent();
    }
    if (taskHistoryUpdateCreateResult == SingularityCreateResult.CREATED && requestState != RequestState.SYSTEM_COOLDOWN) {
        mailer.queueTaskCompletedMail(task, taskId, request, state);
    } else if (requestState == RequestState.SYSTEM_COOLDOWN) {
        LOG.debug("Not sending a task completed email because task {} is in SYSTEM_COOLDOWN", taskId);
    } else {
        LOG.debug("Not sending a task completed email for task {} because Singularity already processed this update", taskId);
    }
    if (!status.hasReason() || !status.getReason().equals(Reason.REASON_INVALID_OFFERS)) {
        if (!state.isSuccess() && taskHistoryUpdateCreateResult == SingularityCreateResult.CREATED && cooldown.shouldEnterCooldown(request, taskId, requestState, deployStatistics, timestamp)) {
            LOG.info("Request {} is entering cooldown due to task {}", request.getId(), taskId);
            requestState = RequestState.SYSTEM_COOLDOWN;
            requestManager.cooldown(request, System.currentTimeMillis());
            mailer.sendRequestInCooldownMail(request);
        }
    } else {
        LOG.debug("Not triggering cooldown due to TASK_LOST from invalid offers for request {}", request.getId());
    }
    PendingType pendingType = PendingType.TASK_DONE;
    Optional<List<String>> cmdLineArgsList = Optional.absent();
    if (!state.isSuccess() && shouldRetryImmediately(request, deployStatistics, task)) {
        LOG.debug("Retrying {} because {}", request.getId(), state);
        pendingType = PendingType.RETRY;
        if (task.isPresent()) {
            cmdLineArgsList = task.get().getTaskRequest().getPendingTask().getCmdLineArgsList();
        }
    } else if (!request.isAlwaysRunning()) {
        return Optional.absent();
    }
    if (state.isSuccess() && requestState == RequestState.SYSTEM_COOLDOWN) {
        // TODO send not cooldown anymore email
        LOG.info("Request {} succeeded a task, removing from cooldown", request.getId());
        requestManager.exitCooldown(request, System.currentTimeMillis(), Optional.<String>absent(), Optional.<String>absent());
    }
    SingularityPendingRequest pendingRequest = new SingularityPendingRequest(request.getId(), requestDeployState.get().getActiveDeploy().get().getDeployId(), System.currentTimeMillis(), Optional.absent(), pendingType, cmdLineArgsList, Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent());
    requestManager.addToPendingQueue(pendingRequest);
    return Optional.of(pendingType);
}

Example 5 with PendingType

use of com.hubspot.singularity.SingularityPendingRequest.PendingType in project Singularity by HubSpot.

the class SingularityScheduler method handleCompletedTask.

@Timed
public void handleCompletedTask(Optional<SingularityTask> task, SingularityTaskId taskId, boolean wasActive, long timestamp, ExtendedTaskState state, SingularityCreateResult taskHistoryUpdateCreateResult, Protos.TaskStatus status) {
    final SingularityDeployStatistics deployStatistics = getDeployStatistics(taskId.getRequestId(), taskId.getDeployId());
    if (wasActive) {
        taskManager.deleteActiveTask(taskId.getId());
    }
    if (!task.isPresent() || task.get().getTaskRequest().getRequest().isLoadBalanced()) {
        taskManager.createLBCleanupTask(taskId);
    }
    if (requestManager.isBouncing(taskId.getRequestId())) {
        List<SingularityTaskId> activeTaskIds = taskManager.getActiveTaskIdsForRequest(taskId.getRequestId());
        boolean foundBouncingTask = false;
        for (SingularityTaskId activeTaskId : activeTaskIds) {
            Optional<SingularityTaskHistoryUpdate> maybeCleaningUpdate = taskManager.getTaskHistoryUpdate(activeTaskId, ExtendedTaskState.TASK_CLEANING);
            if (maybeCleaningUpdate.isPresent()) {
                if (maybeCleaningUpdate.get().getStatusReason().or("").contains("BOUNCE")) {
                    // TaskCleanupType enum is included in status message
                    LOG.debug("Found task {} still waiting for bounce to complete", activeTaskId);
                    foundBouncingTask = true;
                    break;
                } else if (!maybeCleaningUpdate.get().getPrevious().isEmpty()) {
                    for (SingularityTaskHistoryUpdate previousUpdate : maybeCleaningUpdate.get().getPrevious()) {
                        if (previousUpdate.getStatusMessage().or("").contains("BOUNCE")) {
                            LOG.debug("Found task {} still waiting for bounce to complete", activeTaskId);
                            foundBouncingTask = true;
                            break;
                        }
                    }
                }
            }
        }
        if (!foundBouncingTask) {
            LOG.info("Bounce completed for request {}, no cleaning tasks due to bounce found", taskId.getRequestId());
            Optional<SingularityExpiringBounce> expiringBounce = requestManager.getExpiringBounce(taskId.getRequestId());
            if (expiringBounce.isPresent() && expiringBounce.get().getDeployId().equals(taskId.getDeployId())) {
                requestManager.deleteExpiringObject(SingularityExpiringBounce.class, taskId.getRequestId());
            }
            requestManager.markBounceComplete(taskId.getRequestId());
        }
    }
    final Optional<PendingType> scheduleResult = handleCompletedTaskWithStatistics(task, taskId, timestamp, state, deployStatistics, taskHistoryUpdateCreateResult, status);
    if (taskHistoryUpdateCreateResult == SingularityCreateResult.EXISTED) {
        return;
    }
    updateDeployStatistics(deployStatistics, taskId, task, timestamp, state, scheduleResult);
}