Examples with SingularityExpiringBounce com.hubspot.singularity.expiring.SingularityExpiringBounce used on opensource projects

Example 1 with SingularityExpiringBounce

use of com.hubspot.singularity.expiring.SingularityExpiringBounce in project Singularity by HubSpot.

the class RequestHelper method checkReschedule.

private void checkReschedule(SingularityRequest newRequest, Optional<SingularityRequest> maybeOldRequest, Optional<String> user, long timestamp, Optional<Boolean> skipHealthchecks, Optional<String> message, Optional<SingularityBounceRequest> maybeBounceRequest) {
    if (!maybeOldRequest.isPresent()) {
        return;
    }
    if (shouldReschedule(newRequest, maybeOldRequest.get())) {
        Optional<String> maybeDeployId = deployManager.getInUseDeployId(newRequest.getId());
        if (maybeDeployId.isPresent()) {
            if (maybeBounceRequest.isPresent()) {
                Optional<String> actionId = maybeBounceRequest.get().getActionId().or(Optional.of(UUID.randomUUID().toString()));
                Optional<Boolean> removeFromLoadBalancer = Optional.absent();
                SingularityCreateResult createResult = requestManager.createCleanupRequest(new SingularityRequestCleanup(user, maybeBounceRequest.get().getIncremental().or(true) ? RequestCleanupType.INCREMENTAL_BOUNCE : RequestCleanupType.BOUNCE, System.currentTimeMillis(), Optional.<Boolean>absent(), removeFromLoadBalancer, newRequest.getId(), Optional.of(maybeDeployId.get()), skipHealthchecks, message, actionId, maybeBounceRequest.get().getRunShellCommandBeforeKill()));
                if (createResult != SingularityCreateResult.EXISTED) {
                    requestManager.bounce(newRequest, System.currentTimeMillis(), user, Optional.of("Bouncing due to bounce after scale"));
                    final SingularityBounceRequest validatedBounceRequest = validator.checkBounceRequest(maybeBounceRequest.get());
                    requestManager.saveExpiringObject(new SingularityExpiringBounce(newRequest.getId(), maybeDeployId.get(), user, System.currentTimeMillis(), validatedBounceRequest, actionId.get()));
                } else {
                    requestManager.addToPendingQueue(new SingularityPendingRequest(newRequest.getId(), maybeDeployId.get(), timestamp, user, PendingType.UPDATED_REQUEST, skipHealthchecks, message));
                }
            } else {
                requestManager.addToPendingQueue(new SingularityPendingRequest(newRequest.getId(), maybeDeployId.get(), timestamp, user, PendingType.UPDATED_REQUEST, skipHealthchecks, message));
            }
        }
    }
}

Example 2 with SingularityExpiringBounce

use of com.hubspot.singularity.expiring.SingularityExpiringBounce in project Singularity by HubSpot.

the class RequestResource method bounce.

public SingularityRequestParent bounce(String requestId, Optional<SingularityBounceRequest> bounceRequest, SingularityUser user) {
    SingularityRequestWithState requestWithState = fetchRequestWithState(requestId, user);
    authorizationHelper.checkForAuthorization(requestWithState.getRequest(), user, SingularityAuthorizationScope.WRITE);
    validator.checkActionEnabled(SingularityAction.BOUNCE_REQUEST);
    checkBadRequest(requestWithState.getRequest().isLongRunning(), "Can not bounce a %s request (%s)", requestWithState.getRequest().getRequestType(), requestWithState);
    checkConflict(requestWithState.getState() != RequestState.PAUSED, "Request %s is paused. Unable to bounce (it must be manually unpaused first)", requestWithState.getRequest().getId());
    final boolean isIncrementalBounce = bounceRequest.isPresent() && bounceRequest.get().getIncremental().or(false);
    validator.checkResourcesForBounce(requestWithState.getRequest(), isIncrementalBounce);
    validator.checkRequestForPriorityFreeze(requestWithState.getRequest());
    final Optional<Boolean> skipHealthchecks = bounceRequest.isPresent() ? bounceRequest.get().getSkipHealthchecks() : Optional.<Boolean>absent();
    Optional<String> message = Optional.absent();
    Optional<String> actionId = Optional.absent();
    Optional<SingularityShellCommand> runBeforeKill = Optional.absent();
    if (bounceRequest.isPresent()) {
        actionId = bounceRequest.get().getActionId();
        message = bounceRequest.get().getMessage();
        if (bounceRequest.get().getRunShellCommandBeforeKill().isPresent()) {
            validator.checkValidShellCommand(bounceRequest.get().getRunShellCommandBeforeKill().get());
            runBeforeKill = bounceRequest.get().getRunShellCommandBeforeKill();
        }
    }
    if (!actionId.isPresent()) {
        actionId = Optional.of(UUID.randomUUID().toString());
    }
    final String deployId = getAndCheckDeployId(requestId);
    checkConflict(!(requestManager.markAsBouncing(requestId) == SingularityCreateResult.EXISTED), "%s is already bouncing", requestId);
    requestManager.createCleanupRequest(new SingularityRequestCleanup(user.getEmail(), isIncrementalBounce ? RequestCleanupType.INCREMENTAL_BOUNCE : RequestCleanupType.BOUNCE, System.currentTimeMillis(), Optional.absent(), Optional.absent(), requestId, Optional.of(deployId), skipHealthchecks, message, actionId, runBeforeKill));
    requestManager.bounce(requestWithState.getRequest(), System.currentTimeMillis(), Optional.of(user.getId()), message);
    final SingularityBounceRequest validatedBounceRequest = validator.checkBounceRequest(bounceRequest.or(SingularityBounceRequest.defaultRequest()));
    requestManager.saveExpiringObject(new SingularityExpiringBounce(requestId, deployId, Optional.of(user.getId()), System.currentTimeMillis(), validatedBounceRequest, actionId.get()));
    return fillEntireRequest(requestWithState);
}

Example 3 with SingularityExpiringBounce

use of com.hubspot.singularity.expiring.SingularityExpiringBounce in project Singularity by HubSpot.

the class SingularityCleaner method bounce.

private void bounce(SingularityRequestCleanup requestCleanup, final List<SingularityTaskId> activeTaskIds) {
    final long start = System.currentTimeMillis();
    final List<SingularityTaskId> matchingTaskIds = new ArrayList<>();
    for (SingularityTaskId activeTaskId : activeTaskIds) {
        if (activeTaskId.getRequestId().equals(requestCleanup.getRequestId()) && activeTaskId.getDeployId().equals(requestCleanup.getDeployId().get())) {
            matchingTaskIds.add(activeTaskId);
        }
    }
    for (SingularityTaskId matchingTaskId : matchingTaskIds) {
        LOG.debug("Adding task {} to cleanup (bounce)", matchingTaskId.getId());
        Optional<SingularityTaskShellCommandRequestId> runBeforeKillId = Optional.absent();
        if (requestCleanup.getRunShellCommandBeforeKill().isPresent()) {
            SingularityTaskShellCommandRequest shellRequest = new SingularityTaskShellCommandRequest(matchingTaskId, requestCleanup.getUser(), System.currentTimeMillis(), requestCleanup.getRunShellCommandBeforeKill().get());
            taskManager.saveTaskShellCommandRequestToQueue(shellRequest);
            runBeforeKillId = Optional.of(shellRequest.getId());
        }
        taskManager.createTaskCleanup(new SingularityTaskCleanup(requestCleanup.getUser(), requestCleanup.getCleanupType().getTaskCleanupType().get(), start, matchingTaskId, requestCleanup.getMessage(), requestCleanup.getActionId(), runBeforeKillId));
    }
    if (matchingTaskIds.isEmpty() && requestCleanup.getDeployId().isPresent()) {
        Optional<SingularityExpiringBounce> expiringBounce = requestManager.getExpiringBounce(requestCleanup.getRequestId());
        if (expiringBounce.isPresent() && expiringBounce.get().getDeployId().equalsIgnoreCase(requestCleanup.getDeployId().get())) {
            LOG.info("No running tasks for request {}. Marking bounce {} complete and starting new tasks", expiringBounce.get().getRequestId(), expiringBounce.get());
            requestManager.deleteExpiringObject(SingularityExpiringBounce.class, requestCleanup.getRequestId());
        }
        requestManager.markBounceComplete(requestCleanup.getRequestId());
    }
    requestManager.addToPendingQueue(new SingularityPendingRequest(requestCleanup.getRequestId(), requestCleanup.getDeployId().get(), requestCleanup.getTimestamp(), requestCleanup.getUser(), PendingType.BOUNCE, Optional.absent(), Optional.absent(), requestCleanup.getSkipHealthchecks(), requestCleanup.getMessage(), requestCleanup.getActionId()));
    LOG.info("Added {} tasks for request {} to cleanup bounce queue in {}", matchingTaskIds.size(), requestCleanup.getRequestId(), JavaUtils.duration(start));
}

Example 4 with SingularityExpiringBounce

use of com.hubspot.singularity.expiring.SingularityExpiringBounce in project Singularity by HubSpot.

the class SingularityScheduler method handleCompletedTask.

@Timed
public void handleCompletedTask(Optional<SingularityTask> task, SingularityTaskId taskId, boolean wasActive, long timestamp, ExtendedTaskState state, SingularityCreateResult taskHistoryUpdateCreateResult, Protos.TaskStatus status) {
    final SingularityDeployStatistics deployStatistics = getDeployStatistics(taskId.getRequestId(), taskId.getDeployId());
    if (wasActive) {
        taskManager.deleteActiveTask(taskId.getId());
    }
    if (!task.isPresent() || task.get().getTaskRequest().getRequest().isLoadBalanced()) {
        taskManager.createLBCleanupTask(taskId);
    }
    if (requestManager.isBouncing(taskId.getRequestId())) {
        List<SingularityTaskId> activeTaskIds = taskManager.getActiveTaskIdsForRequest(taskId.getRequestId());
        boolean foundBouncingTask = false;
        for (SingularityTaskId activeTaskId : activeTaskIds) {
            Optional<SingularityTaskHistoryUpdate> maybeCleaningUpdate = taskManager.getTaskHistoryUpdate(activeTaskId, ExtendedTaskState.TASK_CLEANING);
            if (maybeCleaningUpdate.isPresent()) {
                if (maybeCleaningUpdate.get().getStatusReason().or("").contains("BOUNCE")) {
                    // TaskCleanupType enum is included in status message
                    LOG.debug("Found task {} still waiting for bounce to complete", activeTaskId);
                    foundBouncingTask = true;
                    break;
                } else if (!maybeCleaningUpdate.get().getPrevious().isEmpty()) {
                    for (SingularityTaskHistoryUpdate previousUpdate : maybeCleaningUpdate.get().getPrevious()) {
                        if (previousUpdate.getStatusMessage().or("").contains("BOUNCE")) {
                            LOG.debug("Found task {} still waiting for bounce to complete", activeTaskId);
                            foundBouncingTask = true;
                            break;
                        }
                    }
                }
            }
        }
        if (!foundBouncingTask) {
            LOG.info("Bounce completed for request {}, no cleaning tasks due to bounce found", taskId.getRequestId());
            Optional<SingularityExpiringBounce> expiringBounce = requestManager.getExpiringBounce(taskId.getRequestId());
            if (expiringBounce.isPresent() && expiringBounce.get().getDeployId().equals(taskId.getDeployId())) {
                requestManager.deleteExpiringObject(SingularityExpiringBounce.class, taskId.getRequestId());
            }
            requestManager.markBounceComplete(taskId.getRequestId());
        }
    }
    final Optional<PendingType> scheduleResult = handleCompletedTaskWithStatistics(task, taskId, timestamp, state, deployStatistics, taskHistoryUpdateCreateResult, status);
    if (taskHistoryUpdateCreateResult == SingularityCreateResult.EXISTED) {
        return;
    }
    updateDeployStatistics(deployStatistics, taskId, task, timestamp, state, scheduleResult);
}

Example 5 with SingularityExpiringBounce

use of com.hubspot.singularity.expiring.SingularityExpiringBounce in project Singularity by HubSpot.

the class RequestHelper method fillDataForRequestsAndFilter.

public List<SingularityRequestParent> fillDataForRequestsAndFilter(List<SingularityRequestWithState> requests, SingularityUser user, boolean filterRelevantForUser, boolean includeFullRequestData, Optional<Integer> limit, List<RequestType> requestTypeFilters) {
    final Map<String, Optional<SingularityTaskIdHistory>> mostRecentTasks = new ConcurrentHashMap<>();
    final Map<String, SingularityRequestDeployState> deployStates = deployManager.getRequestDeployStatesByRequestIds(requests.stream().map((r) -> r.getRequest().getId()).collect(Collectors.toList()));
    final Map<String, Optional<SingularityRequestHistory>> requestIdToLastHistory;
    if (includeFullRequestData) {
        requestIdToLastHistory = requests.parallelStream().collect(Collectors.toMap((r) -> r.getRequest().getId(), (r) -> getMostRecentHistoryFromZk(r.getRequest().getId())));
    } else {
        requestIdToLastHistory = Collections.emptyMap();
    }
    Optional<SingularityUserSettings> maybeUserSettings = userManager.getUserSettings(user.getId());
    return requests.parallelStream().filter((request) -> {
        if (!requestTypeFilters.isEmpty() && !requestTypeFilters.contains(request.getRequest().getRequestType())) {
            return false;
        }
        if (!filterRelevantForUser || user.equals(SingularityUser.DEFAULT_USER)) {
            return true;
        }
        String requestId = request.getRequest().getId();
        if (maybeUserSettings.isPresent() && maybeUserSettings.get().getStarredRequestIds().contains(requestId)) {
            // This is a starred request for the user
            return true;
        }
        if (request.getRequest().getGroup().isPresent() && user.getGroups().contains(request.getRequest().getGroup().get())) {
            // The user is in the group for this request
            return true;
        }
        if (includeFullRequestData) {
            if (userModifiedRequestLast(requestIdToLastHistory.getOrDefault(requestId, Optional.absent()), user)) {
                return true;
            }
        }
        return userAssociatedWithDeploy(Optional.fromNullable(deployStates.get(requestId)), user);
    }).map((request) -> {
        Long lastActionTime = null;
        if (includeFullRequestData) {
            lastActionTime = getLastActionTimeForRequest(request.getRequest(), requestIdToLastHistory.getOrDefault(request.getRequest().getId(), Optional.absent()), Optional.fromNullable(deployStates.get(request.getRequest().getId())), mostRecentTasks.computeIfAbsent(request.getRequest().getId(), (id) -> getMostRecentTask(request.getRequest())));
        } else {
            // To save on zk calls, if not returning all data, use the most recent deploy timestamps
            Optional<SingularityRequestDeployState> deployState = Optional.fromNullable(deployStates.get(request.getRequest().getId()));
            if (deployState.isPresent()) {
                if (deployState.get().getPendingDeploy().isPresent()) {
                    lastActionTime = deployState.get().getPendingDeploy().get().getTimestamp();
                }
                if (deployState.get().getActiveDeploy().isPresent()) {
                    lastActionTime = deployState.get().getActiveDeploy().get().getTimestamp();
                }
            }
            if (lastActionTime == null) {
                lastActionTime = 0L;
            }
        }
        return new RequestParentWithLastActionTime(request, lastActionTime, maybeUserSettings.isPresent() && maybeUserSettings.get().getStarredRequestIds().contains(request.getRequest().getId()));
    }).sorted().limit(limit.or(requests.size())).map((parentWithActionTime) -> {
        SingularityRequestWithState requestWithState = parentWithActionTime.getRequestWithState();
        if (includeFullRequestData) {
            CompletableFuture<Optional<SingularityTaskIdsByStatus>> maybeTaskIdsByStatus = CompletableFuture.supplyAsync(() -> getTaskIdsByStatusForRequest(requestWithState)).exceptionally((throwable) -> Optional.absent());
            CompletableFuture<Optional<SingularityExpiringBounce>> maybeExpiringBounce = CompletableFuture.supplyAsync(() -> requestManager.getExpiringBounce(requestWithState.getRequest().getId())).exceptionally((throwable) -> Optional.absent());
            CompletableFuture<Optional<SingularityExpiringPause>> maybeExpiringPause = CompletableFuture.supplyAsync(() -> requestManager.getExpiringPause(requestWithState.getRequest().getId())).exceptionally((throwable) -> Optional.absent());
            CompletableFuture<Optional<SingularityExpiringScale>> maybeExpiringScale = CompletableFuture.supplyAsync(() -> requestManager.getExpiringScale(requestWithState.getRequest().getId())).exceptionally((throwable) -> Optional.absent());
            CompletableFuture<Optional<SingularityExpiringSkipHealthchecks>> maybeExpiringSkipHealthchecks = CompletableFuture.supplyAsync(() -> requestManager.getExpiringSkipHealthchecks(requestWithState.getRequest().getId())).exceptionally((throwable) -> Optional.absent());
            return new SingularityRequestParent(requestWithState.getRequest(), requestWithState.getState(), Optional.fromNullable(deployStates.get(requestWithState.getRequest().getId())), // full deploy data not provided
            Optional.absent(), // full deploy data not provided
            Optional.absent(), // full deploy data not provided
            Optional.absent(), maybeExpiringBounce.join(), maybeExpiringPause.join(), maybeExpiringScale.join(), maybeExpiringSkipHealthchecks.join(), maybeTaskIdsByStatus.join(), requestIdToLastHistory.getOrDefault(requestWithState.getRequest().getId(), Optional.absent()), mostRecentTasks.computeIfAbsent(requestWithState.getRequest().getId(), (id) -> getMostRecentTask(requestWithState.getRequest())));
        } else {
            return new SingularityRequestParent(requestWithState.getRequest(), requestWithState.getState(), Optional.fromNullable(deployStates.get(requestWithState.getRequest().getId())), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent());
        }
    }).collect(Collectors.toList());
}