Examples with SingularityPendingRequest com.hubspot.singularity.SingularityPendingRequest used on opensource projects

Example 6 with SingularityPendingRequest

use of com.hubspot.singularity.SingularityPendingRequest in project Singularity by HubSpot.

the class SingularityScheduler method drainPendingQueue.

@Timed
public void drainPendingQueue() {
    final long start = System.currentTimeMillis();
    final ImmutableList<SingularityPendingRequest> pendingRequests = ImmutableList.copyOf(requestManager.getPendingRequests());
    if (pendingRequests.isEmpty()) {
        LOG.trace("Pending queue was empty");
        return;
    }
    LOG.info("Pending queue had {} requests", pendingRequests.size());
    Map<SingularityDeployKey, List<SingularityPendingRequest>> deployKeyToPendingRequests = pendingRequests.stream().collect(Collectors.groupingBy(request -> new SingularityDeployKey(request.getRequestId(), request.getDeployId())));
    AtomicInteger totalNewScheduledTasks = new AtomicInteger(0);
    AtomicInteger heldForScheduledActiveTask = new AtomicInteger(0);
    AtomicInteger obsoleteRequests = new AtomicInteger(0);
    List<CompletableFuture<Void>> checkFutures = deployKeyToPendingRequests.entrySet().stream().map(e -> CompletableFuture.runAsync(() -> lock.runWithRequestLock(() -> handlePendingRequestsForDeployKeySafe(obsoleteRequests, heldForScheduledActiveTask, totalNewScheduledTasks, e.getKey(), e.getValue()), e.getKey().getRequestId(), String.format("%s#%s", getClass().getSimpleName(), "drainPendingQueue")), schedulerExecutorService)).collect(Collectors.toList());
    CompletableFutures.allOf(checkFutures).join();
    LOG.info("Scheduled {} new tasks ({} obsolete requests, {} held) in {}", totalNewScheduledTasks.get(), obsoleteRequests.get(), heldForScheduledActiveTask.get(), JavaUtils.duration(start));
}

Example 7 with SingularityPendingRequest

use of com.hubspot.singularity.SingularityPendingRequest in project Singularity by HubSpot.

the class SingularityScheduler method handlePendingRequestsForDeployKey.

private void handlePendingRequestsForDeployKey(AtomicInteger obsoleteRequests, AtomicInteger heldForScheduledActiveTask, AtomicInteger totalNewScheduledTasks, SingularityDeployKey deployKey, List<SingularityPendingRequest> pendingRequestsForDeploy) {
    final String requestId = deployKey.getRequestId();
    final Optional<SingularityRequestWithState> maybeRequest = requestManager.getRequest(requestId);
    final SingularityDeployStatistics deployStatistics = getDeployStatistics(deployKey.getRequestId(), deployKey.getDeployId());
    if (!isRequestActive(maybeRequest)) {
        LOG.info("Pending request {} was obsolete (request {})", requestId, SingularityRequestWithState.getRequestState(maybeRequest));
        obsoleteRequests.getAndIncrement();
        for (SingularityPendingRequest pendingRequest : pendingRequestsForDeploy) {
            requestManager.deletePendingRequest(pendingRequest);
        }
        return;
    }
    SingularityRequestWithState request = maybeRequest.get();
    Optional<SingularityRequestDeployState> maybeRequestDeployState = deployManager.getRequestDeployState(requestId);
    Optional<SingularityPendingDeploy> maybePendingDeploy = deployManager.getPendingDeploy(requestId);
    List<SingularityTaskId> matchingTaskIds = getMatchingTaskIds(request.getRequest(), deployKey);
    List<SingularityPendingRequest> effectivePendingRequests = new ArrayList<>();
    // Things that are closest to now (ie smaller timestamps) should come first in the queue
    pendingRequestsForDeploy.sort(Comparator.comparingLong(SingularityPendingRequest::getTimestamp));
    int scheduledTasks = 0;
    for (SingularityPendingRequest pendingRequest : pendingRequestsForDeploy) {
        final SingularityRequest updatedRequest = updatedRequest(maybePendingDeploy, pendingRequest, request);
        if (!shouldScheduleTasks(updatedRequest, pendingRequest, maybePendingDeploy, maybeRequestDeployState)) {
            LOG.info("Pending request {} was obsolete (request {})", pendingRequest, SingularityRequestWithState.getRequestState(maybeRequest));
            obsoleteRequests.getAndIncrement();
            requestManager.deletePendingRequest(pendingRequest);
            continue;
        }
        int missingInstances = getNumMissingInstances(matchingTaskIds, updatedRequest, pendingRequest, maybePendingDeploy);
        boolean scheduledRequestWithActiveTask = (missingInstances == 0 && !matchingTaskIds.isEmpty() && updatedRequest.isScheduled() && pendingRequest.getPendingType() == PendingType.NEW_DEPLOY);
        boolean scheduledRequestWithOutdatedActiveTask = (missingInstances == 0 && matchingTaskIds.isEmpty() && updatedRequest.isScheduled() && (pendingRequest.getPendingType() == PendingType.NEW_DEPLOY || pendingRequest.getPendingType() == PendingType.STARTUP));
        if (scheduledRequestWithActiveTask || scheduledRequestWithOutdatedActiveTask) {
            LOG.trace("Holding pending request {} because it is scheduled and has an active task", pendingRequest);
            heldForScheduledActiveTask.getAndIncrement();
            continue;
        }
        if (effectivePendingRequests.isEmpty()) {
            effectivePendingRequests.add(pendingRequest);
            RequestState requestState = checkCooldown(request.getState(), request.getRequest(), deployStatistics);
            scheduledTasks += scheduleTasks(request.getRequest(), requestState, pendingRequest, matchingTaskIds, maybePendingDeploy);
            requestManager.deletePendingRequest(pendingRequest);
        } else if (pendingRequest.getPendingType() == PendingType.IMMEDIATE) {
            effectivePendingRequests.add(pendingRequest);
            RequestState requestState = checkCooldown(request.getState(), request.getRequest(), deployStatistics);
            scheduledTasks += scheduleTasks(request.getRequest(), requestState, pendingRequest, matchingTaskIds, maybePendingDeploy);
            requestManager.deletePendingRequest(pendingRequest);
        } else if (pendingRequest.getPendingType() == PendingType.ONEOFF) {
            effectivePendingRequests.add(pendingRequest);
            RequestState requestState = checkCooldown(request.getState(), request.getRequest(), deployStatistics);
            scheduledTasks += scheduleTasks(request.getRequest(), requestState, pendingRequest, matchingTaskIds, maybePendingDeploy);
            requestManager.deletePendingRequest(pendingRequest);
        } else if (updatedRequest.isScheduled() && (pendingRequest.getPendingType() == PendingType.NEW_DEPLOY || pendingRequest.getPendingType() == PendingType.TASK_DONE)) {
            // If we are here, there is already an immediate of run of the scheduled task launched. Drop anything that would
            // leave a second instance of the request in the pending queue.
            requestManager.deletePendingRequest(pendingRequest);
        }
    // Any other subsequent requests are not honored until after the pending queue is cleared.
    }
    totalNewScheduledTasks.getAndAdd(scheduledTasks);
}

Example 8 with SingularityPendingRequest

use of com.hubspot.singularity.SingularityPendingRequest in project Singularity by HubSpot.

the class SingularityScheduler method deleteScheduledTasks.

private void deleteScheduledTasks(final Collection<SingularityPendingTask> scheduledTasks, SingularityPendingRequest pendingRequest) {
    List<SingularityPendingTask> tasksForDeploy = scheduledTasks.stream().filter(task -> pendingRequest.getRequestId().equals(task.getPendingTaskId().getRequestId())).filter(task -> pendingRequest.getDeployId().equals(task.getPendingTaskId().getDeployId())).collect(Collectors.toList());
    for (SingularityPendingTask task : tasksForDeploy) {
        LOG.debug("Deleting pending task {} in order to reschedule {}", task.getPendingTaskId().getId(), pendingRequest);
        taskManager.deletePendingTask(task.getPendingTaskId());
    }
}

Example 9 with SingularityPendingRequest

use of com.hubspot.singularity.SingularityPendingRequest in project Singularity by HubSpot.

the class SingularityDeployChecker method maybeUpdatePendingRequest.

private void maybeUpdatePendingRequest(SingularityPendingDeploy pendingDeploy, SingularityDeploy deploy, SingularityRequest request, Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest) {
    if (updatePendingDeployRequest.isPresent()) {
        SingularityDeployProgress newProgress = pendingDeploy.getDeployProgress().withNewTargetInstances(Math.min(updatePendingDeployRequest.get().getTargetActiveInstances(), request.getInstancesSafe()), pendingDeploy.getDeployProgress().getCurrentActiveInstances(), true);
        updatePendingDeploy(pendingDeploy, DeployState.WAITING, newProgress);
        requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), System.currentTimeMillis(), pendingDeploy.getDeployMarker().getUser(), PendingType.NEXT_DEPLOY_STEP, deploy.getSkipHealthchecksOnDeploy(), pendingDeploy.getDeployMarker().getMessage()));
    }
}

Example 10 with SingularityPendingRequest

use of com.hubspot.singularity.SingularityPendingRequest in project Singularity by HubSpot.

the class TaskTrackerResource method getTaskStateByRunId.

@GET
@Path("/run/{requestId}/{runId}")
@Operation(summary = "Get the current state of a task by taskId whether it is pending, active, or inactive", responses = { @ApiResponse(responseCode = "404", description = "Task with this runId does not exist") })
public Optional<SingularityTaskState> getTaskStateByRunId(@Parameter(hidden = true) @Auth SingularityUser user, @Parameter(required = true, description = "the request id to search for tasks") @PathParam("requestId") String requestId, @Parameter(required = true, description = "the run id to search for") @PathParam("runId") String runId) {
    authorizationHelper.checkForAuthorizationByRequestId(requestId, user, SingularityAuthorizationScope.READ);
    // Check if it's active or inactive
    Optional<SingularityTaskId> maybeTaskId = taskManager.getTaskByRunId(requestId, runId);
    if (maybeTaskId.isPresent()) {
        Optional<SingularityTaskState> maybeTaskState = getTaskStateFromId(maybeTaskId.get());
        if (maybeTaskState.isPresent()) {
            return maybeTaskState;
        }
    } else {
        Optional<SingularityTaskHistory> maybeTaskHistory = historyManager.getTaskHistoryByRunId(requestId, runId);
        if (maybeTaskHistory.isPresent()) {
            return Optional.of(SingularityTaskState.fromTaskHistory(maybeTaskHistory.get()));
        }
    }
    // Check if it's pending
    for (SingularityPendingTask pendingTask : taskManager.getPendingTasksForRequest(requestId, false)) {
        if (pendingTask.getRunId().isPresent() && pendingTask.getRunId().get().equals(runId)) {
            return Optional.of(new SingularityTaskState(Optional.empty(), pendingTask.getPendingTaskId(), pendingTask.getRunId(), Optional.empty(), Collections.emptyList(), true));
        }
    }
    for (SingularityPendingRequest pendingRequest : requestManager.getPendingRequests()) {
        if (pendingRequest.getRequestId().equals(requestId) && pendingRequest.getRunId().isPresent() && pendingRequest.getRunId().get().equals(runId)) {
            return Optional.of(new SingularityTaskState(Optional.empty(), Optional.empty(), pendingRequest.getRunId(), Optional.empty(), Collections.emptyList(), true));
        }
    }
    return Optional.empty();
}