Examples with SingularityTaskId com.hubspot.singularity.SingularityTaskId used on opensource projects

Example 56 with SingularityTaskId

use of com.hubspot.singularity.SingularityTaskId in project Singularity by HubSpot.

the class SingularityCleaner method pause.

private TaskCleanupType pause(SingularityRequestCleanup requestCleanup, Iterable<SingularityTaskId> activeTaskIds) {
    final long start = System.currentTimeMillis();
    boolean killTasks = requestCleanup.getKillTasks().or(configuration.isDefaultValueForKillTasksOfPausedRequests());
    if (requestCleanup.getRunShellCommandBeforeKill().isPresent()) {
        killTasks = false;
    }
    TaskCleanupType cleanupType = killTasks ? TaskCleanupType.PAUSE : TaskCleanupType.PAUSING;
    for (SingularityTaskId taskId : activeTaskIds) {
        LOG.debug("Adding task {} to cleanup (pause)", taskId.getId());
        Optional<SingularityTaskShellCommandRequestId> runBeforeKillId = Optional.absent();
        if (requestCleanup.getRunShellCommandBeforeKill().isPresent()) {
            SingularityTaskShellCommandRequest shellRequest = new SingularityTaskShellCommandRequest(taskId, requestCleanup.getUser(), System.currentTimeMillis(), requestCleanup.getRunShellCommandBeforeKill().get());
            taskManager.saveTaskShellCommandRequestToQueue(shellRequest);
            runBeforeKillId = Optional.of(shellRequest.getId());
        }
        taskManager.createTaskCleanup(new SingularityTaskCleanup(requestCleanup.getUser(), cleanupType, start, taskId, requestCleanup.getMessage(), requestCleanup.getActionId(), runBeforeKillId));
    }
    return cleanupType;
}

Example 57 with SingularityTaskId

use of com.hubspot.singularity.SingularityTaskId in project Singularity by HubSpot.

the class SingularityScheduler method handlePendingRequestsForDeployKey.

private void handlePendingRequestsForDeployKey(AtomicInteger obsoleteRequests, AtomicInteger heldForScheduledActiveTask, AtomicInteger totalNewScheduledTasks, SingularityDeployKey deployKey, List<SingularityPendingRequest> pendingRequestsForDeploy) {
    final String requestId = deployKey.getRequestId();
    final Optional<SingularityRequestWithState> maybeRequest = requestManager.getRequest(requestId);
    final SingularityDeployStatistics deployStatistics = getDeployStatistics(deployKey.getRequestId(), deployKey.getDeployId());
    if (!isRequestActive(maybeRequest)) {
        LOG.debug("Pending request {} was obsolete (request {})", requestId, SingularityRequestWithState.getRequestState(maybeRequest));
        obsoleteRequests.getAndIncrement();
        for (SingularityPendingRequest pendingRequest : pendingRequestsForDeploy) {
            requestManager.deletePendingRequest(pendingRequest);
        }
        return;
    }
    SingularityRequestWithState request = maybeRequest.get();
    Optional<SingularityRequestDeployState> maybeRequestDeployState = deployManager.getRequestDeployState(requestId);
    Optional<SingularityPendingDeploy> maybePendingDeploy = deployManager.getPendingDeploy(requestId);
    List<SingularityTaskId> matchingTaskIds = getMatchingTaskIds(request.getRequest(), deployKey);
    List<SingularityPendingRequest> effectivePendingRequests = new ArrayList<>();
    // Things that are closest to now (ie smaller timestamps) should come first in the queue
    pendingRequestsForDeploy.sort(Comparator.comparingLong(SingularityPendingRequest::getTimestamp));
    int scheduledTasks = 0;
    for (SingularityPendingRequest pendingRequest : pendingRequestsForDeploy) {
        final SingularityRequest updatedRequest = updatedRequest(maybePendingDeploy, pendingRequest, request);
        if (!shouldScheduleTasks(updatedRequest, pendingRequest, maybePendingDeploy, maybeRequestDeployState)) {
            LOG.debug("Pending request {} was obsolete (request {})", pendingRequest, SingularityRequestWithState.getRequestState(maybeRequest));
            obsoleteRequests.getAndIncrement();
            requestManager.deletePendingRequest(pendingRequest);
            continue;
        }
        int missingInstances = getNumMissingInstances(matchingTaskIds, updatedRequest, pendingRequest, maybePendingDeploy);
        if (missingInstances == 0 && !matchingTaskIds.isEmpty() && updatedRequest.isScheduled() && pendingRequest.getPendingType() == PendingType.NEW_DEPLOY) {
            LOG.trace("Holding pending request {} because it is scheduled and has an active task", pendingRequest);
            heldForScheduledActiveTask.getAndIncrement();
            continue;
        }
        if (effectivePendingRequests.isEmpty()) {
            effectivePendingRequests.add(pendingRequest);
            RequestState requestState = checkCooldown(request.getState(), request.getRequest(), deployStatistics);
            scheduledTasks += scheduleTasks(request.getRequest(), requestState, deployStatistics, pendingRequest, matchingTaskIds, maybePendingDeploy);
            requestManager.deletePendingRequest(pendingRequest);
        } else if (pendingRequest.getPendingType() == PendingType.IMMEDIATE) {
            effectivePendingRequests.add(pendingRequest);
            RequestState requestState = checkCooldown(request.getState(), request.getRequest(), deployStatistics);
            scheduledTasks += scheduleTasks(request.getRequest(), requestState, deployStatistics, pendingRequest, matchingTaskIds, maybePendingDeploy);
            requestManager.deletePendingRequest(pendingRequest);
        } else if (pendingRequest.getPendingType() == PendingType.ONEOFF) {
            effectivePendingRequests.add(pendingRequest);
            RequestState requestState = checkCooldown(request.getState(), request.getRequest(), deployStatistics);
            scheduledTasks += scheduleTasks(request.getRequest(), requestState, deployStatistics, pendingRequest, matchingTaskIds, maybePendingDeploy);
            requestManager.deletePendingRequest(pendingRequest);
        } else if (updatedRequest.isScheduled() && (pendingRequest.getPendingType() == PendingType.NEW_DEPLOY || pendingRequest.getPendingType() == PendingType.TASK_DONE)) {
            // If we are here, there is already an immediate of run of the scheduled task launched. Drop anything that would
            // leave a second instance of the request in the pending queue.
            requestManager.deletePendingRequest(pendingRequest);
        }
    // Any other subsequent requests are not honored until after the pending queue is cleared.
    }
    totalNewScheduledTasks.getAndAdd(scheduledTasks);
}

Example 58 with SingularityTaskId

use of com.hubspot.singularity.SingularityTaskId in project Singularity by HubSpot.

the class SingularityTaskReconciliation method startReconciliation.

public ReconciliationState startReconciliation() {
    final long taskReconciliationStartedAt = System.currentTimeMillis();
    if (!isRunningReconciliation.compareAndSet(false, true)) {
        LOG.info("Reconciliation is already running, NOT starting a new reconciliation process");
        return ReconciliationState.ALREADY_RUNNING;
    }
    if (!schedulerClient.isRunning()) {
        LOG.trace("Not running reconciliation - no active scheduler present");
        isRunningReconciliation.set(false);
        return ReconciliationState.NO_DRIVER;
    }
    final List<SingularityTaskId> activeTaskIds = taskManager.getActiveTaskIds();
    LOG.info("Starting a reconciliation cycle - {} current active tasks", activeTaskIds.size());
    schedulerClient.reconcile(Collections.emptyList());
    scheduleReconciliationCheck(taskReconciliationStartedAt, activeTaskIds, 0, new Histogram(new UniformReservoir()));
    return ReconciliationState.STARTED;
}

Example 59 with SingularityTaskId

use of com.hubspot.singularity.SingularityTaskId in project Singularity by HubSpot.

the class SingularityJobPoller method runActionOnPoll.

@Override
public void runActionOnPoll() {
    final long now = System.currentTimeMillis();
    final List<SingularityTaskId> activeTaskIds = taskManager.getActiveTaskIds();
    final Set<String> requestIdsToLookup = Sets.newHashSetWithExpectedSize(activeTaskIds.size());
    for (SingularityTaskId taskId : activeTaskIds) {
        requestIdsToLookup.add(taskId.getRequestId());
    }
    final Map<String, SingularityRequestWithState> idToRequest = Maps.uniqueIndex(requestManager.getRequests(requestIdsToLookup), SingularityRequestWithState.REQUEST_STATE_TO_REQUEST_ID);
    for (SingularityTaskId taskId : activeTaskIds) {
        SingularityRequestWithState requestWithState = idToRequest.get(taskId.getRequestId());
        if (requestWithState == null) {
            LOG.warn("Active request not found for task ID {}", taskId);
            continue;
        }
        SingularityRequest request = requestWithState.getRequest();
        if (!request.isLongRunning()) {
            checkForOverdueScheduledJob(now - taskId.getStartedAt(), taskId, request);
            checkTaskExecutionTimeLimit(now, taskId, request);
        }
    }
}

Example 60 with SingularityTaskId

use of com.hubspot.singularity.SingularityTaskId in project Singularity by HubSpot.

the class SingularityPriorityKillPoller method runActionOnPoll.

@Override
public void runActionOnPoll() {
    if (!priorityManager.checkPriorityKillExists()) {
        LOG.trace("No priority freeze to process.");
        return;
    }
    final Optional<SingularityPriorityFreezeParent> maybePriorityFreeze = priorityManager.getActivePriorityFreeze();
    if (!maybePriorityFreeze.isPresent() || !maybePriorityFreeze.get().getPriorityFreeze().isKillTasks()) {
        LOG.trace("Priority freeze does not exist.");
        priorityManager.clearPriorityKill();
        return;
    }
    LOG.info("Handling priority freeze {}", maybePriorityFreeze.get());
    final long now = System.currentTimeMillis();
    int cancelledPendingTaskCount = 0;
    int killedTaskCount = 0;
    try {
        final double minPriorityLevel = maybePriorityFreeze.get().getPriorityFreeze().getMinimumPriorityLevel();
        // map request ID to priority level
        final Map<String, Double> requestIdToTaskPriority = new HashMap<>();
        for (SingularityRequestWithState requestWithState : requestManager.getRequests()) {
            requestIdToTaskPriority.put(requestWithState.getRequest().getId(), priorityManager.getTaskPriorityLevelForRequest(requestWithState.getRequest()));
        }
        // kill active tasks below minimum priority level
        for (SingularityTaskId taskId : taskManager.getActiveTaskIds()) {
            if (!requestIdToTaskPriority.containsKey(taskId.getRequestId())) {
                LOG.trace("Unable to lookup priority level for task {}, skipping...", taskId);
                continue;
            }
            final double taskPriorityLevel = requestIdToTaskPriority.get(taskId.getRequestId());
            if (taskPriorityLevel < minPriorityLevel) {
                LOG.info("Killing active task {} since priority level {} is less than {}", taskId.getId(), taskPriorityLevel, minPriorityLevel);
                taskManager.createTaskCleanup(new SingularityTaskCleanup(maybePriorityFreeze.get().getUser(), TaskCleanupType.PRIORITY_KILL, now, taskId, maybePriorityFreeze.get().getPriorityFreeze().getMessage(), maybePriorityFreeze.get().getPriorityFreeze().getActionId(), Optional.<SingularityTaskShellCommandRequestId>absent()));
                killedTaskCount++;
            }
        }
    } finally {
        priorityManager.clearPriorityKill();
        LOG.info("Finished killing active tasks for priority freeze {} in {} for {} active tasks, {} pending tasks", maybePriorityFreeze, JavaUtils.duration(now), killedTaskCount, cancelledPendingTaskCount);
    }
}