Examples with SingularityTaskCleanup com.hubspot.singularity.SingularityTaskCleanup used on opensource projects

Example 6 with SingularityTaskCleanup

use of com.hubspot.singularity.SingularityTaskCleanup in project Singularity by HubSpot.

the class SingularityCleaner method processTaskCleanupsForRequest.

private void processTaskCleanupsForRequest(String requestId, List<SingularityTaskCleanup> cleanupTasks, AtomicInteger killedTasks) {
    final Multiset<SingularityDeployKey> incrementalCleaningTasks = HashMultiset.create(cleanupTasks.size());
    final List<String> taskIdsForDeletedRequest = new ArrayList<>();
    boolean isRequestDeleting = false;
    // TODO - Better check for deleting request state
    final Set<SingularityTaskId> cleaningTasks = new HashSet<>(cleanupTasks.size());
    for (SingularityTaskCleanup cleanupTask : cleanupTasks) {
        cleaningTasks.add(cleanupTask.getTaskId());
        if (isIncrementalDeployCleanup(cleanupTask) || cleanupTask.getCleanupType() == TaskCleanupType.INCREMENTAL_BOUNCE) {
            incrementalCleaningTasks.add(SingularityDeployKey.fromTaskId(cleanupTask.getTaskId()));
        }
        if (cleanupTask.getCleanupType() == TaskCleanupType.REQUEST_DELETING) {
            taskIdsForDeletedRequest.add(cleanupTask.getTaskId().getId());
            isRequestDeleting = true;
        }
    }
    LOG.info("Cleaning up {} tasks for request {}", cleanupTasks.size(), requestId);
    final List<SingularityTaskId> activeTaskIds = taskManager.getActiveTaskIds();
    for (SingularityTaskCleanup cleanupTask : cleanupTasks) {
        SingularityTaskId taskId = cleanupTask.getTaskId();
        if (!isValidTask(cleanupTask)) {
            LOG.info("Couldn't find a matching active task for cleanup task {}, deleting..", cleanupTask);
            taskManager.deleteCleanupTask(taskId.getId());
        } else if (shouldKillTask(cleanupTask, activeTaskIds, cleaningTasks, incrementalCleaningTasks) && checkLBStateAndShouldKillTask(cleanupTask)) {
            scheduler.killAndRecord(taskId, cleanupTask.getCleanupType(), cleanupTask.getUser());
            taskManager.deleteCleanupTask(taskId.getId());
            killedTasks.getAndIncrement();
        }
        cleanupRequestIfNoRemainingTasks(cleanupTask, taskIdsForDeletedRequest, isRequestDeleting);
    }
}

Example 7 with SingularityTaskCleanup

use of com.hubspot.singularity.SingularityTaskCleanup in project Singularity by HubSpot.

the class SingularityCleaner method drainTaskCleanupQueue.

private int drainTaskCleanupQueue() {
    final long start = System.currentTimeMillis();
    final Map<String, List<SingularityTaskCleanup>> cleanupTasks = taskManager.getCleanupTasks().stream().collect(Collectors.groupingBy((taskCleanup) -> taskCleanup.getTaskId().getRequestId()));
    if (cleanupTasks.isEmpty()) {
        LOG.trace("Task cleanup queue is empty");
        return 0;
    }
    AtomicInteger killedTasks = new AtomicInteger(0);
    cleanupTasks.entrySet().parallelStream().forEach((taskCleanupsForRequest) -> {
        lock.runWithRequestLock(() -> {
            processTaskCleanupsForRequest(taskCleanupsForRequest.getKey(), taskCleanupsForRequest.getValue(), killedTasks);
        }, taskCleanupsForRequest.getKey(), String.format("%s#%s", getClass().getSimpleName(), "drainTaskCleanupQueue"));
    });
    LOG.info("Killed {} tasks in {}", killedTasks, JavaUtils.duration(start));
    return cleanupTasks.size();
}

Example 8 with SingularityTaskCleanup

use of com.hubspot.singularity.SingularityTaskCleanup in project Singularity by HubSpot.

the class SingularityCleaner method bounce.

private void bounce(SingularityRequestCleanup requestCleanup, final List<SingularityTaskId> activeTaskIds) {
    final long start = System.currentTimeMillis();
    final List<SingularityTaskId> matchingTaskIds = new ArrayList<>();
    for (SingularityTaskId activeTaskId : activeTaskIds) {
        if (activeTaskId.getRequestId().equals(requestCleanup.getRequestId()) && activeTaskId.getDeployId().equals(requestCleanup.getDeployId().get())) {
            matchingTaskIds.add(activeTaskId);
        }
    }
    for (SingularityTaskId matchingTaskId : matchingTaskIds) {
        LOG.debug("Adding task {} to cleanup (bounce)", matchingTaskId.getId());
        Optional<SingularityTaskShellCommandRequestId> runBeforeKillId = Optional.absent();
        if (requestCleanup.getRunShellCommandBeforeKill().isPresent()) {
            SingularityTaskShellCommandRequest shellRequest = new SingularityTaskShellCommandRequest(matchingTaskId, requestCleanup.getUser(), System.currentTimeMillis(), requestCleanup.getRunShellCommandBeforeKill().get());
            taskManager.saveTaskShellCommandRequestToQueue(shellRequest);
            runBeforeKillId = Optional.of(shellRequest.getId());
        }
        taskManager.createTaskCleanup(new SingularityTaskCleanup(requestCleanup.getUser(), requestCleanup.getCleanupType().getTaskCleanupType().get(), start, matchingTaskId, requestCleanup.getMessage(), requestCleanup.getActionId(), runBeforeKillId));
    }
    if (matchingTaskIds.isEmpty() && requestCleanup.getDeployId().isPresent()) {
        Optional<SingularityExpiringBounce> expiringBounce = requestManager.getExpiringBounce(requestCleanup.getRequestId());
        if (expiringBounce.isPresent() && expiringBounce.get().getDeployId().equalsIgnoreCase(requestCleanup.getDeployId().get())) {
            LOG.info("No running tasks for request {}. Marking bounce {} complete and starting new tasks", expiringBounce.get().getRequestId(), expiringBounce.get());
            requestManager.deleteExpiringObject(SingularityExpiringBounce.class, requestCleanup.getRequestId());
        }
        requestManager.markBounceComplete(requestCleanup.getRequestId());
    }
    requestManager.addToPendingQueue(new SingularityPendingRequest(requestCleanup.getRequestId(), requestCleanup.getDeployId().get(), requestCleanup.getTimestamp(), requestCleanup.getUser(), PendingType.BOUNCE, Optional.absent(), Optional.absent(), requestCleanup.getSkipHealthchecks(), requestCleanup.getMessage(), requestCleanup.getActionId()));
    LOG.info("Added {} tasks for request {} to cleanup bounce queue in {}", matchingTaskIds.size(), requestCleanup.getRequestId(), JavaUtils.duration(start));
}

Example 9 with SingularityTaskCleanup

use of com.hubspot.singularity.SingularityTaskCleanup in project Singularity by HubSpot.

the class SingularityCleaner method pause.

private TaskCleanupType pause(SingularityRequestCleanup requestCleanup, Iterable<SingularityTaskId> activeTaskIds) {
    final long start = System.currentTimeMillis();
    boolean killTasks = requestCleanup.getKillTasks().or(configuration.isDefaultValueForKillTasksOfPausedRequests());
    if (requestCleanup.getRunShellCommandBeforeKill().isPresent()) {
        killTasks = false;
    }
    TaskCleanupType cleanupType = killTasks ? TaskCleanupType.PAUSE : TaskCleanupType.PAUSING;
    for (SingularityTaskId taskId : activeTaskIds) {
        LOG.debug("Adding task {} to cleanup (pause)", taskId.getId());
        Optional<SingularityTaskShellCommandRequestId> runBeforeKillId = Optional.absent();
        if (requestCleanup.getRunShellCommandBeforeKill().isPresent()) {
            SingularityTaskShellCommandRequest shellRequest = new SingularityTaskShellCommandRequest(taskId, requestCleanup.getUser(), System.currentTimeMillis(), requestCleanup.getRunShellCommandBeforeKill().get());
            taskManager.saveTaskShellCommandRequestToQueue(shellRequest);
            runBeforeKillId = Optional.of(shellRequest.getId());
        }
        taskManager.createTaskCleanup(new SingularityTaskCleanup(requestCleanup.getUser(), cleanupType, start, taskId, requestCleanup.getMessage(), requestCleanup.getActionId(), runBeforeKillId));
    }
    return cleanupType;
}

Example 10 with SingularityTaskCleanup

use of com.hubspot.singularity.SingularityTaskCleanup in project Singularity by HubSpot.

the class SingularityPriorityKillPoller method runActionOnPoll.

@Override
public void runActionOnPoll() {
    if (!priorityManager.checkPriorityKillExists()) {
        LOG.trace("No priority freeze to process.");
        return;
    }
    final Optional<SingularityPriorityFreezeParent> maybePriorityFreeze = priorityManager.getActivePriorityFreeze();
    if (!maybePriorityFreeze.isPresent() || !maybePriorityFreeze.get().getPriorityFreeze().isKillTasks()) {
        LOG.trace("Priority freeze does not exist.");
        priorityManager.clearPriorityKill();
        return;
    }
    LOG.info("Handling priority freeze {}", maybePriorityFreeze.get());
    final long now = System.currentTimeMillis();
    int cancelledPendingTaskCount = 0;
    int killedTaskCount = 0;
    try {
        final double minPriorityLevel = maybePriorityFreeze.get().getPriorityFreeze().getMinimumPriorityLevel();
        // map request ID to priority level
        final Map<String, Double> requestIdToTaskPriority = new HashMap<>();
        for (SingularityRequestWithState requestWithState : requestManager.getRequests()) {
            requestIdToTaskPriority.put(requestWithState.getRequest().getId(), priorityManager.getTaskPriorityLevelForRequest(requestWithState.getRequest()));
        }
        // kill active tasks below minimum priority level
        for (SingularityTaskId taskId : taskManager.getActiveTaskIds()) {
            if (!requestIdToTaskPriority.containsKey(taskId.getRequestId())) {
                LOG.trace("Unable to lookup priority level for task {}, skipping...", taskId);
                continue;
            }
            final double taskPriorityLevel = requestIdToTaskPriority.get(taskId.getRequestId());
            if (taskPriorityLevel < minPriorityLevel) {
                LOG.info("Killing active task {} since priority level {} is less than {}", taskId.getId(), taskPriorityLevel, minPriorityLevel);
                taskManager.createTaskCleanup(new SingularityTaskCleanup(maybePriorityFreeze.get().getUser(), TaskCleanupType.PRIORITY_KILL, now, taskId, maybePriorityFreeze.get().getPriorityFreeze().getMessage(), maybePriorityFreeze.get().getPriorityFreeze().getActionId(), Optional.<SingularityTaskShellCommandRequestId>absent()));
                killedTaskCount++;
            }
        }
    } finally {
        priorityManager.clearPriorityKill();
        LOG.info("Finished killing active tasks for priority freeze {} in {} for {} active tasks, {} pending tasks", maybePriorityFreeze, JavaUtils.duration(now), killedTaskCount, cancelledPendingTaskCount);
    }
}