Examples with SingularityTaskCleanup com.hubspot.singularity.SingularityTaskCleanup used on opensource projects

Example 11 with SingularityTaskCleanup

use of com.hubspot.singularity.SingularityTaskCleanup in project Singularity by HubSpot.

the class SingularityScheduler method cleanupTaskDueToDecomission.

private void cleanupTaskDueToDecomission(final Map<String, Optional<String>> requestIdsToUserToReschedule, final Set<SingularityTaskId> matchingTaskIds, SingularityTask task, SingularityMachineAbstraction<?> decommissioningObject) {
    requestIdsToUserToReschedule.put(task.getTaskRequest().getRequest().getId(), decommissioningObject.getCurrentState().getUser());
    matchingTaskIds.add(task.getTaskId());
    LOG.trace("Scheduling a cleanup task for {} due to decomissioning {}", task.getTaskId(), decommissioningObject);
    taskManager.createTaskCleanup(new SingularityTaskCleanup(decommissioningObject.getCurrentState().getUser(), TaskCleanupType.DECOMISSIONING, System.currentTimeMillis(), task.getTaskId(), Optional.of(String.format("%s %s is decomissioning", decommissioningObject.getTypeName(), decommissioningObject.getName())), Optional.<String>absent(), Optional.<SingularityTaskShellCommandRequestId>absent()));
}

Example 12 with SingularityTaskCleanup

use of com.hubspot.singularity.SingularityTaskCleanup in project Singularity by HubSpot.

the class TaskResource method killTask.

public SingularityTaskCleanup killTask(String taskId, Optional<SingularityKillTaskRequest> killTaskRequest, SingularityUser user) {
    final SingularityTask task = checkActiveTask(taskId, SingularityAuthorizationScope.WRITE, user);
    Optional<String> message = Optional.absent();
    Optional<Boolean> override = Optional.absent();
    Optional<String> actionId = Optional.absent();
    Optional<Boolean> waitForReplacementTask = Optional.absent();
    Optional<SingularityTaskShellCommandRequestId> runBeforeKillId = Optional.absent();
    if (killTaskRequest.isPresent()) {
        actionId = killTaskRequest.get().getActionId();
        message = killTaskRequest.get().getMessage();
        override = killTaskRequest.get().getOverride();
        waitForReplacementTask = killTaskRequest.get().getWaitForReplacementTask();
        if (killTaskRequest.get().getRunShellCommandBeforeKill().isPresent()) {
            SingularityTaskShellCommandRequest shellCommandRequest = startShellCommand(task.getTaskId(), killTaskRequest.get().getRunShellCommandBeforeKill().get(), user);
            runBeforeKillId = Optional.of(shellCommandRequest.getId());
        }
    }
    TaskCleanupType cleanupType = TaskCleanupType.USER_REQUESTED;
    if (waitForReplacementTask.or(Boolean.FALSE)) {
        cleanupType = TaskCleanupType.USER_REQUESTED_TASK_BOUNCE;
        validator.checkActionEnabled(SingularityAction.BOUNCE_TASK);
    } else {
        validator.checkActionEnabled(SingularityAction.KILL_TASK);
    }
    final long now = System.currentTimeMillis();
    final SingularityTaskCleanup taskCleanup;
    if (override.isPresent() && override.get()) {
        LOG.debug("Requested destroy of {}", taskId);
        cleanupType = TaskCleanupType.USER_REQUESTED_DESTROY;
        taskCleanup = new SingularityTaskCleanup(user.getEmail(), cleanupType, now, task.getTaskId(), message, actionId, runBeforeKillId);
        taskManager.saveTaskCleanup(taskCleanup);
    } else {
        taskCleanup = new SingularityTaskCleanup(user.getEmail(), cleanupType, now, task.getTaskId(), message, actionId, runBeforeKillId);
        SingularityCreateResult result = taskManager.createTaskCleanup(taskCleanup);
        if (result == SingularityCreateResult.EXISTED && userRequestedKillTakesPriority(taskId)) {
            taskManager.saveTaskCleanup(taskCleanup);
        } else {
            while (result == SingularityCreateResult.EXISTED) {
                Optional<SingularityTaskCleanup> cleanup = taskManager.getTaskCleanup(taskId);
                if (cleanup.isPresent()) {
                    throw new WebApplicationException(Response.status(Status.CONFLICT).entity(cleanup.get()).type(MediaType.APPLICATION_JSON).build());
                }
                result = taskManager.createTaskCleanup(taskCleanup);
            }
        }
    }
    if (cleanupType == TaskCleanupType.USER_REQUESTED_TASK_BOUNCE) {
        requestManager.addToPendingQueue(new SingularityPendingRequest(task.getTaskId().getRequestId(), task.getTaskId().getDeployId(), now, user.getEmail(), PendingType.TASK_BOUNCE, Optional.<List<String>>absent(), Optional.<String>absent(), Optional.<Boolean>absent(), message, actionId));
    }
    return taskCleanup;
}

Example 13 with SingularityTaskCleanup

use of com.hubspot.singularity.SingularityTaskCleanup in project Singularity by HubSpot.

the class SingularityUsagePoller method shuffleTasksOnOverloadedHosts.

private void shuffleTasksOnOverloadedHosts(Map<SingularitySlaveUsage, List<TaskIdWithUsage>> overLoadedHosts) {
    List<SingularityTaskCleanup> shuffleCleanups = taskManager.getCleanupTasks().stream().filter((taskCleanup) -> taskCleanup.getCleanupType() == TaskCleanupType.REBALANCE_CPU_USAGE).collect(Collectors.toList());
    long currentShuffleCleanupsTotal = shuffleCleanups.size();
    Set<String> requestsWithShuffledTasks = shuffleCleanups.stream().map((taskCleanup) -> taskCleanup.getTaskId().getRequestId()).collect(Collectors.toSet());
    List<SingularitySlaveUsage> overLoadedSlavesByUsage = overLoadedHosts.keySet().stream().sorted((usage1, usage2) -> Double.compare(getSystemLoadForShuffle(usage2), getSystemLoadForShuffle(usage1))).collect(Collectors.toList());
    for (SingularitySlaveUsage overloadedSlave : overLoadedSlavesByUsage) {
        if (currentShuffleCleanupsTotal >= configuration.getMaxTasksToShuffleTotal()) {
            LOG.debug("Not shuffling any more tasks (totalShuffleCleanups: {})", currentShuffleCleanupsTotal);
            break;
        }
        int shuffledTasksOnSlave = 0;
        List<TaskIdWithUsage> possibleTasksToShuffle = overLoadedHosts.get(overloadedSlave);
        possibleTasksToShuffle.sort((u1, u2) -> Double.compare(u2.getUsage().getCpusUsed() / u2.getRequestedResources().getCpus(), u1.getUsage().getCpusUsed() / u1.getRequestedResources().getCpus()));
        double systemLoad = getSystemLoadForShuffle(overloadedSlave);
        double cpuOverage = systemLoad - overloadedSlave.getSystemCpusTotal();
        for (TaskIdWithUsage taskIdWithUsage : possibleTasksToShuffle) {
            if (requestsWithShuffledTasks.contains(taskIdWithUsage.getTaskId().getRequestId())) {
                LOG.debug("Request {} already has a shuffling task, skipping", taskIdWithUsage.getTaskId().getRequestId());
                continue;
            }
            if (cpuOverage <= 0 || shuffledTasksOnSlave > configuration.getMaxTasksToShufflePerHost() || currentShuffleCleanupsTotal >= configuration.getMaxTasksToShuffleTotal()) {
                LOG.debug("Not shuffling any more tasks (overage: {}, shuffledOnHost: {}, totalShuffleCleanups: {})", cpuOverage, shuffledTasksOnSlave, currentShuffleCleanupsTotal);
                break;
            }
            LOG.debug("Cleaning up task {} to free up cpu on overloaded host (remaining cpu overage: {})", taskIdWithUsage.getTaskId(), cpuOverage);
            Optional<String> message = Optional.of(String.format("Load on slave is %s / %s, shuffling task to less busy host", systemLoad, overloadedSlave.getSystemCpusTotal()));
            taskManager.createTaskCleanup(new SingularityTaskCleanup(Optional.absent(), TaskCleanupType.REBALANCE_CPU_USAGE, System.currentTimeMillis(), taskIdWithUsage.getTaskId(), message, Optional.of(UUID.randomUUID().toString()), Optional.absent(), Optional.absent()));
            requestManager.addToPendingQueue(new SingularityPendingRequest(taskIdWithUsage.getTaskId().getRequestId(), taskIdWithUsage.getTaskId().getDeployId(), System.currentTimeMillis(), Optional.absent(), PendingType.TASK_BOUNCE, Optional.absent(), Optional.absent(), Optional.absent(), message, Optional.of(UUID.randomUUID().toString())));
            cpuOverage -= taskIdWithUsage.getUsage().getCpusUsed();
            shuffledTasksOnSlave++;
            currentShuffleCleanupsTotal++;
            requestsWithShuffledTasks.add(taskIdWithUsage.getTaskId().getRequestId());
        }
    }
}

Example 14 with SingularityTaskCleanup

use of com.hubspot.singularity.SingularityTaskCleanup in project Singularity by HubSpot.

the class SingularityCleaner method delete.

private void delete(SingularityRequestCleanup requestCleanup, Iterable<SingularityTaskId> activeTaskIds) {
    final long start = System.currentTimeMillis();
    for (SingularityTaskId taskId : activeTaskIds) {
        LOG.debug("Adding task {} to cleanup (delete)", taskId.getId());
        Optional<SingularityTaskShellCommandRequestId> runBeforeKillId = Optional.absent();
        if (requestCleanup.getRunShellCommandBeforeKill().isPresent()) {
            SingularityTaskShellCommandRequest shellRequest = new SingularityTaskShellCommandRequest(taskId, requestCleanup.getUser(), System.currentTimeMillis(), requestCleanup.getRunShellCommandBeforeKill().get());
            taskManager.saveTaskShellCommandRequestToQueue(shellRequest);
            runBeforeKillId = Optional.of(shellRequest.getId());
        }
        taskManager.createTaskCleanup(new SingularityTaskCleanup(requestCleanup.getUser(), TaskCleanupType.REQUEST_DELETING, start, taskId, requestCleanup.getMessage(), requestCleanup.getActionId(), runBeforeKillId, requestCleanup.getRemoveFromLoadBalancer()));
    }
}

Example 15 with SingularityTaskCleanup

use of com.hubspot.singularity.SingularityTaskCleanup in project Singularity by HubSpot.

the class SingularitySchedulerTest method testDeployCleanupOverwritesTaskBounceCleanup.

@Test
public void testDeployCleanupOverwritesTaskBounceCleanup() {
    initRequest();
    initFirstDeploy();
    final SingularityTask oldTask = startTask(firstDeploy);
    taskResource.killTask(oldTask.getTaskId().getId(), Optional.of(new SingularityKillTaskRequest(Optional.absent(), Optional.absent(), Optional.absent(), Optional.of(true), Optional.absent())), singularityUser);
    final Optional<SingularityTaskCleanup> taskCleanup = taskManager.getTaskCleanup(oldTask.getTaskId().getId());
    Assert.assertTrue(taskCleanup.isPresent());
    Assert.assertEquals(TaskCleanupType.USER_REQUESTED_TASK_BOUNCE, taskCleanup.get().getCleanupType());
    initSecondDeploy();
    startTask(secondDeploy);
    deployChecker.checkDeploys();
    Assert.assertEquals(DeployState.SUCCEEDED, deployManager.getDeployResult(requestId, secondDeployId).get().getDeployState());
    Assert.assertEquals(TaskCleanupType.DEPLOY_STEP_FINISHED, taskManager.getTaskCleanup(oldTask.getTaskId().getId()).get().getCleanupType());
    cleaner.drainCleanupQueue();
    Assert.assertFalse(taskManager.getTaskCleanup(oldTask.getTaskId().getId()).isPresent());
}