Examples with SingularityTaskCleanup com.hubspot.singularity.SingularityTaskCleanup used on opensource projects

Example 1 with SingularityTaskCleanup

use of com.hubspot.singularity.SingularityTaskCleanup in project Singularity by HubSpot.

the class SingularitySchedulerTest method testBounceReleasesLockWithAlternateCleanupType.

@Test
public void testBounceReleasesLockWithAlternateCleanupType() {
    initRequest();
    initFirstDeploy();
    startTask(firstDeploy, 1);
    List<SingularityTaskId> activeTaskIds = taskManager.getActiveTaskIds();
    Assert.assertEquals(1, activeTaskIds.size());
    SingularityTaskId firstTaskId = activeTaskIds.get(0);
    requestResource.bounce(requestId, Optional.of(new SingularityBounceRequest(Optional.absent(), Optional.of(true), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent())), singularityUser);
    Assert.assertTrue(requestManager.isBouncing(requestId));
    cleaner.drainCleanupQueue();
    scheduler.drainPendingQueue();
    resourceOffers();
    // Save a new cleanup type over the old one, and make sure the bounce lock still releases
    taskManager.saveTaskCleanup(new SingularityTaskCleanup(Optional.absent(), TaskCleanupType.USER_REQUESTED, System.currentTimeMillis(), firstTaskId, Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent()));
    for (SingularityTaskId singularityTaskId : taskManager.getActiveTaskIds()) {
        taskManager.saveTaskHistoryUpdate(new SingularityTaskHistoryUpdate(singularityTaskId, System.currentTimeMillis(), ExtendedTaskState.TASK_RUNNING, Optional.absent(), Optional.absent(), Collections.emptySet()));
    }
    Assert.assertTrue(requestManager.isBouncing(requestId));
    cleaner.drainCleanupQueue();
    killKilledTasks();
    Assert.assertFalse(requestManager.isBouncing(requestId));
}

Example 2 with SingularityTaskCleanup

use of com.hubspot.singularity.SingularityTaskCleanup in project Singularity by HubSpot.

the class SingularityTaskShellCommandTest method testRunCommandBeforeBounceKill.

@Test
public void testRunCommandBeforeBounceKill() {
    setShellCommandsConfiguration();
    initRequest();
    initFirstDeploy();
    launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING);
    requestResource.bounce(requestId, Optional.of(new SingularityBounceRequest(Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.of(new SingularityShellCommand("d1", Optional.of(Arrays.asList("o1", "o2")), user, Optional.absent())))), singularityUser);
    cleaner.drainCleanupQueue();
    List<SingularityTaskCleanup> taskCleanups = taskManager.getCleanupTasks();
    Assert.assertTrue(taskCleanups.get(0).getRunBeforeKillId().isPresent());
    SingularityTaskShellCommandRequestId shellCommandRequestId = taskCleanups.get(0).getRunBeforeKillId().get();
    cleaner.drainCleanupQueue();
    Assert.assertEquals(1, taskManager.getCleanupTaskIds().size());
    launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING);
    Assert.assertEquals(2, taskManager.getActiveTaskIdsForRequest(requestId).size());
    cleaner.drainCleanupQueue();
    Assert.assertEquals(2, taskManager.getActiveTaskIdsForRequest(requestId).size());
    taskManager.saveTaskShellCommandUpdate(new SingularityTaskShellCommandUpdate(shellCommandRequestId, System.currentTimeMillis(), Optional.<String>absent(), Optional.<String>absent(), UpdateType.ACKED));
    cleaner.drainCleanupQueue();
    Assert.assertEquals(2, taskManager.getActiveTaskIdsForRequest(requestId).size());
    taskManager.saveTaskShellCommandUpdate(new SingularityTaskShellCommandUpdate(shellCommandRequestId, System.currentTimeMillis(), Optional.<String>absent(), Optional.<String>absent(), UpdateType.FINISHED));
    cleaner.drainCleanupQueue();
    Assert.assertEquals(1, taskManager.getKilledTaskIdRecords().size());
}

Example 3 with SingularityTaskCleanup

use of com.hubspot.singularity.SingularityTaskCleanup in project Singularity by HubSpot.

the class StateManagerTest method itDoesntCountCleaningTasks.

@Test
public void itDoesntCountCleaningTasks() {
    initRequest();
    initFirstDeploy();
    SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest();
    saveAndSchedule(request.toBuilder().setInstances(Optional.of(3)));
    resourceOffers();
    Assert.assertEquals(3, taskManager.getActiveTaskIds().size());
    Assert.assertEquals(0, stateManager.getState(true, false).getOverProvisionedRequests());
    Assert.assertEquals(0, stateManager.getState(true, false).getUnderProvisionedRequests());
    SingularityTask task = taskManager.getActiveTasks().get(0);
    statusUpdate(task, TaskState.TASK_KILLED);
    scheduler.drainPendingQueue();
    taskManager.createTaskCleanup(new SingularityTaskCleanup(Optional.absent(), TaskCleanupType.BOUNCING, 1L, task.getTaskId(), Optional.absent(), Optional.absent(), Optional.absent()));
    Assert.assertEquals(2, taskManager.getActiveTaskIds().size());
    Assert.assertEquals(0, stateManager.getState(true, false).getOverProvisionedRequests());
    Assert.assertEquals(1, stateManager.getState(true, false).getUnderProvisionedRequests());
    launchTask(request, firstDeploy, 4, TaskState.TASK_RUNNING);
    launchTask(request, firstDeploy, 5, TaskState.TASK_RUNNING);
    Assert.assertEquals(4, taskManager.getActiveTaskIds().size());
    Assert.assertEquals(0, stateManager.getState(true, false).getUnderProvisionedRequests());
    Assert.assertEquals(1, stateManager.getState(true, false).getOverProvisionedRequests());
}

Example 4 with SingularityTaskCleanup

use of com.hubspot.singularity.SingularityTaskCleanup in project Singularity by HubSpot.

the class SingularityDeployChecker method markStepFinished.

private SingularityDeployResult markStepFinished(SingularityPendingDeploy pendingDeploy, Optional<SingularityDeploy> deploy, Collection<SingularityTaskId> deployActiveTasks, Collection<SingularityTaskId> otherActiveTasks, SingularityRequest request, Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest) {
    SingularityDeployProgress deployProgress = pendingDeploy.getDeployProgress().get();
    if (updatePendingDeployRequest.isPresent() && getNewTargetInstances(deployProgress, request, updatePendingDeployRequest) != deployProgress.getTargetActiveInstances()) {
        maybeUpdatePendingRequest(pendingDeploy, deploy, request, updatePendingDeployRequest);
        return new SingularityDeployResult(DeployState.WAITING);
    }
    SingularityDeployProgress newProgress = deployProgress.withNewActiveInstances(deployActiveTasks.size()).withCompletedStep();
    DeployState deployState = isLastStepFinished(newProgress, request) ? DeployState.SUCCEEDED : DeployState.WAITING;
    String message = deployState == DeployState.SUCCEEDED ? "New deploy succeeded" : "New deploy is progressing, this task is being replaced";
    updatePendingDeploy(pendingDeploy, pendingDeploy.getLastLoadBalancerUpdate(), deployState, Optional.of(newProgress));
    for (SingularityTaskId taskId : tasksToShutDown(deployProgress, otherActiveTasks, request)) {
        taskManager.createTaskCleanup(new SingularityTaskCleanup(Optional.<String>absent(), TaskCleanupType.DEPLOY_STEP_FINISHED, System.currentTimeMillis(), taskId, Optional.of(message), Optional.<String>absent(), Optional.<SingularityTaskShellCommandRequestId>absent()));
    }
    return new SingularityDeployResult(deployState);
}

Example 5 with SingularityTaskCleanup

use of com.hubspot.singularity.SingularityTaskCleanup in project Singularity by HubSpot.

the class SingularityNewTaskChecker method checkTask.

private boolean checkTask(SingularityTask task, Optional<SingularityRequestWithState> requestWithState, SingularityHealthchecker healthchecker) {
    final long start = System.currentTimeMillis();
    final CheckTaskState state = getTaskState(task, requestWithState, healthchecker);
    LOG.debug("Got task state {} for task {} in {}", state, task.getTaskId(), JavaUtils.duration(start));
    switch(state) {
        case CHECK_IF_HEALTHCHECK_OVERDUE:
            if (isHealthcheckOverdue(task)) {
                LOG.info("Killing {} because it did not become healthy after {}", task.getTaskId(), JavaUtils.durationFromMillis(getKillAfterHealthcheckRunningForMillis()));
                taskManager.createTaskCleanup(new SingularityTaskCleanup(Optional.absent(), TaskCleanupType.OVERDUE_NEW_TASK, System.currentTimeMillis(), task.getTaskId(), Optional.of(String.format("Task did not become healthy after %s", JavaUtils.durationFromMillis(getKillAfterHealthcheckRunningForMillis()))), Optional.absent(), Optional.absent()));
                return false;
            } else {
                return true;
            }
        case CHECK_IF_TASK_OVERDUE:
            if (isTaskOverdue(task)) {
                LOG.info("Killing {} because it did not reach the task running state after {}", task.getTaskId(), JavaUtils.durationFromMillis(getKillAfterTaskNotRunningMillis()));
                taskManager.createTaskCleanup(new SingularityTaskCleanup(Optional.absent(), TaskCleanupType.OVERDUE_NEW_TASK, System.currentTimeMillis(), task.getTaskId(), Optional.of(String.format("Task did not reach the task running state after %s", JavaUtils.durationFromMillis(getKillAfterTaskNotRunningMillis()))), Optional.absent(), Optional.absent()));
                return false;
            } else {
                return true;
            }
        case LB_IN_PROGRESS_CHECK_AGAIN:
            return true;
        case UNHEALTHY_KILL_TASK:
            LOG.info("Killing {} because it failed healthchecks", task.getTaskId());
            taskManager.createTaskCleanup(new SingularityTaskCleanup(Optional.absent(), TaskCleanupType.UNHEALTHY_NEW_TASK, System.currentTimeMillis(), task.getTaskId(), Optional.of("Task is not healthy"), Optional.absent(), Optional.absent()));
            return false;
        case HEALTHY:
        case OBSOLETE:
            return false;
    }
    return false;
}