Examples with SingularityDeployStatistics com.hubspot.singularity.SingularityDeployStatistics used on opensource projects

Example 21 with SingularityDeployStatistics

use of com.hubspot.singularity.SingularityDeployStatistics in project Singularity by HubSpot.

the class SingularityCrashLoopTest method itDetectsASingleInstanceFailureLoop.

@Test
public void itDetectsASingleInstanceFailureLoop() {
    initRequestWithType(RequestType.WORKER, false);
    initFirstDeploy();
    long now = System.currentTimeMillis();
    createTaskFailure(1, now - TimeUnit.MINUTES.toMillis(1), TaskFailureType.BAD_EXIT_CODE);
    createTaskFailure(1, now - TimeUnit.MINUTES.toMillis(6), TaskFailureType.OOM);
    createTaskFailure(1, now - TimeUnit.MINUTES.toMillis(11), TaskFailureType.OUT_OF_DISK_SPACE);
    createTaskFailure(1, now - TimeUnit.MINUTES.toMillis(16), TaskFailureType.OUT_OF_DISK_SPACE);
    createTaskFailure(1, now - TimeUnit.MINUTES.toMillis(21), TaskFailureType.OOM);
    SingularityDeployStatistics deployStatistics = deployManager.getDeployStatistics(requestId, firstDeployId).get();
    List<CrashLoopInfo> active = crashLoops.getActiveCrashLoops(deployStatistics);
    Assertions.assertEquals(1, active.size());
    Assertions.assertEquals(CrashLoopType.SINGLE_INSTANCE_FAILURE_LOOP, Iterables.getOnlyElement(active).getType());
}

Example 22 with SingularityDeployStatistics

use of com.hubspot.singularity.SingularityDeployStatistics in project Singularity by HubSpot.

the class SingularitySchedulerTest method testRunNowOnDemandJobsDoNotRetryAfterUserRequestedKill.

@Test
public void testRunNowOnDemandJobsDoNotRetryAfterUserRequestedKill() {
    initRequestWithType(RequestType.ON_DEMAND, false);
    SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest();
    SingularityRequest newRequest = request.toBuilder().setNumRetriesOnFailure(Optional.of(2)).build();
    requestResource.postRequest(newRequest, singularityUser);
    initFirstDeploy();
    requestResource.scheduleImmediately(singularityUser, requestId, new SingularityRunNowRequestBuilder().setMessage("foo bar").build());
    scheduler.drainPendingQueue();
    resourceOffers();
    SingularityTask task = taskManager.getActiveTasks().get(0);
    taskManager.saveTaskCleanup(new SingularityTaskCleanup(Optional.of(singularityUser.getId()), TaskCleanupType.USER_REQUESTED, System.currentTimeMillis(), task.getTaskId(), Optional.empty(), Optional.empty(), Optional.empty()));
    cleaner.drainCleanupQueue();
    statusUpdate(task, TaskState.TASK_KILLED);
    scheduler.drainPendingQueue();
    SingularityDeployStatistics deployStatistics = deployManager.getDeployStatistics(task.getTaskId().getRequestId(), task.getTaskId().getDeployId()).get();
    Assertions.assertEquals(0, taskManager.getPendingTaskIds().size());
    Assertions.assertEquals(MesosTaskState.TASK_KILLED, deployStatistics.getLastTaskState().get().toTaskState().get());
    Assertions.assertEquals(0, deployStatistics.getNumFailures());
    Assertions.assertEquals(0, deployStatistics.getNumSequentialRetries());
}

Example 23 with SingularityDeployStatistics

use of com.hubspot.singularity.SingularityDeployStatistics in project Singularity by HubSpot.

the class DeployManager method getDeployHistory.

public Optional<SingularityDeployHistory> getDeployHistory(String requestId, String deployId, boolean loadEntireHistory) {
    Optional<SingularityDeployMarker> deployMarker = getData(getDeployMarkerPath(requestId, deployId), deployMarkerTranscoder);
    if (!deployMarker.isPresent()) {
        return Optional.empty();
    }
    Optional<SingularityDeployResult> deployState = getDeployResult(requestId, deployId);
    if (!loadEntireHistory) {
        return Optional.of(new SingularityDeployHistory(deployState, deployMarker.get(), Optional.<SingularityDeploy>empty(), Optional.<SingularityDeployStatistics>empty()));
    }
    Optional<SingularityDeploy> deploy = getDeploy(requestId, deployId);
    if (!deploy.isPresent()) {
        return Optional.empty();
    }
    Optional<SingularityDeployStatistics> deployStatistics = getDeployStatistics(requestId, deployId);
    return Optional.of(new SingularityDeployHistory(deployState, deployMarker.get(), deploy, deployStatistics));
}

Example 24 with SingularityDeployStatistics

use of com.hubspot.singularity.SingularityDeployStatistics in project Singularity by HubSpot.

the class SingularityScheduler method handleCompletedTask.

public void handleCompletedTask(Optional<SingularityTask> task, SingularityTaskId taskId, long timestamp, ExtendedTaskState state, SingularityCreateResult taskHistoryUpdateCreateResult, Protos.TaskStatus status) {
    final SingularityDeployStatistics deployStatistics = getDeployStatistics(taskId.getRequestId(), taskId.getDeployId());
    if (!task.isPresent() || task.get().getTaskRequest().getRequest().isLoadBalanced()) {
        taskManager.createLBCleanupTask(taskId);
    }
    if (requestManager.isBouncing(taskId.getRequestId())) {
        List<SingularityTaskId> activeTaskIds = taskManager.getActiveTaskIdsForRequest(taskId.getRequestId());
        boolean foundBouncingTask = false;
        for (SingularityTaskId activeTaskId : activeTaskIds) {
            Optional<SingularityTaskHistoryUpdate> maybeCleaningUpdate = taskManager.getTaskHistoryUpdate(activeTaskId, ExtendedTaskState.TASK_CLEANING);
            if (maybeCleaningUpdate.isPresent()) {
                if (maybeCleaningUpdate.get().getStatusReason().orElse("").contains("BOUNCE")) {
                    // TaskCleanupType enum is included in status message
                    LOG.debug("Found task {} still waiting for bounce to complete", activeTaskId);
                    foundBouncingTask = true;
                    break;
                } else if (!maybeCleaningUpdate.get().getPrevious().isEmpty()) {
                    for (SingularityTaskHistoryUpdate previousUpdate : maybeCleaningUpdate.get().getPrevious()) {
                        if (previousUpdate.getStatusMessage().orElse("").contains("BOUNCE")) {
                            LOG.debug("Found task {} still waiting for bounce to complete", activeTaskId);
                            foundBouncingTask = true;
                            break;
                        }
                    }
                }
            }
        }
        if (!foundBouncingTask) {
            LOG.info("Bounce completed for request {}, no cleaning tasks due to bounce found", taskId.getRequestId());
            Optional<SingularityExpiringBounce> expiringBounce = requestManager.getExpiringBounce(taskId.getRequestId());
            if (expiringBounce.isPresent() && expiringBounce.get().getDeployId().equals(taskId.getDeployId())) {
                requestManager.removeExpiringBounce(taskId.getRequestId());
            }
            requestManager.markBounceComplete(taskId.getRequestId());
        }
    }
    final Optional<PendingType> scheduleResult = handleCompletedTaskWithStatistics(task, taskId, timestamp, state, deployStatistics, taskHistoryUpdateCreateResult, status);
    if (taskHistoryUpdateCreateResult == SingularityCreateResult.EXISTED) {
        return;
    }
    updateDeployStatistics(deployStatistics, taskId, task, timestamp, state, scheduleResult, status);
}

Example 25 with SingularityDeployStatistics

use of com.hubspot.singularity.SingularityDeployStatistics in project Singularity by HubSpot.

the class SingularitySchedulerTest method testRunNowScheduledJobDoesNotRetry.

@Test
public void testRunNowScheduledJobDoesNotRetry() {
    initScheduledRequest();
    SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest();
    SingularityRequest newRequest = request.toBuilder().setNumRetriesOnFailure(Optional.of(2)).build();
    requestResource.postRequest(newRequest, singularityUser);
    initFirstDeploy();
    requestResource.scheduleImmediately(singularityUser, requestId, new SingularityRunNowRequestBuilder().build());
    scheduler.drainPendingQueue();
    resourceOffers();
    SingularityTask task = taskManager.getActiveTasks().get(0);
    statusUpdate(task, TaskState.TASK_FAILED);
    scheduler.drainPendingQueue();
    SingularityDeployStatistics deployStatistics = deployManager.getDeployStatistics(task.getTaskId().getRequestId(), task.getTaskId().getDeployId()).get();
    Assertions.assertEquals(MesosTaskState.TASK_FAILED, deployStatistics.getLastTaskState().get().toTaskState().get());
    Assertions.assertEquals(PendingType.TASK_DONE, taskManager.getPendingTaskIds().get(0).getPendingType());
    Assertions.assertEquals(1, deployStatistics.getNumFailures());
    Assertions.assertEquals(0, deployStatistics.getNumSequentialRetries());
    Assertions.assertEquals(Optional.<Long>empty(), deployStatistics.getAverageRuntimeMillis());
}