Examples with SingularityTaskHealthcheckResult com.hubspot.singularity.SingularityTaskHealthcheckResult used on opensource projects

Example 11 with SingularityTaskHealthcheckResult

use of com.hubspot.singularity.SingularityTaskHealthcheckResult in project Singularity by HubSpot.

the class SingularityDeployHealthHelper method getTaskFailure.

private Optional<SingularityDeployFailure> getTaskFailure(SingularityDeploy deploy, Map<SingularityTaskId, List<SingularityTaskHistoryUpdate>> taskUpdates, Map<SingularityTaskId, SingularityTaskHealthcheckResult> healthcheckResults, SingularityTaskId taskId) {
    SingularityTaskHealthcheckResult healthcheckResult = healthcheckResults.get(taskId);
    Optional<SingularityDeployFailure> maybeFailure;
    if (healthcheckResult == null) {
        maybeFailure = getNonHealthcheckedTaskFailure(taskUpdates, taskId);
    } else {
        maybeFailure = getHealthcheckedTaskFailure(deploy, taskUpdates, healthcheckResult, taskId);
    }
    return maybeFailure;
}

Example 12 with SingularityTaskHealthcheckResult

use of com.hubspot.singularity.SingularityTaskHealthcheckResult in project Singularity by HubSpot.

the class SingularityDeployHealthHelper method getTaskFailures.

public List<SingularityDeployFailure> getTaskFailures(final Optional<SingularityDeploy> deploy, final Collection<SingularityTaskId> activeTasks) {
    List<SingularityDeployFailure> failures = new ArrayList<>();
    Map<SingularityTaskId, List<SingularityTaskHistoryUpdate>> taskUpdates = taskManager.getTaskHistoryUpdates(activeTasks);
    Map<SingularityTaskId, SingularityTaskHealthcheckResult> healthcheckResults = taskManager.getLastHealthcheck(activeTasks);
    for (SingularityTaskId taskId : activeTasks) {
        Optional<SingularityDeployFailure> maybeFailure = getTaskFailure(deploy.get(), taskUpdates, healthcheckResults, taskId);
        if (maybeFailure.isPresent()) {
            failures.add(maybeFailure.get());
        }
    }
    return failures;
}

Example 13 with SingularityTaskHealthcheckResult

use of com.hubspot.singularity.SingularityTaskHealthcheckResult in project Singularity by HubSpot.

the class SingularityHealthcheckAsyncHandler method saveResult.

public void saveResult(Optional<Integer> statusCode, Optional<String> responseBody, Optional<String> errorMessage, Optional<Throwable> throwable) {
    boolean inStartup = throwable.isPresent() && throwable.get() instanceof ConnectException;
    try {
        SingularityTaskHealthcheckResult result = new SingularityTaskHealthcheckResult(statusCode, Optional.of(System.currentTimeMillis() - startTime), startTime, responseBody, errorMessage, task.getTaskId(), Optional.of(inStartup));
        LOG.trace("Saving healthcheck result {}", result);
        taskManager.saveHealthcheckResult(result);
        if (result.isFailed()) {
            if (!taskManager.isActiveTask(task.getTaskId().getId())) {
                LOG.trace("Task {} is not active, not re-enqueueing healthcheck", task.getTaskId());
                return;
            }
            if (statusCode.isPresent() && failureStatusCodes.contains(statusCode.get())) {
                LOG.debug("Failed status code present for task {} ({})", task.getTaskId(), statusCode.get());
                healthchecker.markHealthcheckFinished(task.getTaskId().getId());
                newTaskChecker.runNewTaskCheckImmediately(task, healthchecker);
                return;
            }
            healthchecker.enqueueHealthcheck(task, true, inStartup, false);
        } else {
            healthchecker.markHealthcheckFinished(task.getTaskId().getId());
            newTaskChecker.runNewTaskCheckImmediately(task, healthchecker);
        }
    } catch (Throwable t) {
        LOG.error("Caught throwable while saving health check result for {}, will re-enqueue", task.getTaskId(), t);
        exceptionNotifier.notify(String.format("Error saving healthcheck (%s)", t.getMessage()), t, ImmutableMap.of("taskId", task.getTaskId().toString()));
        healthchecker.reEnqueueOrAbort(task, inStartup);
    }
}

Example 14 with SingularityTaskHealthcheckResult

use of com.hubspot.singularity.SingularityTaskHealthcheckResult in project Singularity by HubSpot.

the class SingularityNewTaskChecker method getTaskState.

@VisibleForTesting
CheckTaskState getTaskState(SingularityTask task, Optional<SingularityRequestWithState> requestWithState, SingularityHealthchecker healthchecker) {
    if (!taskManager.isActiveTask(task.getTaskId().getId())) {
        return CheckTaskState.OBSOLETE;
    }
    SimplifiedTaskState taskState = SingularityTaskHistoryUpdate.getCurrentState(taskManager.getTaskHistoryUpdates(task.getTaskId()));
    switch(taskState) {
        case DONE:
            return CheckTaskState.OBSOLETE;
        case WAITING:
        case UNKNOWN:
            return CheckTaskState.CHECK_IF_TASK_OVERDUE;
        case RUNNING:
            break;
    }
    if (hasHealthcheck(task, requestWithState)) {
        Optional<SingularityTaskHealthcheckResult> maybeHealthCheck = taskManager.getLastHealthcheck(task.getTaskId());
        DeployHealth health = deployHealthHelper.getTaskHealth(task.getTaskRequest().getDeploy(), false, maybeHealthCheck, task.getTaskId());
        switch(health) {
            case WAITING:
                healthchecker.checkHealthcheck(task);
                return CheckTaskState.CHECK_IF_HEALTHCHECK_OVERDUE;
            case UNHEALTHY:
                taskManager.clearStartupHealthchecks(task.getTaskId());
                return CheckTaskState.UNHEALTHY_KILL_TASK;
            case HEALTHY:
                taskManager.clearStartupHealthchecks(task.getTaskId());
                break;
        }
    }
    // task is running + has succeeded healthcheck if available.
    if (!task.getTaskRequest().getRequest().isLoadBalanced()) {
        return CheckTaskState.HEALTHY;
    }
    Optional<SingularityLoadBalancerUpdate> lbUpdate = taskManager.getLoadBalancerState(task.getTaskId(), LoadBalancerRequestType.ADD);
    SingularityLoadBalancerUpdate newLbUpdate;
    final LoadBalancerRequestId loadBalancerRequestId = new LoadBalancerRequestId(task.getTaskId().getId(), LoadBalancerRequestType.ADD, Optional.absent());
    boolean taskCleaning = taskManager.getCleanupTaskIds().contains(task.getTaskId());
    if ((!lbUpdate.isPresent() || unknownNotRemoving(lbUpdate.get())) && !taskCleaning) {
        taskManager.saveLoadBalancerState(task.getTaskId(), LoadBalancerRequestType.ADD, new SingularityLoadBalancerUpdate(BaragonRequestState.UNKNOWN, loadBalancerRequestId, Optional.absent(), System.currentTimeMillis(), LoadBalancerMethod.PRE_ENQUEUE, Optional.absent()));
        newLbUpdate = lbClient.enqueue(loadBalancerRequestId, task.getTaskRequest().getRequest(), task.getTaskRequest().getDeploy(), Collections.singletonList(task), Collections.emptyList());
    } else {
        Optional<CheckTaskState> maybeCheckTaskState = checkLbState(lbUpdate.get().getLoadBalancerState());
        if (maybeCheckTaskState.isPresent()) {
            return maybeCheckTaskState.get();
        }
        newLbUpdate = lbClient.getState(loadBalancerRequestId);
    }
    taskManager.saveLoadBalancerState(task.getTaskId(), LoadBalancerRequestType.ADD, newLbUpdate);
    Optional<CheckTaskState> maybeCheckTaskState = checkLbState(newLbUpdate.getLoadBalancerState());
    if (maybeCheckTaskState.isPresent()) {
        return maybeCheckTaskState.get();
    }
    return CheckTaskState.LB_IN_PROGRESS_CHECK_AGAIN;
}

Example 15 with SingularityTaskHealthcheckResult

use of com.hubspot.singularity.SingularityTaskHealthcheckResult in project Singularity by HubSpot.

the class SingularityDeployHealthHelper method getHealthcheckDeployState.

private DeployHealth getHealthcheckDeployState(final SingularityDeploy deploy, final Collection<SingularityTaskId> matchingActiveTasks, final boolean isDeployPending) {
    Map<SingularityTaskId, SingularityTaskHealthcheckResult> healthcheckResults = taskManager.getLastHealthcheck(matchingActiveTasks);
    List<SingularityRequestHistory> requestHistories = requestManager.getRequestHistory(deploy.getRequestId());
    for (SingularityTaskId taskId : matchingActiveTasks) {
        DeployHealth individualTaskHealth;
        if (healthchecksSkipped(taskId, requestHistories, deploy)) {
            LOG.trace("Detected skipped healthchecks for {}", taskId);
            individualTaskHealth = DeployHealth.HEALTHY;
        } else {
            individualTaskHealth = getTaskHealth(deploy, isDeployPending, Optional.fromNullable(healthcheckResults.get(taskId)), taskId);
        }
        if (individualTaskHealth != DeployHealth.HEALTHY) {
            return individualTaskHealth;
        }
    }
    return DeployHealth.HEALTHY;
}