Examples with SingularityLoadBalancerUpdate com.hubspot.singularity.SingularityLoadBalancerUpdate used on opensource projects

Example 6 with SingularityLoadBalancerUpdate

use of com.hubspot.singularity.SingularityLoadBalancerUpdate in project Singularity by HubSpot.

the class SingularityDeployChecker method checkDeployProgress.

private SingularityDeployResult checkDeployProgress(final SingularityRequest request, final Optional<SingularityDeployMarker> cancelRequest, final SingularityPendingDeploy pendingDeploy, final Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest, final Optional<SingularityDeploy> deploy, final Collection<SingularityTaskId> deployActiveTasks, final Collection<SingularityTaskId> otherActiveTasks) {
    SingularityDeployProgress deployProgress = pendingDeploy.getDeployProgress().get();
    if (cancelRequest.isPresent()) {
        LOG.info("Canceling a deploy {} due to cancel request {}", pendingDeploy, cancelRequest.get());
        String userMessage = cancelRequest.get().getUser().isPresent() ? String.format(" by %s", cancelRequest.get().getUser().get()) : "";
        return new SingularityDeployResult(DeployState.CANCELED, Optional.of(String.format("Canceled due to request%s at %s", userMessage, cancelRequest.get().getTimestamp())), pendingDeploy.getLastLoadBalancerUpdate(), Collections.<SingularityDeployFailure>emptyList(), System.currentTimeMillis());
    }
    if (deployProgress.isStepComplete()) {
        return checkCanMoveToNextDeployStep(request, deploy, pendingDeploy, updatePendingDeployRequest);
    }
    final boolean isDeployOverdue = isDeployOverdue(pendingDeploy, deploy);
    if (deployActiveTasks.size() < deployProgress.getTargetActiveInstances()) {
        maybeUpdatePendingRequest(pendingDeploy, deploy, request, updatePendingDeployRequest);
        return checkOverdue(request, deploy, pendingDeploy, deployActiveTasks, isDeployOverdue);
    }
    if (shouldCheckLbState(pendingDeploy)) {
        final SingularityLoadBalancerUpdate lbUpdate = lbClient.getState(getLoadBalancerRequestId(pendingDeploy));
        return processLbState(request, deploy, pendingDeploy, updatePendingDeployRequest, deployActiveTasks, otherActiveTasks, tasksToShutDown(deployProgress, otherActiveTasks, request), lbUpdate);
    }
    if (isDeployOverdue && request.isLoadBalanced() && shouldCancelLoadBalancer(pendingDeploy)) {
        return cancelLoadBalancer(pendingDeploy, getDeployFailures(request, deploy, pendingDeploy, DeployState.OVERDUE, deployActiveTasks));
    }
    if (isWaitingForCurrentLbRequest(pendingDeploy)) {
        return new SingularityDeployResult(DeployState.WAITING, Optional.of("Waiting on load balancer API"), pendingDeploy.getLastLoadBalancerUpdate());
    }
    final DeployHealth deployHealth = deployHealthHelper.getDeployHealth(request, deploy, deployActiveTasks, true);
    switch(deployHealth) {
        case WAITING:
            maybeUpdatePendingRequest(pendingDeploy, deploy, request, updatePendingDeployRequest);
            return checkOverdue(request, deploy, pendingDeploy, deployActiveTasks, isDeployOverdue);
        case HEALTHY:
            if (!request.isLoadBalanced()) {
                return markStepFinished(pendingDeploy, deploy, deployActiveTasks, otherActiveTasks, request, updatePendingDeployRequest);
            }
            if (updatePendingDeployRequest.isPresent() && updatePendingDeployRequest.get().getTargetActiveInstances() != deployProgress.getTargetActiveInstances()) {
                maybeUpdatePendingRequest(pendingDeploy, deploy, request, updatePendingDeployRequest);
                return new SingularityDeployResult(DeployState.WAITING);
            }
            if (configuration.getLoadBalancerUri() == null) {
                LOG.warn("Deploy {} required a load balancer URI but it wasn't set", pendingDeploy);
                return new SingularityDeployResult(DeployState.FAILED, Optional.of("No valid load balancer URI was present"), Optional.<SingularityLoadBalancerUpdate>absent(), Collections.<SingularityDeployFailure>emptyList(), System.currentTimeMillis());
            }
            for (SingularityTaskId activeTaskId : deployActiveTasks) {
                taskManager.markHealthchecksFinished(activeTaskId);
                taskManager.clearStartupHealthchecks(activeTaskId);
            }
            return enqueueAndProcessLbRequest(request, deploy, pendingDeploy, updatePendingDeployRequest, deployActiveTasks, otherActiveTasks);
        case UNHEALTHY:
        default:
            for (SingularityTaskId activeTaskId : deployActiveTasks) {
                taskManager.markHealthchecksFinished(activeTaskId);
                taskManager.clearStartupHealthchecks(activeTaskId);
            }
            return getDeployResultWithFailures(request, deploy, pendingDeploy, DeployState.FAILED, "Not all tasks for deploy were healthy", deployActiveTasks);
    }
}

Example 7 with SingularityLoadBalancerUpdate

use of com.hubspot.singularity.SingularityLoadBalancerUpdate in project Singularity by HubSpot.

the class SingularityNewTaskChecker method getTaskState.

@VisibleForTesting
CheckTaskState getTaskState(SingularityTask task, Optional<SingularityRequestWithState> requestWithState, SingularityHealthchecker healthchecker) {
    if (!taskManager.isActiveTask(task.getTaskId().getId())) {
        return CheckTaskState.OBSOLETE;
    }
    SimplifiedTaskState taskState = SingularityTaskHistoryUpdate.getCurrentState(taskManager.getTaskHistoryUpdates(task.getTaskId()));
    switch(taskState) {
        case DONE:
            return CheckTaskState.OBSOLETE;
        case WAITING:
        case UNKNOWN:
            return CheckTaskState.CHECK_IF_TASK_OVERDUE;
        case RUNNING:
            break;
    }
    if (hasHealthcheck(task, requestWithState)) {
        Optional<SingularityTaskHealthcheckResult> maybeHealthCheck = taskManager.getLastHealthcheck(task.getTaskId());
        DeployHealth health = deployHealthHelper.getTaskHealth(task.getTaskRequest().getDeploy(), false, maybeHealthCheck, task.getTaskId());
        switch(health) {
            case WAITING:
                healthchecker.checkHealthcheck(task);
                return CheckTaskState.CHECK_IF_HEALTHCHECK_OVERDUE;
            case UNHEALTHY:
                taskManager.clearStartupHealthchecks(task.getTaskId());
                return CheckTaskState.UNHEALTHY_KILL_TASK;
            case HEALTHY:
                taskManager.clearStartupHealthchecks(task.getTaskId());
                break;
        }
    }
    // task is running + has succeeded healthcheck if available.
    if (!task.getTaskRequest().getRequest().isLoadBalanced()) {
        return CheckTaskState.HEALTHY;
    }
    Optional<SingularityLoadBalancerUpdate> lbUpdate = taskManager.getLoadBalancerState(task.getTaskId(), LoadBalancerRequestType.ADD);
    SingularityLoadBalancerUpdate newLbUpdate;
    final LoadBalancerRequestId loadBalancerRequestId = new LoadBalancerRequestId(task.getTaskId().getId(), LoadBalancerRequestType.ADD, Optional.absent());
    boolean taskCleaning = taskManager.getCleanupTaskIds().contains(task.getTaskId());
    if ((!lbUpdate.isPresent() || unknownNotRemoving(lbUpdate.get())) && !taskCleaning) {
        taskManager.saveLoadBalancerState(task.getTaskId(), LoadBalancerRequestType.ADD, new SingularityLoadBalancerUpdate(BaragonRequestState.UNKNOWN, loadBalancerRequestId, Optional.absent(), System.currentTimeMillis(), LoadBalancerMethod.PRE_ENQUEUE, Optional.absent()));
        newLbUpdate = lbClient.enqueue(loadBalancerRequestId, task.getTaskRequest().getRequest(), task.getTaskRequest().getDeploy(), Collections.singletonList(task), Collections.emptyList());
    } else {
        Optional<CheckTaskState> maybeCheckTaskState = checkLbState(lbUpdate.get().getLoadBalancerState());
        if (maybeCheckTaskState.isPresent()) {
            return maybeCheckTaskState.get();
        }
        newLbUpdate = lbClient.getState(loadBalancerRequestId);
    }
    taskManager.saveLoadBalancerState(task.getTaskId(), LoadBalancerRequestType.ADD, newLbUpdate);
    Optional<CheckTaskState> maybeCheckTaskState = checkLbState(newLbUpdate.getLoadBalancerState());
    if (maybeCheckTaskState.isPresent()) {
        return maybeCheckTaskState.get();
    }
    return CheckTaskState.LB_IN_PROGRESS_CHECK_AGAIN;
}

Example 8 with SingularityLoadBalancerUpdate

use of com.hubspot.singularity.SingularityLoadBalancerUpdate in project Singularity by HubSpot.

the class SingularityCleaner method checkRequestLbState.

private CheckLBState checkRequestLbState(SingularityRequestLbCleanup cleanup, List<SingularityTaskId> lbCleanupTasks) {
    if (!canRunRequestLbCleanup(cleanup, lbCleanupTasks)) {
        return CheckLBState.RETRY;
    }
    Optional<SingularityLoadBalancerUpdate> maybeDeleteUpdate = cleanup.getLoadBalancerUpdate();
    final LoadBalancerRequestId loadBalancerRequestId = getLoadBalancerRequestId(cleanup.getRequestId(), maybeDeleteUpdate);
    SingularityLoadBalancerUpdate lbDeleteUpdate;
    if (shouldEnqueueLbRequest(maybeDeleteUpdate)) {
        lbDeleteUpdate = lbClient.delete(loadBalancerRequestId, cleanup.getRequestId(), cleanup.getLoadBalancerGroups(), cleanup.getServiceBasePath());
        cleanup.setLoadBalancerUpdate(Optional.of(lbDeleteUpdate));
        requestManager.saveLbCleanupRequest(cleanup);
    } else if (maybeDeleteUpdate.get().getLoadBalancerState() == BaragonRequestState.WAITING || maybeDeleteUpdate.get().getLoadBalancerState() == BaragonRequestState.CANCELING) {
        lbDeleteUpdate = lbClient.getState(loadBalancerRequestId);
        cleanup.setLoadBalancerUpdate(Optional.of(lbDeleteUpdate));
        requestManager.saveLbCleanupRequest(cleanup);
    } else {
        lbDeleteUpdate = maybeDeleteUpdate.get();
    }
    switch(lbDeleteUpdate.getLoadBalancerState()) {
        case SUCCESS:
            return CheckLBState.DONE;
        case FAILED:
        case CANCELED:
            LOG.error("LB delete request {} ({}) got unexpected response {}", lbDeleteUpdate, loadBalancerRequestId, lbDeleteUpdate.getLoadBalancerState());
            exceptionNotifier.notify(String.format("LB delete failed for %s", lbDeleteUpdate.getLoadBalancerRequestId().toString()), ImmutableMap.of("state", lbDeleteUpdate.getLoadBalancerState().name(), "loadBalancerRequestId", loadBalancerRequestId.toString(), "addUpdate", lbDeleteUpdate.toString()));
            return CheckLBState.RETRY;
        case UNKNOWN:
        case CANCELING:
        case WAITING:
            LOG.trace("Waiting on LB delete request {} in state {}", loadBalancerRequestId, lbDeleteUpdate.getLoadBalancerState());
            break;
        case INVALID_REQUEST_NOOP:
            exceptionNotifier.notify(String.format("LB delete failed for %s", lbDeleteUpdate.getLoadBalancerRequestId().toString()), ImmutableMap.of("state", lbDeleteUpdate.getLoadBalancerState().name(), "loadBalancerRequestId", loadBalancerRequestId.toString(), "addUpdate", lbDeleteUpdate.toString()));
            return CheckLBState.LOAD_BALANCE_FAILED;
    }
    return CheckLBState.WAITING;
}

Example 9 with SingularityLoadBalancerUpdate

use of com.hubspot.singularity.SingularityLoadBalancerUpdate in project Singularity by HubSpot.

the class SingularityCleaner method checkLbState.

private CheckLBState checkLbState(SingularityTaskId taskId) {
    Optional<SingularityLoadBalancerUpdate> lbAddUpdate = taskManager.getLoadBalancerState(taskId, LoadBalancerRequestType.ADD);
    if (!lbAddUpdate.isPresent()) {
        return CheckLBState.NOT_LOAD_BALANCED;
    }
    if (!shouldRemoveLbState(taskId, lbAddUpdate.get())) {
        return CheckLBState.LOAD_BALANCE_FAILED;
    }
    Optional<SingularityLoadBalancerUpdate> maybeLbRemoveUpdate = taskManager.getLoadBalancerState(taskId, LoadBalancerRequestType.REMOVE);
    SingularityLoadBalancerUpdate lbRemoveUpdate = null;
    final LoadBalancerRequestId loadBalancerRequestId = getLoadBalancerRequestId(taskId, maybeLbRemoveUpdate);
    if (shouldEnqueueLbRequest(maybeLbRemoveUpdate)) {
        final Optional<SingularityTask> task = taskManager.getTask(taskId);
        if (!task.isPresent()) {
            LOG.error("Missing task {}", taskId);
            return CheckLBState.MISSING_TASK;
        }
        lbRemoveUpdate = lbClient.enqueue(loadBalancerRequestId, task.get().getTaskRequest().getRequest(), task.get().getTaskRequest().getDeploy(), Collections.emptyList(), Collections.singletonList(task.get()));
        taskManager.saveLoadBalancerState(taskId, LoadBalancerRequestType.REMOVE, lbRemoveUpdate);
    } else if (maybeLbRemoveUpdate.get().getLoadBalancerState() == BaragonRequestState.WAITING || maybeLbRemoveUpdate.get().getLoadBalancerState() == BaragonRequestState.CANCELING) {
        lbRemoveUpdate = lbClient.getState(loadBalancerRequestId);
        taskManager.saveLoadBalancerState(taskId, LoadBalancerRequestType.REMOVE, lbRemoveUpdate);
    } else {
        lbRemoveUpdate = maybeLbRemoveUpdate.get();
    }
    switch(lbRemoveUpdate.getLoadBalancerState()) {
        case SUCCESS:
            if (configuration.getLoadBalancerRemovalGracePeriodMillis() > 0) {
                final long duration = System.currentTimeMillis() - lbRemoveUpdate.getTimestamp();
                if (duration < configuration.getLoadBalancerRemovalGracePeriodMillis()) {
                    LOG.trace("LB removal for {} succeeded - waiting at least {} to kill task (current duration {})", taskId, JavaUtils.durationFromMillis(configuration.getLoadBalancerRemovalGracePeriodMillis()), JavaUtils.durationFromMillis(duration));
                    return CheckLBState.WAITING;
                }
            }
            return CheckLBState.DONE;
        case FAILED:
        case CANCELED:
            LOG.error("LB removal request {} ({}) got unexpected response {}", lbRemoveUpdate, loadBalancerRequestId, lbRemoveUpdate.getLoadBalancerState());
            exceptionNotifier.notify("LB removal failed", ImmutableMap.of("state", lbRemoveUpdate.getLoadBalancerState().name(), "loadBalancerRequestId", loadBalancerRequestId.toString(), "addUpdate", lbRemoveUpdate.toString()));
            return CheckLBState.RETRY;
        case UNKNOWN:
        case CANCELING:
        case WAITING:
            LOG.trace("Waiting on LB cleanup request {} in state {}", loadBalancerRequestId, lbRemoveUpdate.getLoadBalancerState());
            break;
        case INVALID_REQUEST_NOOP:
            exceptionNotifier.notify("LB removal failed", ImmutableMap.of("state", lbRemoveUpdate.getLoadBalancerState().name(), "loadBalancerRequestId", loadBalancerRequestId.toString(), "addUpdate", lbRemoveUpdate.toString()));
            return CheckLBState.LOAD_BALANCE_FAILED;
    }
    return CheckLBState.WAITING;
}

Example 10 with SingularityLoadBalancerUpdate

use of com.hubspot.singularity.SingularityLoadBalancerUpdate in project Singularity by HubSpot.

the class SingularityDeployChecker method enqueueAndProcessLbRequest.

private SingularityDeployResult enqueueAndProcessLbRequest(SingularityRequest request, Optional<SingularityDeploy> deploy, SingularityPendingDeploy pendingDeploy, Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest, Collection<SingularityTaskId> deployActiveTasks, Collection<SingularityTaskId> otherActiveTasks) {
    Collection<SingularityTaskId> toShutDown = tasksToShutDown(pendingDeploy.getDeployProgress().get(), otherActiveTasks, request);
    final Map<SingularityTaskId, SingularityTask> tasks = taskManager.getTasks(Iterables.concat(deployActiveTasks, toShutDown));
    final LoadBalancerRequestId lbRequestId = getLoadBalancerRequestId(pendingDeploy);
    List<SingularityTaskId> toRemoveFromLb = new ArrayList<>();
    for (SingularityTaskId taskId : toShutDown) {
        Optional<SingularityLoadBalancerUpdate> maybeAddUpdate = taskManager.getLoadBalancerState(taskId, LoadBalancerRequestType.ADD);
        if (maybeAddUpdate.isPresent() && maybeAddUpdate.get().getLoadBalancerState() == BaragonRequestState.SUCCESS) {
            toRemoveFromLb.add(taskId);
        }
    }
    updateLoadBalancerStateForTasks(deployActiveTasks, LoadBalancerRequestType.ADD, SingularityLoadBalancerUpdate.preEnqueue(lbRequestId));
    updateLoadBalancerStateForTasks(toRemoveFromLb, LoadBalancerRequestType.REMOVE, SingularityLoadBalancerUpdate.preEnqueue(lbRequestId));
    SingularityLoadBalancerUpdate enqueueResult = lbClient.enqueue(lbRequestId, request, deploy.get(), getTasks(deployActiveTasks, tasks), getTasks(toShutDown, tasks));
    return processLbState(request, deploy, pendingDeploy, updatePendingDeployRequest, deployActiveTasks, otherActiveTasks, toShutDown, enqueueResult);
}