Examples with SingularityDeployProgress com.hubspot.singularity.SingularityDeployProgress used on opensource projects

Example 1 with SingularityDeployProgress

use of com.hubspot.singularity.SingularityDeployProgress in project Singularity by HubSpot.

the class SingularitySchedulerTestBase method startDeploy.

protected void startDeploy(SingularityDeployMarker deployMarker, long timestamp) {
    SingularityDeployProgress startingDeployProgress = new SingularityDeployProgress(1, 0, 1, 10, false, true, Collections.<SingularityTaskId>emptySet(), timestamp);
    deployManager.savePendingDeploy(new SingularityPendingDeploy(deployMarker, Optional.<SingularityLoadBalancerUpdate>absent(), DeployState.WAITING, Optional.of(startingDeployProgress), Optional.<SingularityRequest>absent()));
}

Example 2 with SingularityDeployProgress

use of com.hubspot.singularity.SingularityDeployProgress in project Singularity by HubSpot.

the class SingularityDeploysTest method testLbUpdatesAfterEachDeployStep.

@Test
public void testLbUpdatesAfterEachDeployStep() {
    initLoadBalancedRequest();
    SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest();
    requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build(), singularityUser);
    initFirstDeploy();
    SingularityTask firstTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING);
    SingularityTask secondTask = launchTask(request, firstDeploy, 2, TaskState.TASK_RUNNING);
    deploy(secondDeployId, Optional.<Boolean>absent(), Optional.of(1), Optional.<Boolean>absent(), true);
    deployChecker.checkDeploys();
    scheduler.drainPendingQueue();
    Assert.assertEquals(1, taskManager.getPendingTaskIds().size());
    resourceOffers();
    Assert.assertEquals(1, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size());
    SingularityTaskId firstNewTaskId = taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).get(0);
    statusUpdate(taskManager.getTask(firstNewTaskId).get(), TaskState.TASK_RUNNING);
    deployChecker.checkDeploys();
    SingularityPendingDeploy pendingDeploy = deployManager.getPendingDeploy(requestId).get();
    Assert.assertEquals(DeployState.WAITING, pendingDeploy.getCurrentDeployState());
    testingLbClient.setNextBaragonRequestState(BaragonRequestState.WAITING);
    deployChecker.checkDeploys();
    pendingDeploy = deployManager.getPendingDeploy(requestId).get();
    Assert.assertEquals(DeployState.WAITING, pendingDeploy.getCurrentDeployState());
    testingLbClient.setNextBaragonRequestState(BaragonRequestState.SUCCESS);
    deployChecker.checkDeploys();
    Assert.assertTrue(taskManager.getCleanupTaskIds().contains(firstTask.getTaskId()));
    pendingDeploy = deployManager.getPendingDeploy(requestId).get();
    SingularityDeployProgress deployProgressStepOne = pendingDeploy.getDeployProgress().get();
    Assert.assertTrue(deployProgressStepOne.isStepComplete());
    Assert.assertEquals(1, deployProgressStepOne.getTargetActiveInstances());
    cleaner.drainCleanupQueue();
    statusUpdate(firstTask, TaskState.TASK_KILLED);
    deployChecker.checkDeploys();
    pendingDeploy = deployManager.getPendingDeploy(requestId).get();
    Assert.assertFalse(pendingDeploy.getDeployProgress().get().isStepComplete());
    Assert.assertEquals(2, pendingDeploy.getDeployProgress().get().getTargetActiveInstances());
    scheduler.drainPendingQueue();
    Assert.assertEquals(1, taskManager.getPendingTaskIds().size());
    resourceOffers();
    Assert.assertEquals(2, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size());
    SingularityTaskId secondNewTaskId = null;
    for (SingularityTaskId taskId : taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId)) {
        if (taskId.getInstanceNo() == 2) {
            secondNewTaskId = taskId;
        }
    }
    statusUpdate(taskManager.getTask(secondNewTaskId).get(), TaskState.TASK_RUNNING);
    testingLbClient.setNextBaragonRequestState(BaragonRequestState.WAITING);
    deployChecker.checkDeploys();
    pendingDeploy = deployManager.getPendingDeploy(requestId).get();
    Assert.assertEquals(DeployState.WAITING, pendingDeploy.getCurrentDeployState());
    testingLbClient.setNextBaragonRequestState(BaragonRequestState.SUCCESS);
    deployChecker.checkDeploys();
    Assert.assertEquals(2, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size());
    Assert.assertEquals(DeployState.SUCCEEDED, deployManager.getDeployResult(requestId, secondDeployId).get().getDeployState());
}

Example 3 with SingularityDeployProgress

use of com.hubspot.singularity.SingularityDeployProgress in project Singularity by HubSpot.

the class SingularityDeployChecker method getDeployResult.

private SingularityDeployResult getDeployResult(final SingularityRequest request, final RequestState requestState, final Optional<SingularityDeployMarker> cancelRequest, final SingularityPendingDeploy pendingDeploy, final Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest, final Optional<SingularityDeploy> deploy, final Collection<SingularityTaskId> deployActiveTasks, final Collection<SingularityTaskId> otherActiveTasks, final Collection<SingularityTaskId> inactiveDeployMatchingTasks) {
    if (!request.isDeployable() || (configuration.isAllowDeployOfPausedRequests() && requestState == RequestState.PAUSED)) {
        LOG.info("Succeeding a deploy {} because the request {} was not deployable", pendingDeploy, request);
        return new SingularityDeployResult(DeployState.SUCCEEDED, "Request not deployable");
    }
    if (!pendingDeploy.getDeployProgress().isPresent()) {
        return new SingularityDeployResult(DeployState.FAILED, "No deploy progress data present in Zookeeper. Please reattempt your deploy");
    }
    Set<SingularityTaskId> newInactiveDeployTasks = getNewInactiveDeployTasks(pendingDeploy, inactiveDeployMatchingTasks);
    if (!newInactiveDeployTasks.isEmpty()) {
        if (canRetryTasks(deploy, inactiveDeployMatchingTasks)) {
            SingularityDeployProgress newProgress = pendingDeploy.getDeployProgress().get().withFailedTasks(new HashSet<>(inactiveDeployMatchingTasks));
            updatePendingDeploy(pendingDeploy, pendingDeploy.getLastLoadBalancerUpdate(), DeployState.WAITING, Optional.of(newProgress));
            requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), System.currentTimeMillis(), pendingDeploy.getDeployMarker().getUser(), PendingType.NEXT_DEPLOY_STEP, deploy.isPresent() ? deploy.get().getSkipHealthchecksOnDeploy() : Optional.<Boolean>absent(), pendingDeploy.getDeployMarker().getMessage()));
            return new SingularityDeployResult(DeployState.WAITING);
        }
        if (request.isLoadBalanced() && shouldCancelLoadBalancer(pendingDeploy)) {
            LOG.info("Attempting to cancel pending load balancer request, failing deploy {} regardless", pendingDeploy);
            sendCancelToLoadBalancer(pendingDeploy);
        }
        int maxRetries = deploy.get().getMaxTaskRetries().or(configuration.getDefaultDeployMaxTaskRetries());
        return getDeployResultWithFailures(request, deploy, pendingDeploy, DeployState.FAILED, String.format("%s task(s) for this deploy failed", inactiveDeployMatchingTasks.size() - maxRetries), inactiveDeployMatchingTasks);
    }
    return checkDeployProgress(request, cancelRequest, pendingDeploy, updatePendingDeployRequest, deploy, deployActiveTasks, otherActiveTasks);
}

Example 4 with SingularityDeployProgress

use of com.hubspot.singularity.SingularityDeployProgress in project Singularity by HubSpot.

the class SingularityDeployChecker method checkDeployProgress.

private SingularityDeployResult checkDeployProgress(final SingularityRequest request, final Optional<SingularityDeployMarker> cancelRequest, final SingularityPendingDeploy pendingDeploy, final Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest, final Optional<SingularityDeploy> deploy, final Collection<SingularityTaskId> deployActiveTasks, final Collection<SingularityTaskId> otherActiveTasks) {
    SingularityDeployProgress deployProgress = pendingDeploy.getDeployProgress().get();
    if (cancelRequest.isPresent()) {
        LOG.info("Canceling a deploy {} due to cancel request {}", pendingDeploy, cancelRequest.get());
        String userMessage = cancelRequest.get().getUser().isPresent() ? String.format(" by %s", cancelRequest.get().getUser().get()) : "";
        return new SingularityDeployResult(DeployState.CANCELED, Optional.of(String.format("Canceled due to request%s at %s", userMessage, cancelRequest.get().getTimestamp())), pendingDeploy.getLastLoadBalancerUpdate(), Collections.<SingularityDeployFailure>emptyList(), System.currentTimeMillis());
    }
    if (deployProgress.isStepComplete()) {
        return checkCanMoveToNextDeployStep(request, deploy, pendingDeploy, updatePendingDeployRequest);
    }
    final boolean isDeployOverdue = isDeployOverdue(pendingDeploy, deploy);
    if (deployActiveTasks.size() < deployProgress.getTargetActiveInstances()) {
        maybeUpdatePendingRequest(pendingDeploy, deploy, request, updatePendingDeployRequest);
        return checkOverdue(request, deploy, pendingDeploy, deployActiveTasks, isDeployOverdue);
    }
    if (shouldCheckLbState(pendingDeploy)) {
        final SingularityLoadBalancerUpdate lbUpdate = lbClient.getState(getLoadBalancerRequestId(pendingDeploy));
        return processLbState(request, deploy, pendingDeploy, updatePendingDeployRequest, deployActiveTasks, otherActiveTasks, tasksToShutDown(deployProgress, otherActiveTasks, request), lbUpdate);
    }
    if (isDeployOverdue && request.isLoadBalanced() && shouldCancelLoadBalancer(pendingDeploy)) {
        return cancelLoadBalancer(pendingDeploy, getDeployFailures(request, deploy, pendingDeploy, DeployState.OVERDUE, deployActiveTasks));
    }
    if (isWaitingForCurrentLbRequest(pendingDeploy)) {
        return new SingularityDeployResult(DeployState.WAITING, Optional.of("Waiting on load balancer API"), pendingDeploy.getLastLoadBalancerUpdate());
    }
    final DeployHealth deployHealth = deployHealthHelper.getDeployHealth(request, deploy, deployActiveTasks, true);
    switch(deployHealth) {
        case WAITING:
            maybeUpdatePendingRequest(pendingDeploy, deploy, request, updatePendingDeployRequest);
            return checkOverdue(request, deploy, pendingDeploy, deployActiveTasks, isDeployOverdue);
        case HEALTHY:
            if (!request.isLoadBalanced()) {
                return markStepFinished(pendingDeploy, deploy, deployActiveTasks, otherActiveTasks, request, updatePendingDeployRequest);
            }
            if (updatePendingDeployRequest.isPresent() && updatePendingDeployRequest.get().getTargetActiveInstances() != deployProgress.getTargetActiveInstances()) {
                maybeUpdatePendingRequest(pendingDeploy, deploy, request, updatePendingDeployRequest);
                return new SingularityDeployResult(DeployState.WAITING);
            }
            if (configuration.getLoadBalancerUri() == null) {
                LOG.warn("Deploy {} required a load balancer URI but it wasn't set", pendingDeploy);
                return new SingularityDeployResult(DeployState.FAILED, Optional.of("No valid load balancer URI was present"), Optional.<SingularityLoadBalancerUpdate>absent(), Collections.<SingularityDeployFailure>emptyList(), System.currentTimeMillis());
            }
            for (SingularityTaskId activeTaskId : deployActiveTasks) {
                taskManager.markHealthchecksFinished(activeTaskId);
                taskManager.clearStartupHealthchecks(activeTaskId);
            }
            return enqueueAndProcessLbRequest(request, deploy, pendingDeploy, updatePendingDeployRequest, deployActiveTasks, otherActiveTasks);
        case UNHEALTHY:
        default:
            for (SingularityTaskId activeTaskId : deployActiveTasks) {
                taskManager.markHealthchecksFinished(activeTaskId);
                taskManager.clearStartupHealthchecks(activeTaskId);
            }
            return getDeployResultWithFailures(request, deploy, pendingDeploy, DeployState.FAILED, "Not all tasks for deploy were healthy", deployActiveTasks);
    }
}

Example 5 with SingularityDeployProgress

use of com.hubspot.singularity.SingularityDeployProgress in project Singularity by HubSpot.

the class SingularityDeployChecker method markStepFinished.

private SingularityDeployResult markStepFinished(SingularityPendingDeploy pendingDeploy, Optional<SingularityDeploy> deploy, Collection<SingularityTaskId> deployActiveTasks, Collection<SingularityTaskId> otherActiveTasks, SingularityRequest request, Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest) {
    SingularityDeployProgress deployProgress = pendingDeploy.getDeployProgress().get();
    if (updatePendingDeployRequest.isPresent() && getNewTargetInstances(deployProgress, request, updatePendingDeployRequest) != deployProgress.getTargetActiveInstances()) {
        maybeUpdatePendingRequest(pendingDeploy, deploy, request, updatePendingDeployRequest);
        return new SingularityDeployResult(DeployState.WAITING);
    }
    SingularityDeployProgress newProgress = deployProgress.withNewActiveInstances(deployActiveTasks.size()).withCompletedStep();
    DeployState deployState = isLastStepFinished(newProgress, request) ? DeployState.SUCCEEDED : DeployState.WAITING;
    String message = deployState == DeployState.SUCCEEDED ? "New deploy succeeded" : "New deploy is progressing, this task is being replaced";
    updatePendingDeploy(pendingDeploy, pendingDeploy.getLastLoadBalancerUpdate(), deployState, Optional.of(newProgress));
    for (SingularityTaskId taskId : tasksToShutDown(deployProgress, otherActiveTasks, request)) {
        taskManager.createTaskCleanup(new SingularityTaskCleanup(Optional.<String>absent(), TaskCleanupType.DEPLOY_STEP_FINISHED, System.currentTimeMillis(), taskId, Optional.of(message), Optional.<String>absent(), Optional.<SingularityTaskShellCommandRequestId>absent()));
    }
    return new SingularityDeployResult(deployState);
}