Examples with DeployProgressLbUpdateHolder com.hubspot.singularity.DeployProgressLbUpdateHolder used on opensource projects

Example 1 with DeployProgressLbUpdateHolder

use of com.hubspot.singularity.DeployProgressLbUpdateHolder in project Singularity by HubSpot.

the class SingularityDeployChecker method checkLbRevertToActiveTasks.

private SingularityDeployResult checkLbRevertToActiveTasks(SingularityRequest request, SingularityDeploy deploy, SingularityPendingDeploy pendingDeploy, SingularityDeployProgress updatedProgress, DeployState acceptanceHookDeployState, Collection<SingularityTaskId> deployActiveTasks, Collection<SingularityTaskId> otherActiveTasks) {
    final SingularityLoadBalancerUpdate lbUpdate = lbClient.getState(pendingDeploy.getDeployProgress().getPendingLbUpdate().get().getLoadBalancerRequestId());
    DeployProgressLbUpdateHolder lbUpdateHolder = updatedProgress.getLbUpdates().get(lbUpdate.getLoadBalancerRequestId().toString());
    if (lbUpdateHolder == null) {
        return new SingularityDeployResult(DeployState.FAILED_INTERNAL_STATE, "Load balancer update metadata not found");
    }
    updateLoadBalancerStateForTasks(lbUpdateHolder.getAdded(), LoadBalancerRequestType.ADD, lbUpdate);
    switch(lbUpdate.getLoadBalancerState()) {
        case SUCCESS:
            LOG.info("LB revert succeeded, continuing with original deploy status");
            updatePendingDeploy(pendingDeploy, acceptanceHookDeployState, updatedProgress.withFinishedLbUpdate(lbUpdate, lbUpdateHolder));
            return new SingularityDeployResult(acceptanceHookDeployState, String.join(", ", updatedProgress.getAcceptanceResultMessageHistory()));
        case WAITING:
            return new SingularityDeployResult(DeployState.WAITING);
        default:
            // Keep trying until we time out, since abandoning here would leave nothing in the LB
            if (deployCheckHelper.isDeployOverdue(pendingDeploy, deploy)) {
                LOG.error("Unable to revert load balancer for deploy failure");
                return new SingularityDeployResult(DeployState.FAILED_INTERNAL_STATE, "Unable to revert load balancer for deploy failure");
            } else {
                LOG.warn("Retrying failed LB revert for {} {}", pendingDeploy.getDeployMarker().getRequestId(), pendingDeploy.getDeployMarker().getDeployId());
                return enqueueLbRevertToActiveTasks(request, pendingDeploy, updatedProgress, deployActiveTasks, otherActiveTasks);
            }
    }
}

Example 2 with DeployProgressLbUpdateHolder

use of com.hubspot.singularity.DeployProgressLbUpdateHolder in project Singularity by HubSpot.

the class SingularityDeployAcceptanceTest method testLbRevertsAfterFailedAcceptanceStepOnNonCanary.

@Test
public void testLbRevertsAfterFailedAcceptanceStepOnNonCanary() {
    NoopDeployAcceptanceHook hook = (NoopDeployAcceptanceHook) acceptanceHooks.iterator().next();
    hook.setNextResult(new DeployAcceptanceResult(DeployAcceptanceState.FAILED, "ruh-roh"));
    initLoadBalancedRequest();
    initFirstDeploy();
    SingularityTask firstTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING);
    SingularityDeployBuilder builder = new SingularityDeployBuilder(requestId, secondDeployId);
    builder.setCommand(Optional.of("sleep 1")).setCanaryDeploySettings(CanaryDeploySettings.newbuilder().setAcceptanceMode(DeployAcceptanceMode.CHECKS).setEnableCanaryDeploy(false).build()).setServiceBasePath(Optional.of("/basepath")).setLoadBalancerGroups(Optional.of(Collections.singleton("group")));
    deployResource.deploy(new SingularityDeployRequest(builder.build(), Optional.of(false), Optional.empty()), singularityUser);
    deployChecker.checkDeploys();
    scheduler.drainPendingQueue();
    Assertions.assertEquals(1, taskManager.getPendingTaskIds().size());
    resourceOffers();
    Assertions.assertEquals(1, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size());
    SingularityTaskId firstNewTaskId = taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).get(0);
    statusUpdate(taskManager.getTask(firstNewTaskId).get(), TaskState.TASK_RUNNING);
    deployChecker.checkDeploys();
    SingularityPendingDeploy pendingDeploy = deployManager.getPendingDeploy(requestId).get();
    Assertions.assertEquals(DeployState.WAITING, pendingDeploy.getCurrentDeployState());
    testingLbClient.setNextRequestState(LoadBalancerRequestState.WAITING);
    deployChecker.checkDeploys();
    pendingDeploy = deployManager.getPendingDeploy(requestId).get();
    Assertions.assertEquals(DeployState.WAITING, pendingDeploy.getCurrentDeployState());
    testingLbClient.setNextRequestState(LoadBalancerRequestState.SUCCESS);
    deployChecker.checkDeploys();
    // Acceptance checks fail
    testingLbClient.setNextRequestState(LoadBalancerRequestState.WAITING);
    deployChecker.checkDeploys();
    pendingDeploy = deployManager.getPendingDeploy(requestId).get();
    Assertions.assertEquals(DeployState.WAITING, pendingDeploy.getCurrentDeployState());
    Assertions.assertEquals(DeployAcceptanceState.FAILED, pendingDeploy.getDeployProgress().getStepAcceptanceResults().entrySet().iterator().next().getValue());
    SingularityDeployProgress deployProgress = pendingDeploy.getDeployProgress();
    DeployProgressLbUpdateHolder lbUpdateHolder = deployProgress.getLbUpdates().get(deployProgress.getPendingLbUpdate().get().getLoadBalancerRequestId().toString());
    Assertions.assertTrue(lbUpdateHolder.getAdded().contains(firstTask.getTaskId()));
    Assertions.assertTrue(lbUpdateHolder.getRemoved().contains(firstNewTaskId));
    testingLbClient.setNextRequestState(LoadBalancerRequestState.SUCCESS);
    deployChecker.checkDeploys();
    SingularityDeployResult deployResult = deployManager.getDeployResult(requestId, secondDeployId).get();
    Assertions.assertEquals(DeployState.FAILED, deployResult.getDeployState());
    Assertions.assertTrue(deployResult.getMessage().get().contains("ruh-roh"));
}

Example 3 with DeployProgressLbUpdateHolder

use of com.hubspot.singularity.DeployProgressLbUpdateHolder in project Singularity by HubSpot.

the class SingularityDeployChecker method processLbState.

private SingularityDeployResult processLbState(SingularityRequest request, SingularityDeploy deploy, SingularityPendingDeploy pendingDeploy, Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest, Collection<SingularityTaskId> deployActiveTasks, SingularityLoadBalancerUpdate lbUpdate) {
    SingularityDeployProgress deployProgress = pendingDeploy.getDeployProgress();
    DeployProgressLbUpdateHolder lbUpdateHolder = deployProgress.getLbUpdates().get(lbUpdate.getLoadBalancerRequestId().toString());
    if (lbUpdateHolder == null) {
        return new SingularityDeployResult(DeployState.FAILED_INTERNAL_STATE, "Load balancer update metadata not found");
    }
    updateLoadBalancerStateForTasks(lbUpdateHolder.getAdded(), LoadBalancerRequestType.ADD, lbUpdate);
    updateLoadBalancerStateForTasks(lbUpdateHolder.getRemoved(), LoadBalancerRequestType.REMOVE, lbUpdate);
    DeployState deployState = SingularityDeployCheckHelper.interpretLoadBalancerState(lbUpdate, pendingDeploy.getCurrentDeployState());
    if (deployState == DeployState.SUCCEEDED) {
        SingularityDeployProgress updatedProgress = deployProgress.withFinishedLbUpdate(lbUpdate, lbUpdateHolder);
        updatePendingDeploy(pendingDeploy, DeployState.WAITING, updatedProgress);
        // All tasks for current step are launched and in the LB if needed
        return markStepLaunchFinished(pendingDeploy, deploy, deployActiveTasks, request, updatePendingDeployRequest, updatedProgress);
    } else if (deployState == DeployState.WAITING) {
        updatePendingDeploy(pendingDeploy, deployState, deployProgress.withPendingLbUpdate(lbUpdate, lbUpdateHolder.getAdded(), lbUpdateHolder.getRemoved(), false));
        maybeUpdatePendingRequest(pendingDeploy, deploy, request, updatePendingDeployRequest);
        return new SingularityDeployResult(DeployState.WAITING);
    } else {
        updatePendingDeploy(pendingDeploy, deployState, deployProgress.withFinishedLbUpdate(lbUpdate, lbUpdateHolder));
        maybeUpdatePendingRequest(pendingDeploy, deploy, request, updatePendingDeployRequest);
        return new SingularityDeployResult(deployState, lbUpdate, SingularityDeployFailure.lbUpdateFailed());
    }
}