Examples with DeployAcceptanceResult com.hubspot.singularity.DeployAcceptanceResult used on opensource projects

Example 1 with DeployAcceptanceResult

use of com.hubspot.singularity.DeployAcceptanceResult in project Singularity by HubSpot.

the class SingularityDeployAcceptanceTest method testDeployWithAcceptanceConditions.

@Test
public void testDeployWithAcceptanceConditions() {
    NoopDeployAcceptanceHook hook = (NoopDeployAcceptanceHook) acceptanceHooks.iterator().next();
    hook.setNextResult(new DeployAcceptanceResult(DeployAcceptanceState.PENDING, "waiting"));
    initRequest();
    SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest();
    requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build(), singularityUser);
    initFirstDeploy();
    SingularityTask firstTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING);
    SingularityTask secondTask = launchTask(request, firstDeploy, 2, TaskState.TASK_RUNNING);
    SingularityDeployBuilder builder = new SingularityDeployBuilder(requestId, secondDeployId);
    builder.setCommand(Optional.of("sleep 1")).setCanaryDeploySettings(CanaryDeploySettings.newbuilder().setInstanceGroupSize(1).setEnableCanaryDeploy(true).setAcceptanceMode(DeployAcceptanceMode.CHECKS).setInstanceGroupSize(1).build());
    deployResource.deploy(new SingularityDeployRequest(builder.build(), Optional.empty(), Optional.empty()), singularityUser);
    deployChecker.checkDeploys();
    scheduler.drainPendingQueue();
    Assertions.assertEquals(1, taskManager.getPendingTaskIds().size());
    resourceOffers();
    Assertions.assertEquals(1, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size());
    SingularityTaskId firstNewTaskId = taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).get(0);
    statusUpdate(taskManager.getTask(firstNewTaskId).get(), TaskState.TASK_RUNNING);
    deployChecker.checkDeploys();
    deployChecker.checkDeploys();
    SingularityDeployProgress deployProgressStepOne = deployManager.getPendingDeploys().get(0).getDeployProgress();
    Assertions.assertEquals(1, deployProgressStepOne.getTargetActiveInstances());
    Assertions.assertEquals("(no-op - PENDING) waiting", deployProgressStepOne.getAcceptanceResultMessageHistory().iterator().next());
    Assertions.assertEquals(DeployAcceptanceState.PENDING, deployProgressStepOne.getStepAcceptanceResults().get(hook.getName()));
    hook.setNextResult(new DeployAcceptanceResult(DeployAcceptanceState.SUCCEEDED, "YAY"));
    deployChecker.checkDeploys();
    deployProgressStepOne = deployManager.getPendingDeploys().get(0).getDeployProgress();
    Assertions.assertEquals(2, deployProgressStepOne.getTargetActiveInstances());
    Assertions.assertTrue(deployProgressStepOne.getAcceptanceResultMessageHistory().contains("(no-op - SUCCEEDED) YAY"));
    cleaner.drainCleanupQueue();
    statusUpdate(firstTask, TaskState.TASK_KILLED);
    deployChecker.checkDeploys();
    SingularityDeployProgress deployProgressStepTwo = deployManager.getPendingDeploys().get(0).getDeployProgress();
    Assertions.assertFalse(deployProgressStepTwo.isStepLaunchComplete());
    Assertions.assertEquals(2, deployProgressStepTwo.getTargetActiveInstances());
    scheduler.drainPendingQueue();
    Assertions.assertEquals(1, taskManager.getPendingTaskIds().size());
    resourceOffers();
    Assertions.assertEquals(2, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size());
    for (SingularityTaskId taskId : taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId)) {
        statusUpdate(taskManager.getTask(taskId).get(), TaskState.TASK_RUNNING);
    }
    deployChecker.checkDeploys();
    deployChecker.checkDeploys();
    Assertions.assertEquals(2, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size());
    Assertions.assertEquals(DeployState.SUCCEEDED, deployManager.getDeployResult(requestId, secondDeployId).get().getDeployState());
}

Example 2 with DeployAcceptanceResult

use of com.hubspot.singularity.DeployAcceptanceResult in project Singularity by HubSpot.

the class SingularityDeployAcceptanceTest method testLbRevertsAfterFailedAcceptanceStepOnNonCanary.

@Test
public void testLbRevertsAfterFailedAcceptanceStepOnNonCanary() {
    NoopDeployAcceptanceHook hook = (NoopDeployAcceptanceHook) acceptanceHooks.iterator().next();
    hook.setNextResult(new DeployAcceptanceResult(DeployAcceptanceState.FAILED, "ruh-roh"));
    initLoadBalancedRequest();
    initFirstDeploy();
    SingularityTask firstTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING);
    SingularityDeployBuilder builder = new SingularityDeployBuilder(requestId, secondDeployId);
    builder.setCommand(Optional.of("sleep 1")).setCanaryDeploySettings(CanaryDeploySettings.newbuilder().setAcceptanceMode(DeployAcceptanceMode.CHECKS).setEnableCanaryDeploy(false).build()).setServiceBasePath(Optional.of("/basepath")).setLoadBalancerGroups(Optional.of(Collections.singleton("group")));
    deployResource.deploy(new SingularityDeployRequest(builder.build(), Optional.of(false), Optional.empty()), singularityUser);
    deployChecker.checkDeploys();
    scheduler.drainPendingQueue();
    Assertions.assertEquals(1, taskManager.getPendingTaskIds().size());
    resourceOffers();
    Assertions.assertEquals(1, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size());
    SingularityTaskId firstNewTaskId = taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).get(0);
    statusUpdate(taskManager.getTask(firstNewTaskId).get(), TaskState.TASK_RUNNING);
    deployChecker.checkDeploys();
    SingularityPendingDeploy pendingDeploy = deployManager.getPendingDeploy(requestId).get();
    Assertions.assertEquals(DeployState.WAITING, pendingDeploy.getCurrentDeployState());
    testingLbClient.setNextRequestState(LoadBalancerRequestState.WAITING);
    deployChecker.checkDeploys();
    pendingDeploy = deployManager.getPendingDeploy(requestId).get();
    Assertions.assertEquals(DeployState.WAITING, pendingDeploy.getCurrentDeployState());
    testingLbClient.setNextRequestState(LoadBalancerRequestState.SUCCESS);
    deployChecker.checkDeploys();
    // Acceptance checks fail
    testingLbClient.setNextRequestState(LoadBalancerRequestState.WAITING);
    deployChecker.checkDeploys();
    pendingDeploy = deployManager.getPendingDeploy(requestId).get();
    Assertions.assertEquals(DeployState.WAITING, pendingDeploy.getCurrentDeployState());
    Assertions.assertEquals(DeployAcceptanceState.FAILED, pendingDeploy.getDeployProgress().getStepAcceptanceResults().entrySet().iterator().next().getValue());
    SingularityDeployProgress deployProgress = pendingDeploy.getDeployProgress();
    DeployProgressLbUpdateHolder lbUpdateHolder = deployProgress.getLbUpdates().get(deployProgress.getPendingLbUpdate().get().getLoadBalancerRequestId().toString());
    Assertions.assertTrue(lbUpdateHolder.getAdded().contains(firstTask.getTaskId()));
    Assertions.assertTrue(lbUpdateHolder.getRemoved().contains(firstNewTaskId));
    testingLbClient.setNextRequestState(LoadBalancerRequestState.SUCCESS);
    deployChecker.checkDeploys();
    SingularityDeployResult deployResult = deployManager.getDeployResult(requestId, secondDeployId).get();
    Assertions.assertEquals(DeployState.FAILED, deployResult.getDeployState());
    Assertions.assertTrue(deployResult.getMessage().get().contains("ruh-roh"));
}

Example 3 with DeployAcceptanceResult

use of com.hubspot.singularity.DeployAcceptanceResult in project Singularity by HubSpot.

the class SingularityDeployAcceptanceTest method testDeployWithFailingAcceptanceConditions.

@Test
public void testDeployWithFailingAcceptanceConditions() {
    NoopDeployAcceptanceHook hook = (NoopDeployAcceptanceHook) acceptanceHooks.iterator().next();
    hook.setNextResult(new DeployAcceptanceResult(DeployAcceptanceState.PENDING, "waiting"));
    initRequest();
    SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest();
    requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build(), singularityUser);
    initFirstDeploy();
    SingularityTask firstTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING);
    SingularityTask secondTask = launchTask(request, firstDeploy, 2, TaskState.TASK_RUNNING);
    SingularityDeployBuilder builder = new SingularityDeployBuilder(requestId, secondDeployId);
    builder.setCommand(Optional.of("sleep 1")).setCanaryDeploySettings(CanaryDeploySettings.newbuilder().setInstanceGroupSize(1).setEnableCanaryDeploy(true).setAcceptanceMode(DeployAcceptanceMode.CHECKS).setInstanceGroupSize(1).build());
    deployResource.deploy(new SingularityDeployRequest(builder.build(), Optional.empty(), Optional.empty()), singularityUser);
    deployChecker.checkDeploys();
    scheduler.drainPendingQueue();
    Assertions.assertEquals(1, taskManager.getPendingTaskIds().size());
    resourceOffers();
    Assertions.assertEquals(1, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size());
    SingularityTaskId firstNewTaskId = taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).get(0);
    statusUpdate(taskManager.getTask(firstNewTaskId).get(), TaskState.TASK_RUNNING);
    deployChecker.checkDeploys();
    deployChecker.checkDeploys();
    SingularityDeployProgress deployProgressStepOne = deployManager.getPendingDeploys().get(0).getDeployProgress();
    Assertions.assertEquals(1, deployProgressStepOne.getTargetActiveInstances());
    Assertions.assertEquals("(no-op - PENDING) waiting", deployProgressStepOne.getAcceptanceResultMessageHistory().iterator().next());
    Assertions.assertEquals(DeployAcceptanceState.PENDING, deployProgressStepOne.getStepAcceptanceResults().get(hook.getName()));
    hook.setNextResult(new DeployAcceptanceResult(DeployAcceptanceState.FAILED, ":sadpanda:"));
    deployChecker.checkDeploys();
    SingularityDeployResult deployResult = deployManager.getDeployResult(requestId, secondDeployId).get();
    Assertions.assertEquals(DeployState.FAILED, deployResult.getDeployState());
    Assertions.assertTrue(deployResult.getMessage().orElse("").contains(":sadpanda:"));
}

Example 4 with DeployAcceptanceResult

use of com.hubspot.singularity.DeployAcceptanceResult in project Singularity by HubSpot.

the class SingularityDeployAcceptanceTest method afterEach.

@AfterEach
public void afterEach() {
    NoopDeployAcceptanceHook hook = (NoopDeployAcceptanceHook) acceptanceHooks.iterator().next();
    hook.setNextResult(new DeployAcceptanceResult(DeployAcceptanceState.SUCCEEDED, "succeeded"));
    super.clearData();
}

Example 5 with DeployAcceptanceResult

use of com.hubspot.singularity.DeployAcceptanceResult in project Singularity by HubSpot.

the class SingularityDeployChecker method checkAcceptanceOfDeployStep.

private SingularityDeployResult checkAcceptanceOfDeployStep(SingularityRequest request, SingularityDeploy deploy, SingularityPendingDeploy pendingDeploy, Collection<SingularityTaskId> deployActiveTasks, Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest, Collection<SingularityTaskId> inactiveDeployMatchingTasks, Collection<SingularityTaskId> otherActiveTasks) {
    if (updatePendingDeployRequest.isPresent()) {
        maybeUpdatePendingRequest(pendingDeploy, deploy, request, updatePendingDeployRequest);
        return new SingularityDeployResult(DeployState.WAITING);
    }
    switch(deploy.getCanaryDeploySettings().getAcceptanceMode()) {
        case TIMED:
            if (System.currentTimeMillis() > pendingDeploy.getDeployProgress().getTimestamp() + deploy.getCanaryDeploySettings().getWaitMillisBetweenGroups()) {
                return checkCanaryMaybeFinished(request, deploy, pendingDeploy, deployActiveTasks, updatePendingDeployRequest, inactiveDeployMatchingTasks, otherActiveTasks);
            } else {
                LOG.info("Waiting for timed deploy step for {}", request.getId());
                return new SingularityDeployResult(DeployState.WAITING);
            }
        case CHECKS:
            Map<String, DeployAcceptanceResult> results = deployAcceptanceManager.getAcceptanceResults(request, deploy, pendingDeploy, deployActiveTasks, inactiveDeployMatchingTasks, otherActiveTasks);
            SingularityDeployProgress updatedProgress = pendingDeploy.getDeployProgress().withAcceptanceProgress(results);
            DeployState acceptanceHookDeployState = SingularityDeployAcceptanceManager.resultsToDeployState(updatedProgress.getStepAcceptanceResults());
            LOG.info("Acceptance checks had result {}", acceptanceHookDeployState);
            if (deploy.getCanaryDeploySettings().isEnableCanaryDeploy()) {
                if (acceptanceHookDeployState == DeployState.SUCCEEDED) {
                    if (deployActiveTasks.size() >= request.getInstancesSafe()) {
                        cleanupTasks(pendingDeploy, request, DeployState.SUCCEEDED, otherActiveTasks);
                        updatePendingDeploy(pendingDeploy, acceptanceHookDeployState, updatedProgress);
                        return new SingularityDeployResult(acceptanceHookDeployState, String.join(", ", updatedProgress.getAcceptanceResultMessageHistory()));
                    } else {
                        return advanceDeployStep(updatedProgress, request, deploy, pendingDeploy, deployActiveTasks, updatePendingDeployRequest, inactiveDeployMatchingTasks, otherActiveTasks);
                    }
                } else if (acceptanceHookDeployState == DeployState.FAILED) {
                    updatePendingDeploy(pendingDeploy, acceptanceHookDeployState, updatedProgress);
                    return new SingularityDeployResult(acceptanceHookDeployState, String.join(", ", updatedProgress.getAcceptanceResultMessageHistory()));
                }
                updatePendingDeploy(pendingDeploy, acceptanceHookDeployState, updatedProgress);
                return new SingularityDeployResult(acceptanceHookDeployState);
            } else {
                // Clean up all old tasks on acceptance
                switch(acceptanceHookDeployState) {
                    case WAITING:
                        updatePendingDeploy(pendingDeploy, acceptanceHookDeployState, updatedProgress);
                        return new SingularityDeployResult(DeployState.WAITING);
                    case SUCCEEDED:
                        cleanupTasks(pendingDeploy, request, DeployState.SUCCEEDED, otherActiveTasks);
                        updatePendingDeploy(pendingDeploy, acceptanceHookDeployState, updatedProgress);
                        return new SingularityDeployResult(DeployState.SUCCEEDED);
                    default:
                        LOG.info("Acceptance checks failed, cleaning up");
                        updatePendingDeploy(pendingDeploy, acceptanceHookDeployState, updatedProgress);
                        if (request.isLoadBalanced() && lbClient.isEnabled()) {
                            // Add previous tasks back to load balancer, since we previously took them out
                            if (updatedProgress.getPendingLbUpdate().isPresent()) {
                                return checkLbRevertToActiveTasks(request, deploy, pendingDeploy, updatedProgress, acceptanceHookDeployState, deployActiveTasks, otherActiveTasks);
                            } else {
                                return enqueueLbRevertToActiveTasks(request, pendingDeploy, updatedProgress, deployActiveTasks, otherActiveTasks);
                            }
                        } else {
                            return new SingularityDeployResult(acceptanceHookDeployState, String.join(", ", updatedProgress.getAcceptanceResultMessageHistory()));
                        }
                }
            }
        case NONE:
        default:
            return checkCanaryMaybeFinished(request, deploy, pendingDeploy, deployActiveTasks, updatePendingDeployRequest, inactiveDeployMatchingTasks, otherActiveTasks);
    }
}