Examples with SingularityRequest com.hubspot.singularity.SingularityRequest used on opensource projects

Example 61 with SingularityRequest

use of com.hubspot.singularity.SingularityRequest in project Singularity by HubSpot.

the class SingularitySchedulerTest method testSchedulerPriority.

@Test
public void testSchedulerPriority() {
    final SingularityRequest lowPriorityRequest = new SingularityRequestBuilder("lowPriorityRequest", RequestType.WORKER).setTaskPriorityLevel(Optional.of(.25)).build();
    saveRequest(lowPriorityRequest);
    final SingularityRequest mediumPriorityRequest = new SingularityRequestBuilder("mediumPriorityRequest", RequestType.WORKER).setTaskPriorityLevel(Optional.of(.5)).build();
    saveRequest(mediumPriorityRequest);
    final SingularityRequest highPriorityRequest = new SingularityRequestBuilder("highPriorityRequest", RequestType.WORKER).setTaskPriorityLevel(Optional.of(.75)).build();
    saveRequest(highPriorityRequest);
    final SingularityDeploy lowPriorityDeploy = initAndFinishDeploy(lowPriorityRequest, "lowPriorityDeploy");
    final SingularityDeploy mediumPriorityDeploy = initAndFinishDeploy(mediumPriorityRequest, "mediumPriorityDeploy");
    final SingularityDeploy highPriorityDeploy = initAndFinishDeploy(highPriorityRequest, "highPriorityDeploy");
    // Task requests launched at ~ the same time should be in priority order
    long now = System.currentTimeMillis();
    List<SingularityTaskRequest> requestsByPriority = Arrays.asList(buildTaskRequest(lowPriorityRequest, lowPriorityDeploy, now), buildTaskRequest(mediumPriorityRequest, mediumPriorityDeploy, now), buildTaskRequest(highPriorityRequest, highPriorityDeploy, now));
    List<SingularityTaskRequest> sortedRequestsByPriority = taskPrioritizer.getSortedDueTasks(requestsByPriority);
    Assert.assertEquals(sortedRequestsByPriority.get(0).getRequest().getId(), highPriorityRequest.getId());
    Assert.assertEquals(sortedRequestsByPriority.get(1).getRequest().getId(), mediumPriorityRequest.getId());
    Assert.assertEquals(sortedRequestsByPriority.get(2).getRequest().getId(), lowPriorityRequest.getId());
    // A lower priority task that is long overdue should be run before a higher priority task
    now = System.currentTimeMillis();
    List<SingularityTaskRequest> requestsByOverdueAndPriority = Arrays.asList(// 2 min overdue
    buildTaskRequest(lowPriorityRequest, lowPriorityDeploy, now - 120000), // 60s overdue
    buildTaskRequest(mediumPriorityRequest, mediumPriorityDeploy, now - 30000), // Not overdue
    buildTaskRequest(highPriorityRequest, highPriorityDeploy, now));
    List<SingularityTaskRequest> sortedRequestsByOverdueAndPriority = taskPrioritizer.getSortedDueTasks(requestsByOverdueAndPriority);
    Assert.assertEquals(sortedRequestsByOverdueAndPriority.get(0).getRequest().getId(), lowPriorityRequest.getId());
    Assert.assertEquals(sortedRequestsByOverdueAndPriority.get(1).getRequest().getId(), mediumPriorityRequest.getId());
    Assert.assertEquals(sortedRequestsByOverdueAndPriority.get(2).getRequest().getId(), highPriorityRequest.getId());
}

Example 62 with SingularityRequest

use of com.hubspot.singularity.SingularityRequest in project Singularity by HubSpot.

the class SingularitySchedulerTest method testPriorityFreezeDoesntLaunchTasks.

@Test
public void testPriorityFreezeDoesntLaunchTasks() {
    // deploy lowPriorityRequest (affected by priority freeze)
    final SingularityRequest lowPriorityRequest = new SingularityRequestBuilder("lowPriorityRequest", RequestType.ON_DEMAND).setTaskPriorityLevel(Optional.of(.25)).build();
    saveRequest(lowPriorityRequest);
    deployResource.deploy(new SingularityDeployRequest(new SingularityDeployBuilder(lowPriorityRequest.getId(), "d1").setCommand(Optional.of("cmd")).build(), Optional.absent(), Optional.absent()), singularityUser);
    // deploy medium priority request (NOT affected by priority freeze)
    final SingularityRequest mediumPriorityRequest = new SingularityRequestBuilder("mediumPriorityRequest", RequestType.ON_DEMAND).setTaskPriorityLevel(Optional.of(.5)).build();
    saveRequest(mediumPriorityRequest);
    deployResource.deploy(new SingularityDeployRequest(new SingularityDeployBuilder(mediumPriorityRequest.getId(), "d2").setCommand(Optional.of("cmd")).build(), Optional.absent(), Optional.absent()), singularityUser);
    // create priority freeze
    priorityManager.createPriorityFreeze(new SingularityPriorityFreezeParent(new SingularityPriorityFreeze(0.3, true, Optional.<String>absent(), Optional.<String>absent()), System.currentTimeMillis(), Optional.<String>absent()));
    // launch both tasks
    requestResource.scheduleImmediately(singularityUser, lowPriorityRequest.getId());
    requestResource.scheduleImmediately(singularityUser, mediumPriorityRequest.getId());
    // drain pending queue
    scheduler.drainPendingQueue();
    resourceOffers();
    // assert that lowPriorityRequest has a pending task
    Assert.assertEquals(1, taskManager.getPendingTaskIds().size());
    Assert.assertEquals(lowPriorityRequest.getId(), taskManager.getPendingTaskIds().get(0).getRequestId());
    // assert that only mediumPriorityRequest has an active task
    Assert.assertEquals(0, taskManager.getActiveTaskIdsForRequest(lowPriorityRequest.getId()).size());
    Assert.assertEquals(1, taskManager.getActiveTaskIdsForRequest(mediumPriorityRequest.getId()).size());
    // delete priority freeze
    Assert.assertEquals(SingularityDeleteResult.DELETED, priorityManager.deleteActivePriorityFreeze());
    // drain pending
    scheduler.drainPendingQueue();
    resourceOffers();
    // check that both requests have active tasks
    Assert.assertEquals(1, taskManager.getActiveTaskIdsForRequest(lowPriorityRequest.getId()).size());
    Assert.assertEquals(1, taskManager.getActiveTaskIdsForRequest(mediumPriorityRequest.getId()).size());
}

Example 63 with SingularityRequest

use of com.hubspot.singularity.SingularityRequest in project Singularity by HubSpot.

the class SingularitySchedulerTest method testPriorityFreezeKillsActiveTasks.

@Test
public void testPriorityFreezeKillsActiveTasks() {
    final SingularityRequest lowPriorityRequest = new SingularityRequestBuilder("lowPriorityRequest", RequestType.WORKER).setTaskPriorityLevel(Optional.of(.25)).build();
    saveRequest(lowPriorityRequest);
    final SingularityRequest mediumPriorityRequest = new SingularityRequestBuilder("mediumPriorityRequest", RequestType.WORKER).setTaskPriorityLevel(Optional.of(.5)).build();
    saveRequest(mediumPriorityRequest);
    final SingularityRequest highPriorityRequest = new SingularityRequestBuilder("highPriorityRequest", RequestType.WORKER).setTaskPriorityLevel(Optional.of(.75)).build();
    saveRequest(highPriorityRequest);
    final SingularityDeploy lowPriorityDeploy = initAndFinishDeploy(lowPriorityRequest, "lowPriorityDeploy");
    final SingularityDeploy mediumPriorityDeploy = initAndFinishDeploy(mediumPriorityRequest, "mediumPriorityDeploy");
    SingularityDeploy highPriorityDeploy = initAndFinishDeploy(highPriorityRequest, "highPriorityDeploy");
    final SingularityTask lowPriorityTask = launchTask(lowPriorityRequest, lowPriorityDeploy, 2, 1, TaskState.TASK_RUNNING);
    final SingularityTask mediumPriorityTask = launchTask(mediumPriorityRequest, mediumPriorityDeploy, 1, 1, TaskState.TASK_RUNNING);
    final SingularityTask highPriorityTask = launchTask(highPriorityRequest, highPriorityDeploy, 10, 1, TaskState.TASK_RUNNING);
    // priority freeze of .5 means that lowPriorityRequest's task should have a cleanup
    priorityResource.createPriorityFreeze(singularityUser, new SingularityPriorityFreeze(.5, true, Optional.of("test"), Optional.absent()));
    // perform the killing
    priorityKillPoller.runActionOnPoll();
    // assert lowPriorityRequest has a PRIORITY_KILL task cleanup and that mediumPriorityRequest and highPriorityRequest should not have cleanups
    Assert.assertEquals(TaskCleanupType.PRIORITY_KILL, taskManager.getTaskCleanup(lowPriorityTask.getTaskId().getId()).get().getCleanupType());
    Assert.assertEquals(false, taskManager.getTaskCleanup(mediumPriorityTask.getTaskId().getId()).isPresent());
    Assert.assertEquals(false, taskManager.getTaskCleanup(highPriorityTask.getTaskId().getId()).isPresent());
    // kill task(s) with cleanups
    cleaner.drainCleanupQueue();
    killKilledTasks();
    // assert lowPriorityTask was killed, mediumPriorityTask and highPriorityTask are still running
    Assert.assertEquals(ExtendedTaskState.TASK_KILLED, taskManager.getTaskHistory(lowPriorityTask.getTaskId()).get().getLastTaskUpdate().get().getTaskState());
    Assert.assertEquals(ExtendedTaskState.TASK_RUNNING, taskManager.getTaskHistory(mediumPriorityTask.getTaskId()).get().getLastTaskUpdate().get().getTaskState());
    Assert.assertEquals(ExtendedTaskState.TASK_RUNNING, taskManager.getTaskHistory(highPriorityTask.getTaskId()).get().getLastTaskUpdate().get().getTaskState());
    // assert lowPriorityRequest has a pending task
    final SingularityPendingTaskId pendingTaskId = taskManager.getPendingTaskIds().get(0);
    Assert.assertEquals(PendingType.TASK_DONE, pendingTaskId.getPendingType());
    Assert.assertEquals(lowPriorityRequest.getId(), pendingTaskId.getRequestId());
    // end the priority freeze
    priorityResource.deleteActivePriorityFreeze(singularityUser);
    // launch task(s)
    scheduler.drainPendingQueue();
    resourceOffers();
    // assert lowPriorityRequest has a new task running
    Assert.assertNotEquals(lowPriorityTask.getTaskId(), taskManager.getActiveTaskIdsForRequest(lowPriorityRequest.getId()).get(0).getId());
}

Example 64 with SingularityRequest

use of com.hubspot.singularity.SingularityRequest in project Singularity by HubSpot.

the class SingularityDeploysTest method testDeployWithManualStep.

@Test
public void testDeployWithManualStep() {
    initRequest();
    SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest();
    requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build(), singularityUser);
    initFirstDeploy();
    SingularityTask firstTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING);
    SingularityTask secondTask = launchTask(request, firstDeploy, 2, TaskState.TASK_RUNNING);
    deploy(secondDeployId, Optional.<Boolean>absent(), Optional.of(1), Optional.of(false), false);
    deployChecker.checkDeploys();
    scheduler.drainPendingQueue();
    Assert.assertEquals(1, taskManager.getPendingTaskIds().size());
    resourceOffers();
    Assert.assertEquals(1, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size());
    SingularityTaskId firstNewTaskId = taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).get(0);
    statusUpdate(taskManager.getTask(firstNewTaskId).get(), TaskState.TASK_RUNNING);
    deployChecker.checkDeploys();
    Assert.assertEquals(1, taskManager.getCleanupTaskIds().size());
    Assert.assertTrue(taskManager.getCleanupTaskIds().contains(firstTask.getTaskId()));
    SingularityDeployProgress deployProgressStepOne = deployManager.getPendingDeploys().get(0).getDeployProgress().get();
    Assert.assertTrue(deployProgressStepOne.isStepComplete());
    Assert.assertEquals(1, deployProgressStepOne.getTargetActiveInstances());
    cleaner.drainCleanupQueue();
    statusUpdate(firstTask, TaskState.TASK_KILLED);
    deployChecker.checkDeploys();
    // Deploy should not have moved to next step even though instances are launched
    deployProgressStepOne = deployManager.getPendingDeploys().get(0).getDeployProgress().get();
    Assert.assertTrue(deployProgressStepOne.isStepComplete());
    Assert.assertEquals(1, deployProgressStepOne.getTargetActiveInstances());
    // Add the 'ok' to move to the next step
    deployResource.updatePendingDeploy(singularityUser, new SingularityUpdatePendingDeployRequest(requestId, secondDeployId, 2));
    deployChecker.checkDeploys();
    SingularityDeployProgress deployProgressStepTwo = deployManager.getPendingDeploys().get(0).getDeployProgress().get();
    Assert.assertFalse(deployProgressStepTwo.isStepComplete());
    Assert.assertEquals(2, deployProgressStepTwo.getTargetActiveInstances());
    scheduler.drainPendingQueue();
    Assert.assertEquals(1, taskManager.getPendingTaskIds().size());
    resourceOffers();
    Assert.assertEquals(2, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size());
    for (SingularityTaskId taskId : taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId)) {
        statusUpdate(taskManager.getTask(taskId).get(), TaskState.TASK_RUNNING);
    }
    deployChecker.checkDeploys();
    Assert.assertEquals(2, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size());
    Assert.assertEquals(DeployState.SUCCEEDED, deployManager.getDeployResult(requestId, secondDeployId).get().getDeployState());
}

Example 65 with SingularityRequest

use of com.hubspot.singularity.SingularityRequest in project Singularity by HubSpot.

the class SingularityDeploysTest method testIncrementalDeployCancel.

@Test
public void testIncrementalDeployCancel() {
    initRequest();
    // Set up incremental deploy that is partly finished
    SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest();
    requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build(), singularityUser);
    initFirstDeploy();
    SingularityTask firstTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING);
    launchTask(request, firstDeploy, 2, TaskState.TASK_RUNNING);
    deploy(secondDeployId, Optional.<Boolean>absent(), Optional.of(1), Optional.<Boolean>absent(), false);
    deployChecker.checkDeploys();
    scheduler.drainPendingQueue();
    resourceOffers();
    SingularityTaskId firstNewTaskId = taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).get(0);
    statusUpdate(taskManager.getTask(firstNewTaskId).get(), TaskState.TASK_RUNNING);
    deployChecker.checkDeploys();
    cleaner.drainCleanupQueue();
    statusUpdate(firstTask, TaskState.TASK_KILLED);
    deployChecker.checkDeploys();
    scheduler.drainPendingQueue();
    resourceOffers();
    // End in-progress incremental deploy setup
    deployResource.cancelDeploy(singularityUser, requestId, secondDeployId);
    deployChecker.checkDeploys();
    Assert.assertEquals(taskManager.getCleanupTasks().get(0).getCleanupType(), TaskCleanupType.INCREMENTAL_DEPLOY_CANCELLED);
    // Incremental deploy task should not be shut down while active deploy is below target instances
    cleaner.drainCleanupQueue();
    Assert.assertTrue(taskManager.getKilledTaskIdRecords().isEmpty());
    Assert.assertEquals(taskManager.getCleanupTasks().get(0).getCleanupType(), TaskCleanupType.INCREMENTAL_DEPLOY_CANCELLED);
    launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING);
    cleaner.drainCleanupQueue();
    Assert.assertFalse(taskManager.getKilledTaskIdRecords().isEmpty());
}