Examples with SingularityRequest com.hubspot.singularity.SingularityRequest used on opensource projects

Example 1 with SingularityRequest

use of com.hubspot.singularity.SingularityRequest in project Singularity by HubSpot.

the class StateManager method updatePossiblyUnderProvisionedAndOverProvisionedIds.

private void updatePossiblyUnderProvisionedAndOverProvisionedIds(SingularityRequestWithState requestWithState, Map<String, Long> numInstances, List<String> overProvisionedRequestIds, Set<String> possiblyUnderProvisionedRequestIds) {
    if (requestWithState.getState().isRunnable() && requestWithState.getRequest().isAlwaysRunning()) {
        SingularityRequest request = requestWithState.getRequest();
        final int expectedInstances = request.getInstancesSafe();
        final Long numActualInstances = numInstances.get(request.getId());
        if (numActualInstances == null || numActualInstances < expectedInstances) {
            possiblyUnderProvisionedRequestIds.add(request.getId());
        } else if (numActualInstances > expectedInstances) {
            overProvisionedRequestIds.add(request.getId());
        }
    }
}

Example 2 with SingularityRequest

use of com.hubspot.singularity.SingularityRequest in project Singularity by HubSpot.

the class ScheduleMigration method applyMigration.

@Override
public void applyMigration() {
    LOG.info("Starting migration to fix certain CRON schedules");
    final long start = System.currentTimeMillis();
    int num = 0;
    for (SingularityRequestWithState requestWithState : requestManager.getRequests()) {
        if (requestWithState.getRequest().isScheduled()) {
            Optional<String> schedule = requestWithState.getRequest().getSchedule();
            Optional<String> quartzSchedule = requestWithState.getRequest().getQuartzSchedule();
            Optional<ScheduleType> scheduleType = requestWithState.getRequest().getScheduleType();
            if (scheduleType.isPresent() && scheduleType.get() != ScheduleType.CRON) {
                LOG.info("Skipping {}, it had schedule type: {}", requestWithState.getRequest().getId(), scheduleType.get());
                continue;
            }
            if (quartzSchedule.isPresent() && schedule.isPresent() && quartzSchedule.get().equals(schedule.get())) {
                LOG.info("Skipping {}, assuming it was quartz - it had quartz schedule == schedule {}", requestWithState.getRequest().getId(), schedule.get());
                continue;
            }
            if (!schedule.isPresent()) {
                LOG.info("Skipping {}, it had no schedule", requestWithState.getRequest().getId());
                continue;
            }
            String actualSchedule = schedule.get();
            String newQuartzSchedule = null;
            try {
                newQuartzSchedule = validator.getQuartzScheduleFromCronSchedule(actualSchedule);
            } catch (WebApplicationException e) {
                LOG.error("Failed to convert {} ({}) due to {}", requestWithState.getRequest().getId(), actualSchedule, e.getResponse().getEntity());
                continue;
            }
            if (quartzSchedule.isPresent() && quartzSchedule.get().equals(newQuartzSchedule)) {
                LOG.info("Skipping {}, migration had no effect {}", requestWithState.getRequest().getId(), newQuartzSchedule);
                continue;
            }
            SingularityRequest newRequest = requestWithState.getRequest().toBuilder().setQuartzSchedule(Optional.of(newQuartzSchedule)).build();
            try {
                LOG.info("Saving new schedule (quartz {} - from {}) for {}", newQuartzSchedule, actualSchedule, newRequest.getId());
                curator.setData().forPath("/requests/all/" + newRequest.getId(), requestTranscoder.toBytes(new SingularityRequestWithState(newRequest, requestWithState.getState(), requestWithState.getTimestamp())));
                num++;
            } catch (Throwable t) {
                LOG.error("Failed to write {}", newRequest.getId(), t);
                throw Throwables.propagate(t);
            }
        }
    }
    LOG.info("Applied {} in {}", num, JavaUtils.duration(start));
}

Example 3 with SingularityRequest

use of com.hubspot.singularity.SingularityRequest in project Singularity by HubSpot.

the class SingularitySchedulerTest method itSetsRequestStateToDeletedIfTaskCleanupFails.

@Test
public void itSetsRequestStateToDeletedIfTaskCleanupFails() {
    initRequest();
    SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest();
    requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build(), singularityUser);
    initFirstDeploy();
    SingularityTask firstTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING);
    launchTask(request, firstDeploy, 2, TaskState.TASK_RUNNING);
    Assert.assertEquals(requestId, requestManager.getActiveRequests().iterator().next().getRequest().getId());
    Assert.assertEquals(2, taskManager.getActiveTaskIds().size());
    requestManager.startDeletingRequest(request, Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent());
    Assert.assertEquals(requestId, requestManager.getCleanupRequests().get(0).getRequestId());
    Assert.assertEquals(RequestState.DELETING, requestManager.getRequest(requestId).get().getState());
    statusUpdate(firstTask, TaskState.TASK_FAILED);
    Assert.assertEquals(1, taskManager.getActiveTaskIds().size());
    cleaner.drainCleanupQueue();
    Assert.assertEquals(0, taskManager.getCleanupTaskIds().size());
    killKilledTasks();
    cleaner.drainCleanupQueue();
    Assert.assertFalse(requestManager.getRequest(requestId).isPresent());
}

Example 4 with SingularityRequest

use of com.hubspot.singularity.SingularityRequest in project Singularity by HubSpot.

the class SingularitySchedulerTest method testInvalidQuartzTimeZoneErrors.

@Test(expected = WebApplicationException.class)
public void testInvalidQuartzTimeZoneErrors() {
    SingularityRequest req = new SingularityRequestBuilder(requestId, RequestType.SCHEDULED).setQuartzSchedule(Optional.of("*/1 * * * * ? 2020")).setScheduleType(Optional.of(ScheduleType.QUARTZ)).setScheduleTimeZone(Optional.of("invalid_timezone")).build();
    requestResource.postRequest(req, singularityUser);
}

Example 5 with SingularityRequest

use of com.hubspot.singularity.SingularityRequest in project Singularity by HubSpot.

the class SingularitySchedulerTest method testIncrementalBounce.

@Test
public void testIncrementalBounce() {
    initRequest();
    // set up slaves so scale validate will pass
    resourceOffers(2);
    SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest();
    requestResource.postRequest(request.toBuilder().setSlavePlacement(Optional.of(SlavePlacement.SEPARATE_BY_REQUEST)).setInstances(Optional.of(2)).build(), singularityUser);
    initHCDeploy();
    SingularityTask taskOne = startSeparatePlacementTask(firstDeploy, 1);
    SingularityTask taskTwo = startSeparatePlacementTask(firstDeploy, 2);
    requestManager.createCleanupRequest(new SingularityRequestCleanup(user, RequestCleanupType.INCREMENTAL_BOUNCE, System.currentTimeMillis(), Optional.<Boolean>absent(), Optional.absent(), requestId, Optional.of(firstDeployId), Optional.<Boolean>absent(), Optional.<String>absent(), Optional.<String>absent(), Optional.<SingularityShellCommand>absent()));
    Assert.assertTrue(requestManager.cleanupRequestExists(requestId));
    cleaner.drainCleanupQueue();
    Assert.assertTrue(!requestManager.cleanupRequestExists(requestId));
    Assert.assertEquals(2, taskManager.getCleanupTaskIds().size());
    resourceOffers(3);
    SingularityTask taskThree = null;
    for (SingularityTask task : taskManager.getActiveTasks()) {
        if (!task.getTaskId().equals(taskOne.getTaskId()) && !task.getTaskId().equals(taskTwo.getTaskId())) {
            taskThree = task;
        }
    }
    statusUpdate(taskThree, TaskState.TASK_RUNNING, Optional.of(1L));
    Assert.assertEquals(3, taskManager.getActiveTaskIds().size());
    cleaner.drainCleanupQueue();
    // No old tasks should be killed before new ones pass healthchecks
    Assert.assertEquals(2, taskManager.getCleanupTaskIds().size());
    taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.of(200), Optional.of(1000L), System.currentTimeMillis(), Optional.<String>absent(), Optional.<String>absent(), taskThree.getTaskId(), Optional.<Boolean>absent()));
    cleaner.drainCleanupQueue();
    Assert.assertEquals(1, taskManager.getCleanupTaskIds().size());
    statusUpdate(taskOne, TaskState.TASK_KILLED);
    resourceOffers(3);
    SingularityTask taskFour = null;
    for (SingularityTask task : taskManager.getActiveTasks()) {
        if (!task.getTaskId().equals(taskOne.getTaskId()) && !task.getTaskId().equals(taskTwo.getTaskId()) && !task.getTaskId().equals(taskThree.getTaskId())) {
            taskFour = task;
        }
    }
    statusUpdate(taskFour, TaskState.TASK_RUNNING, Optional.of(1L));
    taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.of(200), Optional.of(1000L), System.currentTimeMillis(), Optional.<String>absent(), Optional.<String>absent(), taskFour.getTaskId(), Optional.<Boolean>absent()));
    cleaner.drainCleanupQueue();
    Assert.assertTrue(taskManager.getCleanupTaskIds().isEmpty());
}