Examples with SingularityRequestCleanup com.hubspot.singularity.SingularityRequestCleanup used on opensource projects

Example 1 with SingularityRequestCleanup

use of com.hubspot.singularity.SingularityRequestCleanup in project Singularity by HubSpot.

the class RequestHelper method checkReschedule.

private void checkReschedule(SingularityRequest newRequest, Optional<SingularityRequest> maybeOldRequest, Optional<String> user, long timestamp, Optional<Boolean> skipHealthchecks, Optional<String> message, Optional<SingularityBounceRequest> maybeBounceRequest) {
    if (!maybeOldRequest.isPresent()) {
        return;
    }
    if (shouldReschedule(newRequest, maybeOldRequest.get())) {
        Optional<String> maybeDeployId = deployManager.getInUseDeployId(newRequest.getId());
        if (maybeDeployId.isPresent()) {
            if (maybeBounceRequest.isPresent()) {
                Optional<String> actionId = maybeBounceRequest.get().getActionId().or(Optional.of(UUID.randomUUID().toString()));
                Optional<Boolean> removeFromLoadBalancer = Optional.absent();
                SingularityCreateResult createResult = requestManager.createCleanupRequest(new SingularityRequestCleanup(user, maybeBounceRequest.get().getIncremental().or(true) ? RequestCleanupType.INCREMENTAL_BOUNCE : RequestCleanupType.BOUNCE, System.currentTimeMillis(), Optional.<Boolean>absent(), removeFromLoadBalancer, newRequest.getId(), Optional.of(maybeDeployId.get()), skipHealthchecks, message, actionId, maybeBounceRequest.get().getRunShellCommandBeforeKill()));
                if (createResult != SingularityCreateResult.EXISTED) {
                    requestManager.bounce(newRequest, System.currentTimeMillis(), user, Optional.of("Bouncing due to bounce after scale"));
                    final SingularityBounceRequest validatedBounceRequest = validator.checkBounceRequest(maybeBounceRequest.get());
                    requestManager.saveExpiringObject(new SingularityExpiringBounce(newRequest.getId(), maybeDeployId.get(), user, System.currentTimeMillis(), validatedBounceRequest, actionId.get()));
                } else {
                    requestManager.addToPendingQueue(new SingularityPendingRequest(newRequest.getId(), maybeDeployId.get(), timestamp, user, PendingType.UPDATED_REQUEST, skipHealthchecks, message));
                }
            } else {
                requestManager.addToPendingQueue(new SingularityPendingRequest(newRequest.getId(), maybeDeployId.get(), timestamp, user, PendingType.UPDATED_REQUEST, skipHealthchecks, message));
            }
        }
    }
}

Example 2 with SingularityRequestCleanup

use of com.hubspot.singularity.SingularityRequestCleanup in project Singularity by HubSpot.

the class SingularitySchedulerTest method testIncrementalBounce.

@Test
public void testIncrementalBounce() {
    initRequest();
    // set up slaves so scale validate will pass
    resourceOffers(2);
    SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest();
    requestResource.postRequest(request.toBuilder().setSlavePlacement(Optional.of(SlavePlacement.SEPARATE_BY_REQUEST)).setInstances(Optional.of(2)).build(), singularityUser);
    initHCDeploy();
    SingularityTask taskOne = startSeparatePlacementTask(firstDeploy, 1);
    SingularityTask taskTwo = startSeparatePlacementTask(firstDeploy, 2);
    requestManager.createCleanupRequest(new SingularityRequestCleanup(user, RequestCleanupType.INCREMENTAL_BOUNCE, System.currentTimeMillis(), Optional.<Boolean>absent(), Optional.absent(), requestId, Optional.of(firstDeployId), Optional.<Boolean>absent(), Optional.<String>absent(), Optional.<String>absent(), Optional.<SingularityShellCommand>absent()));
    Assert.assertTrue(requestManager.cleanupRequestExists(requestId));
    cleaner.drainCleanupQueue();
    Assert.assertTrue(!requestManager.cleanupRequestExists(requestId));
    Assert.assertEquals(2, taskManager.getCleanupTaskIds().size());
    resourceOffers(3);
    SingularityTask taskThree = null;
    for (SingularityTask task : taskManager.getActiveTasks()) {
        if (!task.getTaskId().equals(taskOne.getTaskId()) && !task.getTaskId().equals(taskTwo.getTaskId())) {
            taskThree = task;
        }
    }
    statusUpdate(taskThree, TaskState.TASK_RUNNING, Optional.of(1L));
    Assert.assertEquals(3, taskManager.getActiveTaskIds().size());
    cleaner.drainCleanupQueue();
    // No old tasks should be killed before new ones pass healthchecks
    Assert.assertEquals(2, taskManager.getCleanupTaskIds().size());
    taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.of(200), Optional.of(1000L), System.currentTimeMillis(), Optional.<String>absent(), Optional.<String>absent(), taskThree.getTaskId(), Optional.<Boolean>absent()));
    cleaner.drainCleanupQueue();
    Assert.assertEquals(1, taskManager.getCleanupTaskIds().size());
    statusUpdate(taskOne, TaskState.TASK_KILLED);
    resourceOffers(3);
    SingularityTask taskFour = null;
    for (SingularityTask task : taskManager.getActiveTasks()) {
        if (!task.getTaskId().equals(taskOne.getTaskId()) && !task.getTaskId().equals(taskTwo.getTaskId()) && !task.getTaskId().equals(taskThree.getTaskId())) {
            taskFour = task;
        }
    }
    statusUpdate(taskFour, TaskState.TASK_RUNNING, Optional.of(1L));
    taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.of(200), Optional.of(1000L), System.currentTimeMillis(), Optional.<String>absent(), Optional.<String>absent(), taskFour.getTaskId(), Optional.<Boolean>absent()));
    cleaner.drainCleanupQueue();
    Assert.assertTrue(taskManager.getCleanupTaskIds().isEmpty());
}

Example 3 with SingularityRequestCleanup

use of com.hubspot.singularity.SingularityRequestCleanup in project Singularity by HubSpot.

the class SingularitySchedulerTest method badPauseExpires.

@Test
public void badPauseExpires() {
    initRequest();
    requestManager.createCleanupRequest(new SingularityRequestCleanup(Optional.<String>absent(), RequestCleanupType.PAUSING, System.currentTimeMillis(), Optional.<Boolean>absent(), Optional.absent(), requestId, Optional.<String>absent(), Optional.<Boolean>absent(), Optional.<String>absent(), Optional.<String>absent(), Optional.<SingularityShellCommand>absent()));
    cleaner.drainCleanupQueue();
    Assert.assertTrue(!requestManager.getCleanupRequests().isEmpty());
    configuration.setCleanupEverySeconds(0);
    sleep(1);
    cleaner.drainCleanupQueue();
    Assert.assertTrue(requestManager.getCleanupRequests().isEmpty());
}

Example 4 with SingularityRequestCleanup

use of com.hubspot.singularity.SingularityRequestCleanup in project Singularity by HubSpot.

the class RequestResource method bounce.

public SingularityRequestParent bounce(String requestId, Optional<SingularityBounceRequest> bounceRequest, SingularityUser user) {
    SingularityRequestWithState requestWithState = fetchRequestWithState(requestId, user);
    authorizationHelper.checkForAuthorization(requestWithState.getRequest(), user, SingularityAuthorizationScope.WRITE);
    validator.checkActionEnabled(SingularityAction.BOUNCE_REQUEST);
    checkBadRequest(requestWithState.getRequest().isLongRunning(), "Can not bounce a %s request (%s)", requestWithState.getRequest().getRequestType(), requestWithState);
    checkConflict(requestWithState.getState() != RequestState.PAUSED, "Request %s is paused. Unable to bounce (it must be manually unpaused first)", requestWithState.getRequest().getId());
    final boolean isIncrementalBounce = bounceRequest.isPresent() && bounceRequest.get().getIncremental().or(false);
    validator.checkResourcesForBounce(requestWithState.getRequest(), isIncrementalBounce);
    validator.checkRequestForPriorityFreeze(requestWithState.getRequest());
    final Optional<Boolean> skipHealthchecks = bounceRequest.isPresent() ? bounceRequest.get().getSkipHealthchecks() : Optional.<Boolean>absent();
    Optional<String> message = Optional.absent();
    Optional<String> actionId = Optional.absent();
    Optional<SingularityShellCommand> runBeforeKill = Optional.absent();
    if (bounceRequest.isPresent()) {
        actionId = bounceRequest.get().getActionId();
        message = bounceRequest.get().getMessage();
        if (bounceRequest.get().getRunShellCommandBeforeKill().isPresent()) {
            validator.checkValidShellCommand(bounceRequest.get().getRunShellCommandBeforeKill().get());
            runBeforeKill = bounceRequest.get().getRunShellCommandBeforeKill();
        }
    }
    if (!actionId.isPresent()) {
        actionId = Optional.of(UUID.randomUUID().toString());
    }
    final String deployId = getAndCheckDeployId(requestId);
    checkConflict(!(requestManager.markAsBouncing(requestId) == SingularityCreateResult.EXISTED), "%s is already bouncing", requestId);
    requestManager.createCleanupRequest(new SingularityRequestCleanup(user.getEmail(), isIncrementalBounce ? RequestCleanupType.INCREMENTAL_BOUNCE : RequestCleanupType.BOUNCE, System.currentTimeMillis(), Optional.absent(), Optional.absent(), requestId, Optional.of(deployId), skipHealthchecks, message, actionId, runBeforeKill));
    requestManager.bounce(requestWithState.getRequest(), System.currentTimeMillis(), Optional.of(user.getId()), message);
    final SingularityBounceRequest validatedBounceRequest = validator.checkBounceRequest(bounceRequest.or(SingularityBounceRequest.defaultRequest()));
    requestManager.saveExpiringObject(new SingularityExpiringBounce(requestId, deployId, Optional.of(user.getId()), System.currentTimeMillis(), validatedBounceRequest, actionId.get()));
    return fillEntireRequest(requestWithState);
}

Example 5 with SingularityRequestCleanup

use of com.hubspot.singularity.SingularityRequestCleanup in project Singularity by HubSpot.

the class SingularityCleaner method drainRequestCleanupQueue.

private void drainRequestCleanupQueue() {
    final long start = System.currentTimeMillis();
    final List<SingularityRequestCleanup> cleanupRequests = requestManager.getCleanupRequests();
    if (cleanupRequests.isEmpty()) {
        LOG.trace("Request cleanup queue is empty");
        return;
    }
    LOG.info("Cleaning up {} requests", cleanupRequests.size());
    AtomicInteger numTasksKilled = new AtomicInteger(0);
    AtomicInteger numScheduledTasksRemoved = new AtomicInteger(0);
    cleanupRequests.parallelStream().forEach((requestCleanup) -> {
        lock.runWithRequestLock(() -> {
            processRequestCleanup(start, numTasksKilled, numScheduledTasksRemoved, requestCleanup);
        }, requestCleanup.getRequestId(), String.format("%s#%s", getClass().getSimpleName(), "drainRequestCleanupQueue"));
    });
    LOG.info("Killed {} tasks (removed {} scheduled) in {}", numTasksKilled.get(), numScheduledTasksRemoved.get(), JavaUtils.duration(start));
}