Examples with SingularityRequest com.hubspot.singularity.SingularityRequest used on opensource projects

Example 46 with SingularityRequest

use of com.hubspot.singularity.SingularityRequest in project Singularity by HubSpot.

the class SingularityJobPoller method runActionOnPoll.

@Override
public void runActionOnPoll() {
    final long now = System.currentTimeMillis();
    final List<SingularityTaskId> activeTaskIds = taskManager.getActiveTaskIds();
    final Set<String> requestIdsToLookup = Sets.newHashSetWithExpectedSize(activeTaskIds.size());
    for (SingularityTaskId taskId : activeTaskIds) {
        requestIdsToLookup.add(taskId.getRequestId());
    }
    final Map<String, SingularityRequestWithState> idToRequest = Maps.uniqueIndex(requestManager.getRequests(requestIdsToLookup), SingularityRequestWithState.REQUEST_STATE_TO_REQUEST_ID);
    for (SingularityTaskId taskId : activeTaskIds) {
        SingularityRequestWithState requestWithState = idToRequest.get(taskId.getRequestId());
        if (requestWithState == null) {
            LOG.warn("Active request not found for task ID {}", taskId);
            continue;
        }
        SingularityRequest request = requestWithState.getRequest();
        if (!request.isLongRunning()) {
            checkForOverdueScheduledJob(now - taskId.getStartedAt(), taskId, request);
            checkTaskExecutionTimeLimit(now, taskId, request);
        }
    }
}

Example 47 with SingularityRequest

use of com.hubspot.singularity.SingularityRequest in project Singularity by HubSpot.

the class SingularityAutoScaleSpreadAllPoller method submitScaleRequest.

private void submitScaleRequest(SingularityRequestWithState oldRequestWithState, Integer newRequestedInstances) {
    SingularityRequest oldRequest = oldRequestWithState.getRequest();
    SingularityRequest newRequest = oldRequest.toBuilder().setInstances(Optional.of((newRequestedInstances))).build();
    Optional<SingularityRequestHistory.RequestHistoryType> historyType = Optional.of(SingularityRequestHistory.RequestHistoryType.SCALED);
    Optional<String> message = Optional.of(String.format("Auto scale number of instances to spread to all %d available slaves", newRequestedInstances));
    requestHelper.updateRequest(newRequest, Optional.of(oldRequest), oldRequestWithState.getState(), historyType, Optional.<String>absent(), oldRequest.getSkipHealthchecks(), message, Optional.<SingularityBounceRequest>absent());
}

Example 48 with SingularityRequest

use of com.hubspot.singularity.SingularityRequest in project Singularity by HubSpot.

the class SingularityCleaner method shouldKillTask.

private boolean shouldKillTask(SingularityTaskCleanup taskCleanup, List<SingularityTaskId> activeTaskIds, Set<SingularityTaskId> cleaningTasks, Multiset<SingularityDeployKey> incrementalCleaningTasks) {
    final Optional<SingularityRequestWithState> requestWithState = requestManager.getRequest(taskCleanup.getTaskId().getRequestId());
    if (!requestWithState.isPresent()) {
        LOG.debug("Killing a task {} immediately because the request was missing", taskCleanup);
        return true;
    }
    final SingularityRequest request = requestWithState.get().getRequest();
    if (taskCleanup.getRunBeforeKillId().isPresent()) {
        List<SingularityTaskShellCommandUpdate> shellCommandUpdates = taskManager.getTaskShellCommandUpdates(taskCleanup.getRunBeforeKillId().get());
        boolean finished = false;
        for (SingularityTaskShellCommandUpdate update : shellCommandUpdates) {
            if (update.getUpdateType().isFinished()) {
                finished = true;
                break;
            }
        }
        if (!finished) {
            LOG.debug("Waiting for pre-kill shell command {} to finish before killing task", taskCleanup.getRunBeforeKillId());
            return false;
        }
    }
    if (taskCleanup.getCleanupType().shouldKillTaskInstantly(request)) {
        LOG.debug("Killing a task {} immediately because of its cleanup type", taskCleanup);
        return true;
    }
    // If pausing, must be a long-running task to kill here
    if (requestWithState.get().getState() == RequestState.PAUSED && (!(taskCleanup.getCleanupType() == TaskCleanupType.PAUSING) || request.isLongRunning())) {
        LOG.debug("Killing a task {} immediately because the request was paused", taskCleanup);
        return true;
    }
    if (!request.isLongRunning()) {
        final long timeSinceCleanup = System.currentTimeMillis() - taskCleanup.getTimestamp();
        final long maxWaitTime = request.getKillOldNonLongRunningTasksAfterMillis().or(killNonLongRunningTasksInCleanupAfterMillis);
        final boolean tooOld = (maxWaitTime < 1) || (timeSinceCleanup > maxWaitTime);
        if (!tooOld) {
            LOG.trace("Not killing a non-longRunning task {}, running time since cleanup {} (max wait time is {})", taskCleanup, timeSinceCleanup, maxWaitTime);
        } else {
            LOG.debug("Killing a non-longRunning task {} - running time since cleanup {} exceeded max wait time {}", taskCleanup, timeSinceCleanup, maxWaitTime);
        }
        return tooOld;
    }
    final String requestId = request.getId();
    final Optional<SingularityRequestDeployState> deployState = deployManager.getRequestDeployState(requestId);
    if (taskCleanup.getCleanupType() == TaskCleanupType.DECOMISSIONING && deployState.get().getPendingDeploy().isPresent() && deployState.get().getPendingDeploy().get().getDeployId().equals(taskCleanup.getTaskId().getDeployId())) {
        final long timeSinceCleanup = System.currentTimeMillis() - taskCleanup.getTimestamp();
        final long maxWaitTime = configuration.getPendingDeployHoldTaskDuringDecommissionMillis();
        final boolean tooOld = (maxWaitTime < 1) || (timeSinceCleanup > maxWaitTime);
        if (!tooOld) {
            LOG.trace("Not killing {} - part of pending deploy - running time since cleanup {} (max wait time is {})", taskCleanup, timeSinceCleanup, maxWaitTime);
            return false;
        } else {
            LOG.debug("Killing {} - part of pending deploy but running time since cleanup {} exceeded max wait time {}", taskCleanup, timeSinceCleanup, maxWaitTime);
            return true;
        }
    }
    if (!deployState.isPresent() || !deployState.get().getActiveDeploy().isPresent()) {
        LOG.debug("Killing a task {} immediately because there is no active deploy state {}", taskCleanup, deployState);
        return true;
    }
    final String activeDeployId = deployState.get().getActiveDeploy().get().getDeployId();
    final String matchingTasksDeployId = taskCleanup.getCleanupType() == TaskCleanupType.INCREMENTAL_DEPLOY_CANCELLED || taskCleanup.getCleanupType() == TaskCleanupType.INCREMENTAL_DEPLOY_FAILED ? activeDeployId : taskCleanup.getTaskId().getDeployId();
    // check to see if there are enough active tasks out there that have been active for long enough that we can safely shut this task down.
    final List<SingularityTaskId> matchingTasks = new ArrayList<>();
    for (SingularityTaskId taskId : activeTaskIds) {
        if (!taskId.getRequestId().equals(requestId) || !taskId.getDeployId().equals(matchingTasksDeployId)) {
            continue;
        }
        if (cleaningTasks.contains(taskId)) {
            continue;
        }
        matchingTasks.add(taskId);
    }
    // For an incremental bounce or incremental deploy cleanup, shut down old tasks as new ones are started
    final SingularityDeployKey key = SingularityDeployKey.fromTaskId(taskCleanup.getTaskId());
    if (taskCleanup.getCleanupType() == TaskCleanupType.INCREMENTAL_BOUNCE) {
        return shouldKillIncrementalBounceTask(request, taskCleanup, matchingTasksDeployId, matchingTasks, key, incrementalCleaningTasks);
    } else if (isIncrementalDeployCleanup(taskCleanup)) {
        return shouldKillIncrementalDeployCleanupTask(request, taskCleanup, matchingTasksDeployId, matchingTasks, key, incrementalCleaningTasks);
    } else {
        if (matchingTasks.size() < request.getInstancesSafe()) {
            LOG.trace("Not killing a task {} yet, only {} matching out of a required {}", taskCleanup, matchingTasks.size(), request.getInstancesSafe());
            return false;
        }
    }
    final Optional<SingularityDeploy> deploy = deployManager.getDeploy(requestId, activeDeployId);
    final DeployHealth deployHealth = deployHealthHelper.getDeployHealth(requestWithState.get().getRequest(), deploy, matchingTasks, false);
    switch(deployHealth) {
        case HEALTHY:
            for (SingularityTaskId taskId : matchingTasks) {
                DeployHealth lbHealth = getLbHealth(request, taskId);
                if (lbHealth != DeployHealth.HEALTHY) {
                    LOG.trace("Not killing a task {}, waiting for new replacement tasks to be added to LB (current state: {})", taskCleanup, lbHealth);
                    return false;
                }
            }
            LOG.debug("Killing a task {}, all replacement tasks are healthy", taskCleanup);
            return true;
        case WAITING:
        case UNHEALTHY:
        default:
            LOG.trace("Not killing a task {}, waiting for new replacement tasks to be healthy (current state: {})", taskCleanup, deployHealth);
            return false;
    }
}

Example 49 with SingularityRequest

use of com.hubspot.singularity.SingularityRequest in project Singularity by HubSpot.

the class StateManagerTest method itDoesntFlagPendingRequestsForUnderOrOverProvisioning.

@Test
public void itDoesntFlagPendingRequestsForUnderOrOverProvisioning() {
    initRequest();
    initFirstDeploy();
    SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest();
    requestManager.activate(request.toBuilder().setInstances(Optional.of(0)).build(), RequestHistoryType.UPDATED, System.currentTimeMillis(), Optional.<String>absent(), Optional.<String>absent());
    requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), firstDeployId, System.currentTimeMillis(), Optional.<String>absent(), PendingType.ONEOFF, Optional.<Boolean>absent(), Optional.<String>absent()));
    Assert.assertEquals(0, taskManager.getActiveTaskIds().size());
    SingularityState state = stateManager.getState(true, false);
    Assert.assertEquals(0, state.getOverProvisionedRequests());
    Assert.assertEquals(0, state.getUnderProvisionedRequests());
}

Example 50 with SingularityRequest

use of com.hubspot.singularity.SingularityRequest in project Singularity by HubSpot.

the class ValidatorTest method whenDeployHasRunNowSetItValidatesThatItIsLessThanACertaionLength.

@Test(expected = WebApplicationException.class)
public void whenDeployHasRunNowSetItValidatesThatItIsLessThanACertaionLength() {
    String requestId = "request";
    String deployID = "deploy";
    SingularityRequest request = new SingularityRequestBuilder(requestId, RequestType.ON_DEMAND).build();
    Optional<SingularityRunNowRequest> runNowRequest = Optional.of(runNowRequest(tooLongId()));
    SingularityDeploy deploy = SingularityDeploy.newBuilder(requestId, deployID).setCommand(Optional.of("printenv")).setRunImmediately(runNowRequest).build();
    validator.checkDeploy(request, deploy, Collections.emptyList(), Collections.emptyList());
}