Examples with SingularityPendingDeploy com.hubspot.singularity.SingularityPendingDeploy used on opensource projects

Example 6 with SingularityPendingDeploy

use of com.hubspot.singularity.SingularityPendingDeploy in project Singularity by HubSpot.

the class SingularityScheduler method handleCompletedTaskWithStatistics.

private Optional<PendingType> handleCompletedTaskWithStatistics(Optional<SingularityTask> task, SingularityTaskId taskId, long timestamp, ExtendedTaskState state, SingularityDeployStatistics deployStatistics, SingularityCreateResult taskHistoryUpdateCreateResult, Protos.TaskStatus status) {
    final Optional<SingularityRequestWithState> maybeRequestWithState = requestManager.getRequest(taskId.getRequestId());
    final Optional<SingularityPendingDeploy> maybePendingDeploy = deployManager.getPendingDeploy(taskId.getRequestId());
    if (!isRequestActive(maybeRequestWithState)) {
        LOG.warn("Not scheduling a new task, {} is {}", taskId.getRequestId(), SingularityRequestWithState.getRequestState(maybeRequestWithState));
        return Optional.absent();
    }
    RequestState requestState = maybeRequestWithState.get().getState();
    final SingularityRequest request = maybePendingDeploy.isPresent() ? maybePendingDeploy.get().getUpdatedRequest().or(maybeRequestWithState.get().getRequest()) : maybeRequestWithState.get().getRequest();
    final Optional<SingularityRequestDeployState> requestDeployState = deployManager.getRequestDeployState(request.getId());
    if (!isDeployInUse(requestDeployState, taskId.getDeployId(), true)) {
        LOG.debug("Task {} completed, but it didn't match active deploy state {} - ignoring", taskId.getId(), requestDeployState);
        return Optional.absent();
    }
    if (taskHistoryUpdateCreateResult == SingularityCreateResult.CREATED && requestState != RequestState.SYSTEM_COOLDOWN) {
        mailer.queueTaskCompletedMail(task, taskId, request, state);
    } else if (requestState == RequestState.SYSTEM_COOLDOWN) {
        LOG.debug("Not sending a task completed email because task {} is in SYSTEM_COOLDOWN", taskId);
    } else {
        LOG.debug("Not sending a task completed email for task {} because Singularity already processed this update", taskId);
    }
    if (!status.hasReason() || !status.getReason().equals(Reason.REASON_INVALID_OFFERS)) {
        if (!state.isSuccess() && taskHistoryUpdateCreateResult == SingularityCreateResult.CREATED && cooldown.shouldEnterCooldown(request, taskId, requestState, deployStatistics, timestamp)) {
            LOG.info("Request {} is entering cooldown due to task {}", request.getId(), taskId);
            requestState = RequestState.SYSTEM_COOLDOWN;
            requestManager.cooldown(request, System.currentTimeMillis());
            mailer.sendRequestInCooldownMail(request);
        }
    } else {
        LOG.debug("Not triggering cooldown due to TASK_LOST from invalid offers for request {}", request.getId());
    }
    PendingType pendingType = PendingType.TASK_DONE;
    Optional<List<String>> cmdLineArgsList = Optional.absent();
    if (!state.isSuccess() && shouldRetryImmediately(request, deployStatistics, task)) {
        LOG.debug("Retrying {} because {}", request.getId(), state);
        pendingType = PendingType.RETRY;
        if (task.isPresent()) {
            cmdLineArgsList = task.get().getTaskRequest().getPendingTask().getCmdLineArgsList();
        }
    } else if (!request.isAlwaysRunning()) {
        return Optional.absent();
    }
    if (state.isSuccess() && requestState == RequestState.SYSTEM_COOLDOWN) {
        // TODO send not cooldown anymore email
        LOG.info("Request {} succeeded a task, removing from cooldown", request.getId());
        requestManager.exitCooldown(request, System.currentTimeMillis(), Optional.<String>absent(), Optional.<String>absent());
    }
    SingularityPendingRequest pendingRequest = new SingularityPendingRequest(request.getId(), requestDeployState.get().getActiveDeploy().get().getDeployId(), System.currentTimeMillis(), Optional.absent(), pendingType, cmdLineArgsList, Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent());
    requestManager.addToPendingQueue(pendingRequest);
    return Optional.of(pendingType);
}

Example 7 with SingularityPendingDeploy

use of com.hubspot.singularity.SingularityPendingDeploy in project Singularity by HubSpot.

the class RequestResource method submitRequest.

private void submitRequest(SingularityRequest request, Optional<SingularityRequestWithState> oldRequestWithState, Optional<RequestHistoryType> historyType, Optional<Boolean> skipHealthchecks, Optional<String> message, Optional<SingularityBounceRequest> maybeBounceRequest, SingularityUser user) {
    checkNotNullBadRequest(request.getId(), "Request must have an id");
    checkConflict(!requestManager.cleanupRequestExists(request.getId()), "Request %s is currently cleaning. Try again after a few moments", request.getId());
    Optional<SingularityPendingDeploy> maybePendingDeploy = deployManager.getPendingDeploy(request.getId());
    checkConflict(!(maybePendingDeploy.isPresent() && maybePendingDeploy.get().getUpdatedRequest().isPresent()), "Request %s has a pending deploy that may change the request data. Try again when the deploy has finished", request.getId());
    Optional<SingularityRequest> oldRequest = oldRequestWithState.isPresent() ? Optional.of(oldRequestWithState.get().getRequest()) : Optional.<SingularityRequest>absent();
    if (oldRequest.isPresent()) {
        authorizationHelper.checkForAuthorization(oldRequest.get(), user, SingularityAuthorizationScope.WRITE);
        authorizationHelper.checkForAuthorizedChanges(request, oldRequest.get(), user);
        validator.checkActionEnabled(SingularityAction.UPDATE_REQUEST);
    } else {
        validator.checkActionEnabled(SingularityAction.CREATE_REQUEST);
    }
    if (request.getSlavePlacement().isPresent() && request.getSlavePlacement().get() == SlavePlacement.SPREAD_ALL_SLAVES) {
        checkBadRequest(validator.isSpreadAllSlavesEnabled(), "You must enabled spread to all slaves in order to use the SPREAD_ALL_SLAVES request type");
        int currentActiveSlaveCount = slaveManager.getNumObjectsAtState(MachineState.ACTIVE);
        request = request.toBuilder().setInstances(Optional.of(currentActiveSlaveCount)).build();
    }
    if (!oldRequest.isPresent() || !(oldRequest.get().getInstancesSafe() == request.getInstancesSafe())) {
        validator.checkScale(request, Optional.<Integer>absent());
    }
    authorizationHelper.checkForAuthorization(request, user, SingularityAuthorizationScope.WRITE);
    RequestState requestState = RequestState.ACTIVE;
    if (oldRequestWithState.isPresent()) {
        requestState = oldRequestWithState.get().getState();
    }
    requestHelper.updateRequest(request, oldRequest, requestState, historyType, user.getEmail(), skipHealthchecks, message, maybeBounceRequest);
}

Example 8 with SingularityPendingDeploy

use of com.hubspot.singularity.SingularityPendingDeploy in project Singularity by HubSpot.

the class SingularityScheduler method handlePendingRequestsForDeployKey.

private void handlePendingRequestsForDeployKey(AtomicInteger obsoleteRequests, AtomicInteger heldForScheduledActiveTask, AtomicInteger totalNewScheduledTasks, SingularityDeployKey deployKey, List<SingularityPendingRequest> pendingRequestsForDeploy) {
    final String requestId = deployKey.getRequestId();
    final Optional<SingularityRequestWithState> maybeRequest = requestManager.getRequest(requestId);
    final SingularityDeployStatistics deployStatistics = getDeployStatistics(deployKey.getRequestId(), deployKey.getDeployId());
    if (!isRequestActive(maybeRequest)) {
        LOG.debug("Pending request {} was obsolete (request {})", requestId, SingularityRequestWithState.getRequestState(maybeRequest));
        obsoleteRequests.getAndIncrement();
        for (SingularityPendingRequest pendingRequest : pendingRequestsForDeploy) {
            requestManager.deletePendingRequest(pendingRequest);
        }
        return;
    }
    SingularityRequestWithState request = maybeRequest.get();
    Optional<SingularityRequestDeployState> maybeRequestDeployState = deployManager.getRequestDeployState(requestId);
    Optional<SingularityPendingDeploy> maybePendingDeploy = deployManager.getPendingDeploy(requestId);
    List<SingularityTaskId> matchingTaskIds = getMatchingTaskIds(request.getRequest(), deployKey);
    List<SingularityPendingRequest> effectivePendingRequests = new ArrayList<>();
    // Things that are closest to now (ie smaller timestamps) should come first in the queue
    pendingRequestsForDeploy.sort(Comparator.comparingLong(SingularityPendingRequest::getTimestamp));
    int scheduledTasks = 0;
    for (SingularityPendingRequest pendingRequest : pendingRequestsForDeploy) {
        final SingularityRequest updatedRequest = updatedRequest(maybePendingDeploy, pendingRequest, request);
        if (!shouldScheduleTasks(updatedRequest, pendingRequest, maybePendingDeploy, maybeRequestDeployState)) {
            LOG.debug("Pending request {} was obsolete (request {})", pendingRequest, SingularityRequestWithState.getRequestState(maybeRequest));
            obsoleteRequests.getAndIncrement();
            requestManager.deletePendingRequest(pendingRequest);
            continue;
        }
        int missingInstances = getNumMissingInstances(matchingTaskIds, updatedRequest, pendingRequest, maybePendingDeploy);
        if (missingInstances == 0 && !matchingTaskIds.isEmpty() && updatedRequest.isScheduled() && pendingRequest.getPendingType() == PendingType.NEW_DEPLOY) {
            LOG.trace("Holding pending request {} because it is scheduled and has an active task", pendingRequest);
            heldForScheduledActiveTask.getAndIncrement();
            continue;
        }
        if (effectivePendingRequests.isEmpty()) {
            effectivePendingRequests.add(pendingRequest);
            RequestState requestState = checkCooldown(request.getState(), request.getRequest(), deployStatistics);
            scheduledTasks += scheduleTasks(request.getRequest(), requestState, deployStatistics, pendingRequest, matchingTaskIds, maybePendingDeploy);
            requestManager.deletePendingRequest(pendingRequest);
        } else if (pendingRequest.getPendingType() == PendingType.IMMEDIATE) {
            effectivePendingRequests.add(pendingRequest);
            RequestState requestState = checkCooldown(request.getState(), request.getRequest(), deployStatistics);
            scheduledTasks += scheduleTasks(request.getRequest(), requestState, deployStatistics, pendingRequest, matchingTaskIds, maybePendingDeploy);
            requestManager.deletePendingRequest(pendingRequest);
        } else if (pendingRequest.getPendingType() == PendingType.ONEOFF) {
            effectivePendingRequests.add(pendingRequest);
            RequestState requestState = checkCooldown(request.getState(), request.getRequest(), deployStatistics);
            scheduledTasks += scheduleTasks(request.getRequest(), requestState, deployStatistics, pendingRequest, matchingTaskIds, maybePendingDeploy);
            requestManager.deletePendingRequest(pendingRequest);
        } else if (updatedRequest.isScheduled() && (pendingRequest.getPendingType() == PendingType.NEW_DEPLOY || pendingRequest.getPendingType() == PendingType.TASK_DONE)) {
            // If we are here, there is already an immediate of run of the scheduled task launched. Drop anything that would
            // leave a second instance of the request in the pending queue.
            requestManager.deletePendingRequest(pendingRequest);
        }
    // Any other subsequent requests are not honored until after the pending queue is cleared.
    }
    totalNewScheduledTasks.getAndAdd(scheduledTasks);
}

Example 9 with SingularityPendingDeploy

use of com.hubspot.singularity.SingularityPendingDeploy in project Singularity by HubSpot.

the class DeployResource method deploy.

public SingularityRequestParent deploy(SingularityDeployRequest deployRequest, SingularityUser user) {
    validator.checkActionEnabled(SingularityAction.DEPLOY);
    SingularityDeploy deploy = deployRequest.getDeploy();
    checkNotNullBadRequest(deploy, "DeployRequest must have a deploy object");
    final Optional<String> deployUser = user.getEmail();
    final String requestId = checkNotNullBadRequest(deploy.getRequestId(), "DeployRequest must have a non-null requestId");
    SingularityRequestWithState requestWithState = fetchRequestWithState(requestId, user);
    authorizationHelper.checkForAuthorization(requestWithState.getRequest(), user, SingularityAuthorizationScope.WRITE);
    SingularityRequest request = requestWithState.getRequest();
    final Optional<SingularityRequest> updatedValidatedRequest;
    if (deployRequest.getUpdatedRequest().isPresent()) {
        authorizationHelper.checkForAuthorizedChanges(deployRequest.getUpdatedRequest().get(), requestWithState.getRequest(), user);
        updatedValidatedRequest = Optional.of(validator.checkSingularityRequest(deployRequest.getUpdatedRequest().get(), Optional.of(requestWithState.getRequest()), Optional.<SingularityDeploy>absent(), Optional.of(deploy)));
    } else {
        updatedValidatedRequest = Optional.absent();
    }
    if (updatedValidatedRequest.isPresent()) {
        request = updatedValidatedRequest.get();
    }
    validator.checkScale(request, Optional.of(taskManager.getActiveTaskIdsForRequest(request.getId()).size()));
    if (!deployRequest.isUnpauseOnSuccessfulDeploy() && !configuration.isAllowDeployOfPausedRequests()) {
        checkConflict(requestWithState.getState() != RequestState.PAUSED, "Request %s is paused. Unable to deploy (it must be manually unpaused first)", requestWithState.getRequest().getId());
    }
    deploy = validator.checkDeploy(request, deploy, taskManager.getActiveTaskIdsForRequest(requestId), taskManager.getPendingTaskIdsForRequest(requestId));
    final long now = System.currentTimeMillis();
    SingularityDeployMarker deployMarker = new SingularityDeployMarker(requestId, deploy.getId(), now, deployUser, deployRequest.getMessage());
    Optional<SingularityDeployProgress> deployProgress = Optional.absent();
    if (request.isLongRunning()) {
        deployProgress = Optional.of(new SingularityDeployProgress(Math.min(deploy.getDeployInstanceCountPerStep().or(request.getInstancesSafe()), request.getInstancesSafe()), 0, deploy.getDeployInstanceCountPerStep().or(request.getInstancesSafe()), deploy.getDeployStepWaitTimeMs().or(configuration.getDefaultDeployStepWaitTimeMs()), false, deploy.getAutoAdvanceDeploySteps().or(true), Collections.emptySet(), System.currentTimeMillis()));
    }
    SingularityPendingDeploy pendingDeployObj = new SingularityPendingDeploy(deployMarker, Optional.<SingularityLoadBalancerUpdate>absent(), DeployState.WAITING, deployProgress, updatedValidatedRequest);
    boolean deployToUnpause = false;
    if (requestWithState.getState() == RequestState.PAUSED && deployRequest.isUnpauseOnSuccessfulDeploy()) {
        deployToUnpause = true;
        requestManager.deployToUnpause(request, now, deployUser, deployRequest.getMessage());
    }
    boolean deployAlreadyInProgress = deployManager.createPendingDeploy(pendingDeployObj) == SingularityCreateResult.EXISTED;
    if (deployAlreadyInProgress && deployToUnpause) {
        requestManager.pause(request, now, deployUser, Optional.absent());
    }
    checkConflict(!deployAlreadyInProgress, "Pending deploy already in progress for %s - cancel it or wait for it to complete (%s)", requestId, deployManager.getPendingDeploy(requestId).orNull());
    deployManager.saveDeploy(request, deployMarker, deploy);
    if (request.isDeployable() && !(requestWithState.getState() == RequestState.PAUSED && configuration.isAllowDeployOfPausedRequests())) {
        requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, deployMarker.getDeployId(), now, deployUser, PendingType.NEW_DEPLOY, deployRequest.getDeploy().getSkipHealthchecksOnDeploy(), deployRequest.getMessage()));
    }
    return fillEntireRequest(requestWithState, Optional.of(request));
}

Example 10 with SingularityPendingDeploy

use of com.hubspot.singularity.SingularityPendingDeploy in project Singularity by HubSpot.

the class StateManager method generateState.

public SingularityState generateState(boolean includeRequestIds) {
    final int launchingTasks = taskManager.getNumLaunchingTasks();
    final int activeTasks = taskManager.getNumActiveTasks() - launchingTasks;
    final int scheduledTasks = taskManager.getNumScheduledTasks();
    final int cleaningTasks = taskManager.getNumCleanupTasks();
    final int lbCleanupTasks = taskManager.getNumLbCleanupTasks();
    final int lbCleanupRequests = requestManager.getNumLbCleanupRequests();
    final SingularityScheduledTasksInfo scheduledTasksInfo = SingularityScheduledTasksInfo.getInfo(taskManager.getPendingTasks(), singularityConfiguration.getDeltaAfterWhichTasksAreLateMillis());
    final List<String> overProvisionedRequestIds = new ArrayList<>();
    final Set<String> possiblyUnderProvisionedRequestIds = new HashSet<>();
    final List<SingularityRequestWithState> requests = requestManager.getRequests();
    final Map<String, Long> numInstances = getNumTasks(requests);
    int numActiveRequests = 0;
    int numPausedRequests = 0;
    int cooldownRequests = 0;
    int numFinishedRequests = 0;
    for (SingularityRequestWithState requestWithState : requests) {
        switch(requestWithState.getState()) {
            case DEPLOYING_TO_UNPAUSE:
            case ACTIVE:
                numActiveRequests++;
                break;
            case FINISHED:
                numFinishedRequests++;
                break;
            case PAUSED:
                numPausedRequests++;
                break;
            case SYSTEM_COOLDOWN:
                cooldownRequests++;
                break;
            case DELETED:
                break;
        }
        updatePossiblyUnderProvisionedAndOverProvisionedIds(requestWithState, numInstances, overProvisionedRequestIds, possiblyUnderProvisionedRequestIds);
    }
    filterForPendingRequests(possiblyUnderProvisionedRequestIds);
    final List<String> underProvisionedRequestIds = getUnderProvisionedRequestIds(possiblyUnderProvisionedRequestIds);
    final int pendingRequests = requestManager.getSizeOfPendingQueue();
    final int cleaningRequests = requestManager.getSizeOfCleanupQueue();
    List<SingularityRack> racks = rackManager.getObjects();
    int activeRacks = 0;
    int deadRacks = 0;
    int decommissioningRacks = 0;
    int unknownRacks = 0;
    for (SingularityRack rack : racks) {
        switch(rack.getCurrentState().getState()) {
            case ACTIVE:
                activeRacks++;
                break;
            case DEAD:
                deadRacks++;
                break;
            case MISSING_ON_STARTUP:
                unknownRacks++;
                break;
            case DECOMMISSIONED:
            case STARTING_DECOMMISSION:
            case DECOMMISSIONING:
                decommissioningRacks++;
                break;
            default:
                unknownRacks++;
                break;
        }
    }
    List<SingularitySlave> slaves = slaveManager.getObjects();
    int activeSlaves = 0;
    int deadSlaves = 0;
    int decommissioningSlaves = 0;
    int unknownSlaves = 0;
    for (SingularitySlave slave : slaves) {
        switch(slave.getCurrentState().getState()) {
            case ACTIVE:
                activeSlaves++;
                break;
            case DEAD:
                deadSlaves++;
                break;
            case MISSING_ON_STARTUP:
                unknownSlaves++;
                break;
            case DECOMMISSIONED:
            case STARTING_DECOMMISSION:
            case DECOMMISSIONING:
                decommissioningSlaves++;
                break;
            default:
                unknownSlaves++;
                break;
        }
    }
    final List<SingularityHostState> states = getHostStates();
    int numDeploys = 0;
    long oldestDeploy = 0;
    long oldestDeployStep = 0;
    List<SingularityDeployMarker> activeDeploys = new ArrayList<>();
    final long now = System.currentTimeMillis();
    for (SingularityPendingDeploy pendingDeploy : deployManager.getPendingDeploys()) {
        activeDeploys.add(pendingDeploy.getDeployMarker());
        if (pendingDeploy.getDeployProgress().isPresent() && !pendingDeploy.getDeployProgress().get().isStepComplete()) {
            long deployStepDelta = now - pendingDeploy.getDeployProgress().get().getTimestamp();
            if (deployStepDelta > oldestDeployStep) {
                oldestDeployStep = deployStepDelta;
            }
        }
        long delta = now - pendingDeploy.getDeployMarker().getTimestamp();
        if (delta > oldestDeploy) {
            oldestDeploy = delta;
        }
        numDeploys++;
    }
    final Optional<Boolean> authDatastoreHealthy = authDatastore.isHealthy();
    final Optional<Double> minimumPriorityLevel = getMinimumPriorityLevel();
    return new SingularityState(activeTasks, launchingTasks, numActiveRequests, cooldownRequests, numPausedRequests, scheduledTasks, pendingRequests, lbCleanupTasks, lbCleanupRequests, cleaningRequests, activeSlaves, deadSlaves, decommissioningSlaves, activeRacks, deadRacks, decommissioningRacks, cleaningTasks, states, oldestDeploy, numDeploys, oldestDeployStep, activeDeploys, scheduledTasksInfo.getNumLateTasks(), scheduledTasksInfo.getNumFutureTasks(), scheduledTasksInfo.getMaxTaskLag(), System.currentTimeMillis(), includeRequestIds ? overProvisionedRequestIds : null, includeRequestIds ? underProvisionedRequestIds : null, overProvisionedRequestIds.size(), underProvisionedRequestIds.size(), numFinishedRequests, unknownRacks, unknownSlaves, authDatastoreHealthy, minimumPriorityLevel, statusUpdateDeltaAvg.get());
}