Examples with SingularityPendingDeploy com.hubspot.singularity.SingularityPendingDeploy used on opensource projects

Example 16 with SingularityPendingDeploy

use of com.hubspot.singularity.SingularityPendingDeploy in project Singularity by HubSpot.

the class SingularityDeployChecker method enqueueLbRevertToActiveTasks.

private SingularityDeployResult enqueueLbRevertToActiveTasks(SingularityRequest request, SingularityPendingDeploy pendingDeploy, SingularityDeployProgress updatedProgress, Collection<SingularityTaskId> deployActiveTasks, Collection<SingularityTaskId> otherActiveTasks) {
    Optional<String> activeDeployIdToRevertTo = deployManager.getActiveDeployId(request.getId());
    if (!activeDeployIdToRevertTo.isPresent()) {
        return new SingularityDeployResult(DeployState.FAILED_INTERNAL_STATE, "No active deploy id  found to revert to");
    }
    Optional<SingularityDeploy> activeDeployToRevertTo = deployManager.getDeploy(request.getId(), activeDeployIdToRevertTo.get());
    if (!activeDeployToRevertTo.isPresent()) {
        return new SingularityDeployResult(DeployState.FAILED_INTERNAL_STATE, "No active deploy found to revert to");
    }
    SingularityDeploy toRevertTo = activeDeployToRevertTo.get();
    Set<SingularityTaskId> toAddBack = otherActiveTasks.stream().filter(t -> t.getDeployId().equals(toRevertTo.getId())).collect(Collectors.toSet());
    LOG.info("Attempting to roll back load balancer to previous active tasks");
    // this to make it as if the task was a net-new add to the LB, making cleanup act as before
    for (SingularityTaskId toAddId : toAddBack) {
        taskManager.clearLoadBalancerHistory(toAddId);
    }
    Map<SingularityTaskId, SingularityTask> tasks = taskManager.getTasks(Iterables.concat(deployActiveTasks, toAddBack));
    final LoadBalancerRequestId lbRequestId = SingularityDeployCheckHelper.getRequestIdForRevert(request.getId(), toRevertTo.getId());
    SingularityLoadBalancerUpdate enqueueResult = lbClient.enqueue(lbRequestId, request, toRevertTo, SingularityDeployCheckHelper.getTasks(toAddBack, tasks), SingularityDeployCheckHelper.getTasks(deployActiveTasks, tasks));
    SingularityDeployProgress deployProgress = updatedProgress.withPendingLbUpdate(enqueueResult, toAddBack, deployActiveTasks, true);
    updatePendingDeploy(pendingDeploy, DeployState.WAITING, deployProgress);
    return new SingularityDeployResult(DeployState.WAITING);
}

Example 17 with SingularityPendingDeploy

use of com.hubspot.singularity.SingularityPendingDeploy in project Singularity by HubSpot.

the class StateManager method generateState.

public SingularityState generateState(boolean includeRequestIds) {
    final int launchingTasks = taskManager.getNumLaunchingTasks();
    final int activeTasks = taskManager.getNumActiveTasks() - launchingTasks;
    final int scheduledTasks = taskManager.getNumScheduledTasks();
    final int cleaningTasks = taskManager.getNumCleanupTasks();
    final int lbCleanupTasks = taskManager.getNumLbCleanupTasks();
    final int lbCleanupRequests = requestManager.getNumLbCleanupRequests();
    final SingularityScheduledTasksInfo scheduledTasksInfo = getScheduledTasksInfo();
    final List<String> overProvisionedRequestIds = new ArrayList<>();
    final Set<String> possiblyUnderProvisionedRequestIds = new HashSet<>();
    final List<SingularityRequestWithState> requests = requestManager.getRequests();
    final Map<String, Long> numInstances = getNumTasks(requests);
    int numActiveRequests = 0;
    int numPausedRequests = 0;
    int cooldownRequests = 0;
    int numFinishedRequests = 0;
    for (SingularityRequestWithState requestWithState : requests) {
        switch(requestWithState.getState()) {
            case DEPLOYING_TO_UNPAUSE:
            case ACTIVE:
                numActiveRequests++;
                break;
            case FINISHED:
                numFinishedRequests++;
                break;
            case PAUSED:
                numPausedRequests++;
                break;
            case SYSTEM_COOLDOWN:
                cooldownRequests++;
                break;
            case DELETED:
                break;
        }
        updatePossiblyUnderProvisionedAndOverProvisionedIds(requestWithState, numInstances, overProvisionedRequestIds, possiblyUnderProvisionedRequestIds);
    }
    filterForPendingRequests(possiblyUnderProvisionedRequestIds);
    final List<String> underProvisionedRequestIds = getUnderProvisionedRequestIds(possiblyUnderProvisionedRequestIds);
    final int pendingRequests = requestManager.getSizeOfPendingQueue();
    final int cleaningRequests = requestManager.getSizeOfCleanupQueue();
    List<SingularityRack> racks = rackManager.getObjects();
    int activeRacks = 0;
    int deadRacks = 0;
    int decommissioningRacks = 0;
    int unknownRacks = 0;
    for (SingularityRack rack : racks) {
        switch(rack.getCurrentState().getState()) {
            case ACTIVE:
                activeRacks++;
                break;
            case DEAD:
                deadRacks++;
                break;
            case MISSING_ON_STARTUP:
                unknownRacks++;
                break;
            case DECOMMISSIONED:
            case STARTING_DECOMMISSION:
            case DECOMMISSIONING:
                decommissioningRacks++;
                break;
            default:
                unknownRacks++;
                break;
        }
    }
    List<SingularityAgent> slaves = agentManager.getObjects();
    int activeSlaves = 0;
    int deadSlaves = 0;
    int decommissioningSlaves = 0;
    int unknownSlaves = 0;
    for (SingularityAgent slave : slaves) {
        switch(slave.getCurrentState().getState()) {
            case ACTIVE:
                activeSlaves++;
                break;
            case DEAD:
                deadSlaves++;
                break;
            case MISSING_ON_STARTUP:
                unknownSlaves++;
                break;
            case DECOMMISSIONED:
            case STARTING_DECOMMISSION:
            case DECOMMISSIONING:
                decommissioningSlaves++;
                break;
            default:
                unknownSlaves++;
                break;
        }
    }
    final List<SingularityHostState> states = getHostStates();
    int numDeploys = 0;
    long oldestDeploy = 0;
    long oldestDeployStep = 0;
    List<SingularityDeployMarker> activeDeploys = new ArrayList<>();
    final long now = System.currentTimeMillis();
    for (SingularityPendingDeploy pendingDeploy : deployManager.getPendingDeploys()) {
        activeDeploys.add(pendingDeploy.getDeployMarker());
        if (!pendingDeploy.getDeployProgress().isStepLaunchComplete()) {
            long deployStepDelta = now - pendingDeploy.getDeployProgress().getTimestamp();
            if (deployStepDelta > oldestDeployStep) {
                oldestDeployStep = deployStepDelta;
            }
        }
        long delta = now - pendingDeploy.getDeployMarker().getTimestamp();
        if (delta > oldestDeploy) {
            oldestDeploy = delta;
        }
        numDeploys++;
    }
    final Optional<Boolean> authDatastoreHealthy = authDatastore.isHealthy();
    final Optional<Double> minimumPriorityLevel = getMinimumPriorityLevel();
    return new SingularityState(activeTasks, launchingTasks, numActiveRequests, cooldownRequests, numPausedRequests, scheduledTasks, pendingRequests, lbCleanupTasks, lbCleanupRequests, cleaningRequests, activeSlaves, deadSlaves, decommissioningSlaves, activeRacks, deadRacks, decommissioningRacks, cleaningTasks, states, oldestDeploy, numDeploys, oldestDeployStep, activeDeploys, scheduledTasksInfo.getLateTasks().size(), scheduledTasksInfo.getLateTasks(), scheduledTasksInfo.getOnDemandLateTasks().size(), scheduledTasksInfo.getOnDemandLateTasks(), scheduledTasksInfo.getNumFutureTasks(), scheduledTasksInfo.getMaxTaskLag(), System.currentTimeMillis(), includeRequestIds ? overProvisionedRequestIds : null, includeRequestIds ? underProvisionedRequestIds : null, overProvisionedRequestIds.size(), underProvisionedRequestIds.size(), numFinishedRequests, unknownRacks, unknownSlaves, authDatastoreHealthy, minimumPriorityLevel, (long) statusUpdateDeltas.getSnapshot().getMean(), lastHeartbeatTime.get(), disasterManager.getFireAlarm());
}

Example 18 with SingularityPendingDeploy

use of com.hubspot.singularity.SingularityPendingDeploy in project Singularity by HubSpot.

the class TaskRequestManager method getTaskRequests.

public List<SingularityTaskRequest> getTaskRequests(List<SingularityPendingTask> tasks) {
    final Multimap<String, SingularityPendingTask> requestIdToPendingTaskId = ArrayListMultimap.create(tasks.size(), 1);
    for (SingularityPendingTask task : tasks) {
        requestIdToPendingTaskId.put(task.getPendingTaskId().getRequestId(), task);
    }
    final List<SingularityRequestWithState> matchingRequests = requestManager.getRequests(requestIdToPendingTaskId.keySet());
    final Map<SingularityPendingTask, SingularityDeployKey> deployKeys = SingularityDeployKey.fromPendingTasks(requestIdToPendingTaskId.values());
    final Map<SingularityDeployKey, SingularityDeploy> matchingDeploys = deployManager.getDeploysForKeys(Sets.newHashSet(deployKeys.values()));
    final List<SingularityTaskRequest> taskRequests = Lists.newArrayListWithCapacity(matchingRequests.size());
    for (SingularityRequestWithState request : matchingRequests) {
        Optional<SingularityPendingDeploy> maybePendingDeploy = deployManager.getPendingDeploy(request.getRequest().getId());
        for (SingularityPendingTask task : requestIdToPendingTaskId.get(request.getRequest().getId())) {
            SingularityDeploy foundDeploy = matchingDeploys.get(deployKeys.get(task));
            if (foundDeploy == null) {
                LOG.warn("Couldn't find a matching deploy for pending task {}", task);
                continue;
            }
            if (!request.getState().isRunnable()) {
                LOG.warn("Request was in state {} for pending task {}", request.getState(), task);
                continue;
            }
            Optional<SingularityRequest> updatedRequest = maybePendingDeploy.isPresent() && maybePendingDeploy.get().getDeployMarker().getDeployId().equals(task.getPendingTaskId().getDeployId()) ? maybePendingDeploy.get().getUpdatedRequest() : Optional.<SingularityRequest>empty();
            taskRequests.add(new SingularityTaskRequest(updatedRequest.orElse(request.getRequest()), foundDeploy, task));
        }
    }
    return taskRequests;
}

Example 19 with SingularityPendingDeploy

use of com.hubspot.singularity.SingularityPendingDeploy in project Singularity by HubSpot.

the class SingularityStartup method enqueueHealthAndNewTaskChecks.

private List<CompletableFuture<Void>> enqueueHealthAndNewTaskChecks(ExecutorService startupExecutor) {
    final List<SingularityTask> activeTasks = taskManager.getActiveTasks();
    final Map<SingularityTaskId, SingularityTask> activeTaskMap = Maps.uniqueIndex(activeTasks, SingularityTaskIdHolder.getTaskIdFunction());
    final Map<SingularityTaskId, List<SingularityTaskHistoryUpdate>> taskUpdates = taskManager.getTaskHistoryUpdates(activeTaskMap.keySet());
    final Map<SingularityDeployKey, SingularityPendingDeploy> pendingDeploys = Maps.uniqueIndex(deployManager.getPendingDeploys(), SingularityDeployKey.FROM_PENDING_TO_DEPLOY_KEY::apply);
    final Map<String, SingularityRequestWithState> idToRequest = Maps.uniqueIndex(requestManager.getRequests(), SingularityRequestWithState.REQUEST_STATE_TO_REQUEST_ID::apply);
    AtomicInteger enqueuedNewTaskChecks = new AtomicInteger(0);
    AtomicInteger enqueuedHealthchecks = new AtomicInteger(0);
    List<CompletableFuture<Void>> enqueueFutures = new ArrayList<>();
    for (Entry<SingularityTaskId, SingularityTask> entry : activeTaskMap.entrySet()) {
        enqueueFutures.add(CompletableFuture.runAsync(() -> {
            SingularityTaskId taskId = entry.getKey();
            SingularityTask task = entry.getValue();
            SimplifiedTaskState simplifiedTaskState = SingularityTaskHistoryUpdate.getCurrentState(taskUpdates.get(taskId));
            if (simplifiedTaskState != SimplifiedTaskState.DONE) {
                SingularityDeployKey deployKey = new SingularityDeployKey(taskId.getRequestId(), taskId.getDeployId());
                Optional<SingularityPendingDeploy> pendingDeploy = Optional.ofNullable(pendingDeploys.get(deployKey));
                Optional<SingularityRequestWithState> request = Optional.ofNullable(idToRequest.get(taskId.getRequestId()));
                if (!pendingDeploy.isPresent()) {
                    newTaskChecker.enqueueNewTaskCheck(task, request, healthchecker);
                    enqueuedNewTaskChecks.getAndIncrement();
                }
                if (simplifiedTaskState == SimplifiedTaskState.RUNNING) {
                    if (healthchecker.enqueueHealthcheck(task, pendingDeploy, request)) {
                        enqueuedHealthchecks.getAndIncrement();
                    }
                }
            }
        }, startupExecutor));
    }
    LOG.info("Enqueued {} health checks and {} new task checks (out of {} active tasks)", enqueuedHealthchecks.get(), enqueuedNewTaskChecks.get(), activeTasks.size());
    return enqueueFutures;
}