Examples with SingularityDeployKey com.hubspot.singularity.SingularityDeployKey used on opensource projects

Example 6 with SingularityDeployKey

use of com.hubspot.singularity.SingularityDeployKey in project Singularity by HubSpot.

the class SingularityDeployChecker method checkDeploy.

private void checkDeploy(final SingularityPendingDeploy pendingDeploy, final List<SingularityDeployMarker> cancelDeploys, final Map<SingularityPendingDeploy, SingularityDeployKey> pendingDeployToKey, final Map<SingularityDeployKey, SingularityDeploy> deployKeyToDeploy, List<SingularityUpdatePendingDeployRequest> updateRequests) {
    final SingularityDeployKey deployKey = pendingDeployToKey.get(pendingDeploy);
    final Optional<SingularityDeploy> deploy = Optional.fromNullable(deployKeyToDeploy.get(deployKey));
    Optional<SingularityRequestWithState> maybeRequestWithState = requestManager.getRequest(pendingDeploy.getDeployMarker().getRequestId());
    if (!(maybeRequestWithState.isPresent() && maybeRequestWithState.get().getState() == RequestState.FINISHED) && !(configuration.isAllowDeployOfPausedRequests() && maybeRequestWithState.isPresent() && maybeRequestWithState.get().getState() == RequestState.PAUSED) && !SingularityRequestWithState.isActive(maybeRequestWithState)) {
        LOG.warn("Deploy {} request was {}, removing deploy", pendingDeploy, SingularityRequestWithState.getRequestState(maybeRequestWithState));
        if (shouldCancelLoadBalancer(pendingDeploy)) {
            cancelLoadBalancer(pendingDeploy, SingularityDeployFailure.deployRemoved());
        }
        failPendingDeployDueToState(pendingDeploy, maybeRequestWithState, deploy);
        return;
    }
    final SingularityDeployMarker pendingDeployMarker = pendingDeploy.getDeployMarker();
    final Optional<SingularityDeployMarker> cancelRequest = findCancel(cancelDeploys, pendingDeployMarker);
    final Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest = findUpdateRequest(updateRequests, pendingDeploy);
    final SingularityRequestWithState requestWithState = maybeRequestWithState.get();
    final SingularityRequest request = pendingDeploy.getUpdatedRequest().or(requestWithState.getRequest());
    final List<SingularityTaskId> requestTasks = taskManager.getTaskIdsForRequest(request.getId());
    final List<SingularityTaskId> activeTasks = taskManager.filterActiveTaskIds(requestTasks);
    final List<SingularityTaskId> inactiveDeployMatchingTasks = new ArrayList<>(requestTasks.size());
    for (SingularityTaskId taskId : requestTasks) {
        if (taskId.getDeployId().equals(pendingDeployMarker.getDeployId()) && !activeTasks.contains(taskId)) {
            inactiveDeployMatchingTasks.add(taskId);
        }
    }
    final List<SingularityTaskId> deployMatchingTasks = new ArrayList<>(activeTasks.size());
    final List<SingularityTaskId> allOtherMatchingTasks = new ArrayList<>(activeTasks.size());
    for (SingularityTaskId taskId : activeTasks) {
        if (taskId.getDeployId().equals(pendingDeployMarker.getDeployId())) {
            deployMatchingTasks.add(taskId);
        } else {
            allOtherMatchingTasks.add(taskId);
        }
    }
    SingularityDeployResult deployResult = getDeployResult(request, requestWithState.getState(), cancelRequest, pendingDeploy, updatePendingDeployRequest, deploy, deployMatchingTasks, allOtherMatchingTasks, inactiveDeployMatchingTasks);
    LOG.info("Deploy {} had result {} after {}", pendingDeployMarker, deployResult, JavaUtils.durationFromMillis(System.currentTimeMillis() - pendingDeployMarker.getTimestamp()));
    if (deployResult.getDeployState() == DeployState.SUCCEEDED) {
        if (saveNewDeployState(pendingDeployMarker, Optional.of(pendingDeployMarker))) {
            if (!(request.getRequestType() == RequestType.RUN_ONCE)) {
                deleteObsoletePendingTasks(pendingDeploy);
            }
            finishDeploy(requestWithState, deploy, pendingDeploy, allOtherMatchingTasks, deployResult);
            return;
        } else {
            LOG.warn("Failing deploy {} because it failed to save deploy state", pendingDeployMarker);
            deployResult = new SingularityDeployResult(DeployState.FAILED_INTERNAL_STATE, Optional.of(String.format("Deploy had state %s but failed to persist it correctly", deployResult.getDeployState())), deployResult.getLbUpdate(), SingularityDeployFailure.failedToSave(), deployResult.getTimestamp());
        }
    } else if (!deployResult.getDeployState().isDeployFinished()) {
        return;
    }
    // success case is handled, handle failure cases:
    saveNewDeployState(pendingDeployMarker, Optional.<SingularityDeployMarker>absent());
    finishDeploy(requestWithState, deploy, pendingDeploy, deployMatchingTasks, deployResult);
}

Example 7 with SingularityDeployKey

use of com.hubspot.singularity.SingularityDeployKey in project Singularity by HubSpot.

the class SingularityScheduler method drainPendingQueue.

@Timed
public void drainPendingQueue() {
    final long start = System.currentTimeMillis();
    final ImmutableList<SingularityPendingRequest> pendingRequests = ImmutableList.copyOf(requestManager.getPendingRequests());
    if (pendingRequests.isEmpty()) {
        LOG.trace("Pending queue was empty");
        return;
    }
    LOG.info("Pending queue had {} requests", pendingRequests.size());
    Map<SingularityDeployKey, List<SingularityPendingRequest>> deployKeyToPendingRequests = pendingRequests.stream().collect(Collectors.groupingBy((request) -> new SingularityDeployKey(request.getRequestId(), request.getDeployId())));
    AtomicInteger totalNewScheduledTasks = new AtomicInteger(0);
    AtomicInteger heldForScheduledActiveTask = new AtomicInteger(0);
    AtomicInteger obsoleteRequests = new AtomicInteger(0);
    deployKeyToPendingRequests.forEach((deployKey, pendingRequestsForDeployKey) -> {
        lock.runWithRequestLock(() -> handlePendingRequestsForDeployKey(obsoleteRequests, heldForScheduledActiveTask, totalNewScheduledTasks, deployKey, pendingRequestsForDeployKey), deployKey.getRequestId(), String.format("%s#%s", getClass().getSimpleName(), "drainPendingQueue"));
    });
    LOG.info("Scheduled {} new tasks ({} obsolete requests, {} held) in {}", totalNewScheduledTasks.get(), obsoleteRequests.get(), heldForScheduledActiveTask.get(), JavaUtils.duration(start));
}

Example 8 with SingularityDeployKey

use of com.hubspot.singularity.SingularityDeployKey in project Singularity by HubSpot.

the class SingularityCleaner method processTaskCleanupsForRequest.

private void processTaskCleanupsForRequest(String requestId, List<SingularityTaskCleanup> cleanupTasks, AtomicInteger killedTasks) {
    final Multiset<SingularityDeployKey> incrementalCleaningTasks = HashMultiset.create(cleanupTasks.size());
    final List<String> taskIdsForDeletedRequest = new ArrayList<>();
    boolean isRequestDeleting = false;
    // TODO - Better check for deleting request state
    final Set<SingularityTaskId> cleaningTasks = new HashSet<>(cleanupTasks.size());
    for (SingularityTaskCleanup cleanupTask : cleanupTasks) {
        cleaningTasks.add(cleanupTask.getTaskId());
        if (isIncrementalDeployCleanup(cleanupTask) || cleanupTask.getCleanupType() == TaskCleanupType.INCREMENTAL_BOUNCE) {
            incrementalCleaningTasks.add(SingularityDeployKey.fromTaskId(cleanupTask.getTaskId()));
        }
        if (cleanupTask.getCleanupType() == TaskCleanupType.REQUEST_DELETING) {
            taskIdsForDeletedRequest.add(cleanupTask.getTaskId().getId());
            isRequestDeleting = true;
        }
    }
    LOG.info("Cleaning up {} tasks for request {}", cleanupTasks.size(), requestId);
    final List<SingularityTaskId> activeTaskIds = taskManager.getActiveTaskIds();
    for (SingularityTaskCleanup cleanupTask : cleanupTasks) {
        SingularityTaskId taskId = cleanupTask.getTaskId();
        if (!isValidTask(cleanupTask)) {
            LOG.info("Couldn't find a matching active task for cleanup task {}, deleting..", cleanupTask);
            taskManager.deleteCleanupTask(taskId.getId());
        } else if (shouldKillTask(cleanupTask, activeTaskIds, cleaningTasks, incrementalCleaningTasks) && checkLBStateAndShouldKillTask(cleanupTask)) {
            scheduler.killAndRecord(taskId, cleanupTask.getCleanupType(), cleanupTask.getUser());
            taskManager.deleteCleanupTask(taskId.getId());
            killedTasks.getAndIncrement();
        }
        cleanupRequestIfNoRemainingTasks(cleanupTask, taskIdsForDeletedRequest, isRequestDeleting);
    }
}

Example 9 with SingularityDeployKey

use of com.hubspot.singularity.SingularityDeployKey in project Singularity by HubSpot.

the class SingularityStartup method getDeployKeyToPendingTaskId.

private Map<SingularityDeployKey, SingularityPendingTaskId> getDeployKeyToPendingTaskId() {
    final List<SingularityPendingTaskId> pendingTaskIds = taskManager.getPendingTaskIds();
    final Map<SingularityDeployKey, SingularityPendingTaskId> deployKeyToPendingTaskId = Maps.newHashMapWithExpectedSize(pendingTaskIds.size());
    for (SingularityPendingTaskId taskId : pendingTaskIds) {
        SingularityDeployKey deployKey = new SingularityDeployKey(taskId.getRequestId(), taskId.getDeployId());
        deployKeyToPendingTaskId.put(deployKey, taskId);
    }
    return deployKeyToPendingTaskId;
}

Example 10 with SingularityDeployKey

use of com.hubspot.singularity.SingularityDeployKey in project Singularity by HubSpot.

the class SingularityStartup method checkSchedulerForInconsistentState.

/**
 * We need to run this check for the various situations where the scheduler could get in an inconsistent state due
 * to a crash/network failure during series of state transactions.
 *
 *  1) Unpausing
 *  2) Launching Task
 */
@VisibleForTesting
void checkSchedulerForInconsistentState() {
    final long now = System.currentTimeMillis();
    final Map<SingularityDeployKey, SingularityPendingTaskId> deployKeyToPendingTaskId = getDeployKeyToPendingTaskId();
    for (SingularityRequestWithState requestWithState : requestManager.getRequests()) {
        switch(requestWithState.getState()) {
            case ACTIVE:
            case SYSTEM_COOLDOWN:
            case DEPLOYING_TO_UNPAUSE:
                checkActiveRequest(requestWithState, deployKeyToPendingTaskId, now);
                break;
            case DELETED:
            case PAUSED:
            case FINISHED:
                break;
        }
    }
}