Examples with AgentPlacement com.hubspot.singularity.AgentPlacement used on opensource projects

Example 1 with AgentPlacement

use of com.hubspot.singularity.AgentPlacement in project Singularity by HubSpot.

the class SingularityAutoScaleSpreadAllPoller method runActionOnPoll.

@Override
public void runActionOnPoll() {
    int currentActiveSlaveCount = agentManager.getNumObjectsAtState(MachineState.ACTIVE);
    for (SingularityRequestWithState requestWithState : requestManager.getActiveRequests()) {
        lock.runWithRequestLock(() -> {
            SingularityRequest request = requestWithState.getRequest();
            // global override not supported here
            AgentPlacement placement = request.getAgentPlacement().orElse(defaultAgentPlacement);
            if (placement != AgentPlacement.SPREAD_ALL_SLAVES && placement != AgentPlacement.SPREAD_ALL_AGENTS) {
                return;
            }
            int requestInstanceCount = request.getInstancesSafe();
            if (requestInstanceCount == currentActiveSlaveCount) {
                LOG.trace("Active Request {} is already spread to all {} available slaves", request.getId(), currentActiveSlaveCount);
            } else {
                LOG.info("Scaling request {} from {} instances to {} available slaves", request.getId(), requestInstanceCount, currentActiveSlaveCount);
                submitScaleRequest(requestWithState, currentActiveSlaveCount);
            }
        }, requestWithState.getRequest().getId(), getClass().getSimpleName(), SingularitySchedulerLock.Priority.LOW);
    }
}

Example 2 with AgentPlacement

use of com.hubspot.singularity.AgentPlacement in project Singularity by HubSpot.

the class SingularityAgentAndRackManager method doesOfferMatch.

AgentMatchState doesOfferMatch(SingularityOfferHolder offerHolder, SingularityTaskRequest taskRequest, List<SingularityTaskId> activeTaskIdsForRequest, boolean isPreemptibleTask, RequestUtilization requestUtilization) {
    final String host = offerHolder.getHostname();
    final String rackId = offerHolder.getRackId();
    final String agentId = offerHolder.getAgentId();
    Optional<SingularityAgent> maybeAgent = agentManager.getObject(agentId);
    if (!maybeAgent.isPresent()) {
        return AgentMatchState.RESOURCES_DO_NOT_MATCH;
    }
    final MachineState currentState = maybeAgent.get().getCurrentState().getState();
    if (currentState == MachineState.FROZEN) {
        return AgentMatchState.AGENT_FROZEN;
    }
    if (currentState.isDecommissioning()) {
        return AgentMatchState.AGENT_DECOMMISSIONING;
    }
    final MachineState currentRackState = rackManager.getObject(rackId).get().getCurrentState().getState();
    if (currentRackState == MachineState.FROZEN) {
        return AgentMatchState.RACK_FROZEN;
    }
    if (currentRackState.isDecommissioning()) {
        return AgentMatchState.RACK_DECOMMISSIONING;
    }
    if (!taskRequest.getRequest().getRackAffinity().orElse(Collections.emptyList()).isEmpty()) {
        if (!taskRequest.getRequest().getRackAffinity().get().contains(rackId)) {
            LOG.trace("Task {} requires a rack in {} (current rack {})", taskRequest.getPendingTask().getPendingTaskId(), taskRequest.getRequest().getRackAffinity().get(), rackId);
            return AgentMatchState.RACK_AFFINITY_NOT_MATCHING;
        }
    }
    if (!isAttributesMatch(offerHolder, taskRequest, isPreemptibleTask)) {
        return AgentMatchState.AGENT_ATTRIBUTES_DO_NOT_MATCH;
    } else if (!areAttributeMinimumsFeasible(offerHolder, taskRequest, activeTaskIdsForRequest)) {
        return AgentMatchState.AGENT_ATTRIBUTES_DO_NOT_MATCH;
    }
    final AgentPlacement agentPlacement = maybeOverrideAgentPlacement(taskRequest.getRequest().getAgentPlacement().orElse(configuration.getDefaultAgentPlacement()));
    if (!taskRequest.getRequest().isRackSensitive() && agentPlacement == AgentPlacement.GREEDY) {
        // todo: account for this or let this behavior continue?
        return AgentMatchState.NOT_RACK_OR_AGENT_PARTICULAR;
    }
    final int numDesiredInstances = taskRequest.getRequest().getInstancesSafe();
    boolean allowBounceToSameHost = isAllowBounceToSameHost(taskRequest.getRequest());
    int activeRacksWithCapacityCount = getActiveRacksWithCapacityCount();
    Multiset<String> countPerRack = HashMultiset.create(activeRacksWithCapacityCount);
    double numOnAgent = 0;
    double numCleaningOnAgent = 0;
    double numFromSameBounceOnAgent = 0;
    double numOtherDeploysOnAgent = 0;
    boolean taskLaunchedFromBounceWithActionId = taskRequest.getPendingTask().getPendingTaskId().getPendingType() == PendingType.BOUNCE && taskRequest.getPendingTask().getActionId().isPresent();
    final String sanitizedHost = offerHolder.getSanitizedHost();
    final String sanitizedRackId = offerHolder.getSanitizedRackId();
    Collection<SingularityTaskId> cleaningTasks = leaderCache.getCleanupTaskIds();
    for (SingularityTaskId taskId : activeTaskIdsForRequest) {
        if (!cleaningTasks.contains(taskId) && !taskManager.isKilledTask(taskId) && taskRequest.getDeploy().getId().equals(taskId.getDeployId())) {
            countPerRack.add(taskId.getSanitizedRackId());
        }
        if (!taskId.getSanitizedHost().equals(sanitizedHost)) {
            continue;
        }
        if (taskRequest.getDeploy().getId().equals(taskId.getDeployId())) {
            if (cleaningTasks.contains(taskId)) {
                numCleaningOnAgent++;
            } else {
                numOnAgent++;
            }
            if (taskLaunchedFromBounceWithActionId) {
                Optional<SingularityTask> maybeTask = taskManager.getTask(taskId);
                boolean errorInTaskData = false;
                if (maybeTask.isPresent()) {
                    SingularityPendingTask pendingTask = maybeTask.get().getTaskRequest().getPendingTask();
                    if (pendingTask.getPendingTaskId().getPendingType() == PendingType.BOUNCE) {
                        if (pendingTask.getActionId().isPresent()) {
                            if (pendingTask.getActionId().get().equals(taskRequest.getPendingTask().getActionId().get())) {
                                numFromSameBounceOnAgent++;
                            }
                        } else {
                            // No actionId present on bounce, fall back to more restrictive placement strategy
                            errorInTaskData = true;
                        }
                    }
                } else {
                    // Could not find appropriate task data, fall back to more restrictive placement strategy
                    errorInTaskData = true;
                }
                if (errorInTaskData) {
                    allowBounceToSameHost = false;
                }
            }
        } else {
            numOtherDeploysOnAgent++;
        }
    }
    if (overrides.isAllowRackSensitivity() && taskRequest.getRequest().isRackSensitive()) {
        final boolean isRackOk = isRackOk(countPerRack, sanitizedRackId, numDesiredInstances, taskRequest.getRequest().getId(), agentId, host, numCleaningOnAgent);
        if (!isRackOk) {
            return AgentMatchState.RACK_SATURATED;
        }
    }
    switch(agentPlacement) {
        case SEPARATE:
        case SEPARATE_BY_DEPLOY:
        case SPREAD_ALL_SLAVES:
        case SPREAD_ALL_AGENTS:
            if (allowBounceToSameHost && taskLaunchedFromBounceWithActionId) {
                if (numFromSameBounceOnAgent > 0) {
                    LOG.trace("Rejecting SEPARATE task {} from agent {} ({}) due to numFromSameBounceOnAgent {}", taskRequest.getRequest().getId(), agentId, host, numFromSameBounceOnAgent);
                    return AgentMatchState.AGENT_SATURATED;
                }
            } else {
                if (numOnAgent > 0 || numCleaningOnAgent > 0) {
                    LOG.trace("Rejecting {} task {} from agent {} ({}) due to numOnAgent {} numCleaningOnAgent {}", agentPlacement.name(), taskRequest.getRequest().getId(), agentId, host, numOnAgent, numCleaningOnAgent);
                    return AgentMatchState.AGENT_SATURATED;
                }
            }
            break;
        case SEPARATE_BY_REQUEST:
            if (numOnAgent > 0 || numCleaningOnAgent > 0 || numOtherDeploysOnAgent > 0) {
                LOG.trace("Rejecting SEPARATE_BY_REQUEST task {} from agent {} ({}) due to numOnAgent {} numCleaningOnAgent {} numOtherDeploysOnAgent {}", taskRequest.getRequest().getId(), agentId, host, numOnAgent, numCleaningOnAgent, numOtherDeploysOnAgent);
                return AgentMatchState.AGENT_SATURATED;
            }
            break;
        case OPTIMISTIC:
            // If no tasks are active for this request yet, we can fall back to greedy.
            if (activeTaskIdsForRequest.size() > 0) {
                Collection<SingularityPendingTaskId> pendingTasksForRequestClusterwide = leaderCache.getPendingTaskIdsForRequest(taskRequest.getRequest().getId());
                Set<String> currentHostsForRequest = activeTaskIdsForRequest.stream().map(SingularityTaskId::getSanitizedHost).collect(Collectors.toSet());
                final double numPerAgent = activeTaskIdsForRequest.size() / (double) currentHostsForRequest.size();
                final double leniencyCoefficient = configuration.getPlacementLeniency();
                final double threshold = numPerAgent * (1 + (pendingTasksForRequestClusterwide.size() * leniencyCoefficient));
                final boolean isOk = numOnAgent <= threshold;
                if (!isOk) {
                    LOG.trace("Rejecting OPTIMISTIC task {} from agent {} ({}) because numOnAgent {} violates threshold {} (based on active tasks for request {}, current hosts for request {}, pending tasks for request {})", taskRequest.getRequest().getId(), agentId, host, numOnAgent, threshold, activeTaskIdsForRequest.size(), currentHostsForRequest.size(), pendingTasksForRequestClusterwide.size());
                    return AgentMatchState.AGENT_SATURATED;
                }
            }
            break;
        case GREEDY:
    }
    if (isPreferred(offerHolder, taskRequest, requestUtilization)) {
        LOG.debug("Agent {} is preferred", offerHolder.getHostname());
        return AgentMatchState.PREFERRED_AGENT;
    }
    return AgentMatchState.OK;
}

Example 3 with AgentPlacement

use of com.hubspot.singularity.AgentPlacement in project Singularity by HubSpot.

the class SingularityValidator method checkScale.

public void checkScale(SingularityRequest request, Optional<Integer> previousScale) {
    AgentPlacement placement = request.getAgentPlacement().orElse(defaultAgentPlacement);
    if (placement != AgentPlacement.GREEDY && placement != AgentPlacement.OPTIMISTIC) {
        int currentActiveAgentCount = agentManager.getNumObjectsAtState(MachineState.ACTIVE);
        int requiredAgentCount = request.getInstancesSafe();
        if (previousScale.isPresent() && placement == AgentPlacement.SEPARATE_BY_REQUEST) {
            requiredAgentCount += previousScale.get();
        }
        checkBadRequest(currentActiveAgentCount >= requiredAgentCount, "Not enough active agents to successfully complete a bounce of request %s (minimum required: %s, current: %s). Consider deploying, or changing the agent placement strategy instead.", request.getId(), requiredAgentCount, currentActiveAgentCount);
        if (request.getRequestType().isLongRunning() && request.getMaxScale().isPresent()) {
            checkBadRequest(request.getInstancesSafe() <= request.getMaxScale().get(), "Instances (%s) cannot be greater than %s (maxScale in request)", request.getInstancesSafe(), request.getMaxScale().get());
        }
    }
}

Example 4 with AgentPlacement

use of com.hubspot.singularity.AgentPlacement in project Singularity by HubSpot.

the class SingularityValidator method checkResourcesForBounce.

public void checkResourcesForBounce(SingularityRequest request, boolean isIncremental) {
    AgentPlacement placement = request.getAgentPlacement().orElse(defaultAgentPlacement);
    if ((isAllowBounceToSameHost(request) && placement == AgentPlacement.SEPARATE_BY_REQUEST) || (!isAllowBounceToSameHost(request) && placement != AgentPlacement.GREEDY && placement != AgentPlacement.OPTIMISTIC)) {
        int currentActiveAgentCount = agentManager.getNumObjectsAtState(MachineState.ACTIVE);
        int requiredAgentCount = isIncremental ? request.getInstancesSafe() + 1 : request.getInstancesSafe() * 2;
        checkBadRequest(currentActiveAgentCount >= requiredAgentCount, "Not enough active agents to successfully scale request %s to %s instances (minimum required: %s, current: %s).", request.getId(), request.getInstancesSafe(), requiredAgentCount, currentActiveAgentCount);
    }
}