Examples with SingularityAgent com.hubspot.singularity.SingularityAgent used on opensource projects

Example 11 with SingularityAgent

use of com.hubspot.singularity.SingularityAgent in project Singularity by HubSpot.

the class SingularityClientTest method itGetsSingularitySlaves.

@Test
public void itGetsSingularitySlaves() {
    SingularityAgent agent = new SingularityAgent("agentId", "host", "rackId", new HashMap<>(), Optional.empty());
    when(httpClient.execute(any())).thenReturn(response);
    when(response.getStatusCode()).thenReturn(200);
    when(response.isError()).thenReturn(false);
    when(response.getAs(ArgumentMatchers.<TypeReference<Collection<SingularityAgent>>>any())).thenReturn(ImmutableList.of(agent));
    Collection<SingularitySlave> agents = singularityClient.getSlaves(Optional.empty());
    assertThat(agents).isNotEmpty().hasSize(1);
}

Example 12 with SingularityAgent

use of com.hubspot.singularity.SingularityAgent in project Singularity by HubSpot.

the class RebalancingHelper method rebalanceAttributeDistribution.

public Set<SingularityTaskId> rebalanceAttributeDistribution(SingularityRequest request, Optional<String> user, List<SingularityTaskId> remainingActiveTasks) {
    Map<String, Map<String, Set<SingularityTaskId>>> attributeTaskMap = new HashMap<>();
    for (SingularityTaskId taskId : remainingActiveTasks) {
        SingularityAgent slave = agentManager.getObject(taskManager.getTask(taskId).get().getMesosTask().getSlaveId().getValue()).get();
        for (Entry<String, String> entry : slave.getAttributes().entrySet()) {
            attributeTaskMap.computeIfAbsent(entry.getKey(), key -> new HashMap<>()).computeIfAbsent(entry.getValue(), key -> new HashSet<>()).add(taskId);
        }
    }
    Integer numDesiredInstances = request.getInstancesSafe();
    Set<SingularityTaskId> extraTasksToClean = new HashSet<>();
    for (Entry<String, Map<String, Integer>> keyEntry : request.getAgentAttributeMinimums().get().entrySet()) {
        for (Entry<String, Integer> valueEntry : keyEntry.getValue().entrySet()) {
            String attributeName = keyEntry.getKey();
            String attributeValue = valueEntry.getKey();
            Integer attributePercent = valueEntry.getValue();
            Set<SingularityTaskId> matchingTaskIds = attributeTaskMap.get(attributeName).get(attributeValue);
            matchingTaskIds.removeAll(extraTasksToClean);
            int minInstancesWithAttr = Math.max(1, (int) Math.ceil((attributePercent / 100.0) * numDesiredInstances));
            int numInstancesWithAttr = attributeTaskMap.containsKey(attributeName) && attributeTaskMap.get(attributeName).containsKey(attributeValue) ? matchingTaskIds.size() : 0;
            int maxPotentialInstancesWithAttr = numInstancesWithAttr + (numDesiredInstances - remainingActiveTasks.size());
            int numTasksToClean = minInstancesWithAttr - maxPotentialInstancesWithAttr;
            if (numTasksToClean > 0) {
                Set<SingularityTaskId> tasksToClean = remainingActiveTasks.stream().filter(t -> !matchingTaskIds.contains(t)).limit(numTasksToClean).collect(Collectors.toSet());
                LOG.info("Marking tasks {} for cleanup to satisfy attribute {}={} on at least {}% of instances", tasksToClean, attributeName, attributeValue, valueEntry.getValue());
                extraTasksToClean.addAll(tasksToClean);
            }
        }
    }
    for (SingularityTaskId taskId : extraTasksToClean) {
        taskManager.createTaskCleanup(new SingularityTaskCleanup(user, TaskCleanupType.REBALANCE_SLAVE_ATTRIBUTES, System.currentTimeMillis(), taskId, Optional.empty(), Optional.empty(), Optional.empty()));
    }
    return extraTasksToClean;
}

Example 13 with SingularityAgent

use of com.hubspot.singularity.SingularityAgent in project Singularity by HubSpot.

the class SingularityScheduler method checkForDecomissions.

@Timed
public void checkForDecomissions() {
    final long start = System.currentTimeMillis();
    final Map<String, Optional<String>> requestIdsToUserToReschedule = Maps.newHashMap();
    final Set<SingularityTaskId> matchingTaskIds = Sets.newHashSet();
    final Collection<SingularityTaskId> activeTaskIds = leaderCache.getActiveTaskIds();
    final Map<SingularityAgent, MachineState> agents = getDefaultMap(agentManager.getObjectsFiltered(MachineState.STARTING_DECOMMISSION));
    for (SingularityAgent agent : agents.keySet()) {
        boolean foundTask = false;
        for (SingularityTask activeTask : taskManager.getTasksOnAgent(activeTaskIds, agent)) {
            cleanupTaskDueToDecomission(requestIdsToUserToReschedule, matchingTaskIds, activeTask, agent);
            foundTask = true;
        }
        if (!foundTask) {
            agents.put(agent, MachineState.DECOMMISSIONED);
        }
    }
    final Map<SingularityRack, MachineState> racks = getDefaultMap(rackManager.getObjectsFiltered(MachineState.STARTING_DECOMMISSION));
    for (SingularityRack rack : racks.keySet()) {
        final String sanitizedRackId = JavaUtils.getReplaceHyphensWithUnderscores(rack.getId());
        boolean foundTask = false;
        for (SingularityTaskId activeTaskId : activeTaskIds) {
            if (sanitizedRackId.equals(activeTaskId.getSanitizedRackId())) {
                foundTask = true;
            }
            if (matchingTaskIds.contains(activeTaskId)) {
                continue;
            }
            if (sanitizedRackId.equals(activeTaskId.getSanitizedRackId())) {
                Optional<SingularityTask> maybeTask = taskManager.getTask(activeTaskId);
                cleanupTaskDueToDecomission(requestIdsToUserToReschedule, matchingTaskIds, maybeTask.get(), rack);
            }
        }
        if (!foundTask) {
            racks.put(rack, MachineState.DECOMMISSIONED);
        }
    }
    for (Entry<String, Optional<String>> requestIdAndUser : requestIdsToUserToReschedule.entrySet()) {
        final String requestId = requestIdAndUser.getKey();
        LOG.trace("Rescheduling request {} due to decomissions", requestId);
        Optional<String> maybeDeployId = deployManager.getInUseDeployId(requestId);
        if (maybeDeployId.isPresent()) {
            requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, maybeDeployId.get(), start, requestIdAndUser.getValue(), PendingType.DECOMISSIONED_SLAVE_OR_RACK, Optional.<Boolean>empty(), Optional.<String>empty()));
        } else {
            LOG.warn("Not rescheduling a request ({}) because of no active deploy", requestId);
        }
    }
    changeState(agents, agentManager);
    changeState(racks, rackManager);
    if (agents.isEmpty() && racks.isEmpty() && requestIdsToUserToReschedule.isEmpty() && matchingTaskIds.isEmpty()) {
        LOG.trace("Decomission check found nothing");
    } else {
        LOG.info("Found {} decomissioning agents, {} decomissioning racks, rescheduling {} requests and scheduling {} tasks for cleanup in {}", agents.size(), racks.size(), requestIdsToUserToReschedule.size(), matchingTaskIds.size(), JavaUtils.duration(start));
    }
}

Example 14 with SingularityAgent

use of com.hubspot.singularity.SingularityAgent in project Singularity by HubSpot.

the class SingularityExecutorCleanup method isDecommissioned.

private boolean isDecommissioned() {
    Collection<SingularityAgent> agents = singularityClient.getAgents(Optional.of(MachineState.DECOMMISSIONED));
    boolean decommissioned = false;
    for (SingularityAgent agent : agents) {
        if (agent.getHost().equals(hostname)) {
            decommissioned = true;
        }
    }
    return decommissioned;
}

Example 15 with SingularityAgent

use of com.hubspot.singularity.SingularityAgent in project Singularity by HubSpot.

the class StateManager method generateState.

public SingularityState generateState(boolean includeRequestIds) {
    final int launchingTasks = taskManager.getNumLaunchingTasks();
    final int activeTasks = taskManager.getNumActiveTasks() - launchingTasks;
    final int scheduledTasks = taskManager.getNumScheduledTasks();
    final int cleaningTasks = taskManager.getNumCleanupTasks();
    final int lbCleanupTasks = taskManager.getNumLbCleanupTasks();
    final int lbCleanupRequests = requestManager.getNumLbCleanupRequests();
    final SingularityScheduledTasksInfo scheduledTasksInfo = getScheduledTasksInfo();
    final List<String> overProvisionedRequestIds = new ArrayList<>();
    final Set<String> possiblyUnderProvisionedRequestIds = new HashSet<>();
    final List<SingularityRequestWithState> requests = requestManager.getRequests();
    final Map<String, Long> numInstances = getNumTasks(requests);
    int numActiveRequests = 0;
    int numPausedRequests = 0;
    int cooldownRequests = 0;
    int numFinishedRequests = 0;
    for (SingularityRequestWithState requestWithState : requests) {
        switch(requestWithState.getState()) {
            case DEPLOYING_TO_UNPAUSE:
            case ACTIVE:
                numActiveRequests++;
                break;
            case FINISHED:
                numFinishedRequests++;
                break;
            case PAUSED:
                numPausedRequests++;
                break;
            case SYSTEM_COOLDOWN:
                cooldownRequests++;
                break;
            case DELETED:
                break;
        }
        updatePossiblyUnderProvisionedAndOverProvisionedIds(requestWithState, numInstances, overProvisionedRequestIds, possiblyUnderProvisionedRequestIds);
    }
    filterForPendingRequests(possiblyUnderProvisionedRequestIds);
    final List<String> underProvisionedRequestIds = getUnderProvisionedRequestIds(possiblyUnderProvisionedRequestIds);
    final int pendingRequests = requestManager.getSizeOfPendingQueue();
    final int cleaningRequests = requestManager.getSizeOfCleanupQueue();
    List<SingularityRack> racks = rackManager.getObjects();
    int activeRacks = 0;
    int deadRacks = 0;
    int decommissioningRacks = 0;
    int unknownRacks = 0;
    for (SingularityRack rack : racks) {
        switch(rack.getCurrentState().getState()) {
            case ACTIVE:
                activeRacks++;
                break;
            case DEAD:
                deadRacks++;
                break;
            case MISSING_ON_STARTUP:
                unknownRacks++;
                break;
            case DECOMMISSIONED:
            case STARTING_DECOMMISSION:
            case DECOMMISSIONING:
                decommissioningRacks++;
                break;
            default:
                unknownRacks++;
                break;
        }
    }
    List<SingularityAgent> slaves = agentManager.getObjects();
    int activeSlaves = 0;
    int deadSlaves = 0;
    int decommissioningSlaves = 0;
    int unknownSlaves = 0;
    for (SingularityAgent slave : slaves) {
        switch(slave.getCurrentState().getState()) {
            case ACTIVE:
                activeSlaves++;
                break;
            case DEAD:
                deadSlaves++;
                break;
            case MISSING_ON_STARTUP:
                unknownSlaves++;
                break;
            case DECOMMISSIONED:
            case STARTING_DECOMMISSION:
            case DECOMMISSIONING:
                decommissioningSlaves++;
                break;
            default:
                unknownSlaves++;
                break;
        }
    }
    final List<SingularityHostState> states = getHostStates();
    int numDeploys = 0;
    long oldestDeploy = 0;
    long oldestDeployStep = 0;
    List<SingularityDeployMarker> activeDeploys = new ArrayList<>();
    final long now = System.currentTimeMillis();
    for (SingularityPendingDeploy pendingDeploy : deployManager.getPendingDeploys()) {
        activeDeploys.add(pendingDeploy.getDeployMarker());
        if (!pendingDeploy.getDeployProgress().isStepLaunchComplete()) {
            long deployStepDelta = now - pendingDeploy.getDeployProgress().getTimestamp();
            if (deployStepDelta > oldestDeployStep) {
                oldestDeployStep = deployStepDelta;
            }
        }
        long delta = now - pendingDeploy.getDeployMarker().getTimestamp();
        if (delta > oldestDeploy) {
            oldestDeploy = delta;
        }
        numDeploys++;
    }
    final Optional<Boolean> authDatastoreHealthy = authDatastore.isHealthy();
    final Optional<Double> minimumPriorityLevel = getMinimumPriorityLevel();
    return new SingularityState(activeTasks, launchingTasks, numActiveRequests, cooldownRequests, numPausedRequests, scheduledTasks, pendingRequests, lbCleanupTasks, lbCleanupRequests, cleaningRequests, activeSlaves, deadSlaves, decommissioningSlaves, activeRacks, deadRacks, decommissioningRacks, cleaningTasks, states, oldestDeploy, numDeploys, oldestDeployStep, activeDeploys, scheduledTasksInfo.getLateTasks().size(), scheduledTasksInfo.getLateTasks(), scheduledTasksInfo.getOnDemandLateTasks().size(), scheduledTasksInfo.getOnDemandLateTasks(), scheduledTasksInfo.getNumFutureTasks(), scheduledTasksInfo.getMaxTaskLag(), System.currentTimeMillis(), includeRequestIds ? overProvisionedRequestIds : null, includeRequestIds ? underProvisionedRequestIds : null, overProvisionedRequestIds.size(), underProvisionedRequestIds.size(), numFinishedRequests, unknownRacks, unknownSlaves, authDatastoreHealthy, minimumPriorityLevel, (long) statusUpdateDeltas.getSnapshot().getMean(), lastHeartbeatTime.get(), disasterManager.getFireAlarm());
}