Examples with SingularitySlave com.hubspot.singularity.SingularitySlave used on opensource projects

Example 11 with SingularitySlave

use of com.hubspot.singularity.SingularitySlave in project Singularity by HubSpot.

the class SingularitySlaveAndRackManager method loadSlavesAndRacksFromMaster.

public void loadSlavesAndRacksFromMaster(MesosMasterStateObject state, boolean isStartup) {
    Map<String, SingularitySlave> activeSlavesById = slaveManager.getObjectsByIdForState(MachineState.ACTIVE);
    Map<String, SingularityRack> activeRacksById = rackManager.getObjectsByIdForState(MachineState.ACTIVE);
    Map<String, SingularityRack> remainingActiveRacks = Maps.newHashMap(activeRacksById);
    int slaves = 0;
    int racks = 0;
    for (MesosMasterSlaveObject slaveJsonObject : state.getSlaves()) {
        String slaveId = slaveJsonObject.getId();
        String rackId = slaveAndRackHelper.getRackId(slaveJsonObject.getAttributes());
        Map<String, String> textAttributes = slaveAndRackHelper.getTextAttributes(slaveJsonObject.getAttributes());
        String host = slaveAndRackHelper.getMaybeTruncatedHost(slaveJsonObject.getHostname());
        if (activeSlavesById.containsKey(slaveId)) {
            SingularitySlave slave = activeSlavesById.get(slaveId);
            if (slave != null && (!slave.getResources().isPresent() || !slave.getResources().get().equals(slaveJsonObject.getResources()))) {
                LOG.trace("Found updated resources ({}) for slave {}", slaveJsonObject.getResources(), slave);
                slaveManager.saveObject(slave.withResources(slaveJsonObject.getResources()));
            }
            activeSlavesById.remove(slaveId);
        } else {
            SingularitySlave newSlave = new SingularitySlave(slaveId, host, rackId, textAttributes, Optional.of(slaveJsonObject.getResources()));
            if (check(newSlave, slaveManager) == CheckResult.NEW) {
                slaves++;
            }
        }
        if (activeRacksById.containsKey(rackId)) {
            remainingActiveRacks.remove(rackId);
        } else {
            SingularityRack rack = new SingularityRack(rackId);
            if (check(rack, rackManager) == CheckResult.NEW) {
                racks++;
            }
        }
    }
    for (SingularitySlave leftOverSlave : activeSlavesById.values()) {
        slaveManager.changeState(leftOverSlave, isStartup ? MachineState.MISSING_ON_STARTUP : MachineState.DEAD, Optional.absent(), Optional.absent());
    }
    for (SingularityRack leftOverRack : remainingActiveRacks.values()) {
        rackManager.changeState(leftOverRack, isStartup ? MachineState.MISSING_ON_STARTUP : MachineState.DEAD, Optional.absent(), Optional.absent());
    }
    LOG.info("Found {} new racks ({} missing) and {} new slaves ({} missing)", racks, remainingActiveRacks.size(), slaves, activeSlavesById.size());
}

Example 12 with SingularitySlave

use of com.hubspot.singularity.SingularitySlave in project Singularity by HubSpot.

the class SingularitySlaveAndRackManager method checkOffer.

public CheckResult checkOffer(Offer offer) {
    final String slaveId = offer.getAgentId().getValue();
    final String rackId = slaveAndRackHelper.getRackIdOrDefault(offer);
    final String host = slaveAndRackHelper.getMaybeTruncatedHost(offer);
    final Map<String, String> textAttributes = slaveAndRackHelper.getTextAttributes(offer);
    final SingularitySlave slave = new SingularitySlave(slaveId, host, rackId, textAttributes, Optional.absent());
    CheckResult result = check(slave, slaveManager);
    if (result == CheckResult.NEW) {
        if (inactiveSlaveManager.isInactive(slave.getHost())) {
            LOG.info("Slave {} on inactive host {} attempted to rejoin. Marking as decommissioned.", slave, host);
            slaveManager.changeState(slave, MachineState.STARTING_DECOMMISSION, Optional.of(String.format("Slave %s on inactive host %s attempted to rejoin cluster.", slaveId, host)), Optional.absent());
        } else {
            LOG.info("Offer revealed a new slave {}", slave);
        }
    }
    final SingularityRack rack = new SingularityRack(rackId);
    if (check(rack, rackManager) == CheckResult.NEW) {
        LOG.info("Offer revealed a new rack {}", rack);
    }
    return result;
}

Example 13 with SingularitySlave

use of com.hubspot.singularity.SingularitySlave in project Singularity by HubSpot.

the class SingularitySlaveReconciliationPoller method checkDeadSlaves.

private void checkDeadSlaves() {
    final long start = System.currentTimeMillis();
    final List<SingularitySlave> deadSlaves = slaveManager.getObjectsFiltered(MachineState.DEAD);
    if (deadSlaves.isEmpty()) {
        LOG.trace("No dead slaves");
        return;
    }
    int deleted = 0;
    final long maxDuration = TimeUnit.HOURS.toMillis(configuration.getDeleteDeadSlavesAfterHours());
    for (SingularitySlave deadSlave : slaveManager.getObjectsFiltered(MachineState.DEAD)) {
        final long duration = System.currentTimeMillis() - deadSlave.getCurrentState().getTimestamp();
        if (duration > maxDuration) {
            SingularityDeleteResult result = slaveManager.deleteObject(deadSlave.getId());
            deleted++;
            LOG.info("Removing dead slave {} ({}) after {} (max {})", deadSlave.getId(), result, JavaUtils.durationFromMillis(duration), JavaUtils.durationFromMillis(maxDuration));
        }
    }
    LOG.debug("Checked {} dead slaves, deleted {} in {}", deadSlaves.size(), deleted, JavaUtils.duration(start));
}

Example 14 with SingularitySlave

use of com.hubspot.singularity.SingularitySlave in project Singularity by HubSpot.

the class SingularityExecutorCleanup method isDecommissioned.

private boolean isDecommissioned() {
    Collection<SingularitySlave> slaves = singularityClient.getSlaves(Optional.of(MachineState.DECOMMISSIONED));
    boolean decommissioned = false;
    for (SingularitySlave slave : slaves) {
        if (slave.getHost().equals(hostname)) {
            decommissioned = true;
        }
    }
    return decommissioned;
}

Example 15 with SingularitySlave

use of com.hubspot.singularity.SingularitySlave in project Singularity by HubSpot.

the class SingularityScheduler method checkForDecomissions.

@Timed
public void checkForDecomissions() {
    final long start = System.currentTimeMillis();
    final Map<String, Optional<String>> requestIdsToUserToReschedule = Maps.newHashMap();
    final Set<SingularityTaskId> matchingTaskIds = Sets.newHashSet();
    final Collection<SingularityTaskId> activeTaskIds = leaderCache.getActiveTaskIds();
    final Map<SingularitySlave, MachineState> slaves = getDefaultMap(slaveManager.getObjectsFiltered(MachineState.STARTING_DECOMMISSION));
    for (SingularitySlave slave : slaves.keySet()) {
        boolean foundTask = false;
        for (SingularityTask activeTask : taskManager.getTasksOnSlave(activeTaskIds, slave)) {
            cleanupTaskDueToDecomission(requestIdsToUserToReschedule, matchingTaskIds, activeTask, slave);
            foundTask = true;
        }
        if (!foundTask) {
            slaves.put(slave, MachineState.DECOMMISSIONED);
        }
    }
    final Map<SingularityRack, MachineState> racks = getDefaultMap(rackManager.getObjectsFiltered(MachineState.STARTING_DECOMMISSION));
    for (SingularityRack rack : racks.keySet()) {
        final String sanitizedRackId = JavaUtils.getReplaceHyphensWithUnderscores(rack.getId());
        boolean foundTask = false;
        for (SingularityTaskId activeTaskId : activeTaskIds) {
            if (sanitizedRackId.equals(activeTaskId.getSanitizedRackId())) {
                foundTask = true;
            }
            if (matchingTaskIds.contains(activeTaskId)) {
                continue;
            }
            if (sanitizedRackId.equals(activeTaskId.getSanitizedRackId())) {
                Optional<SingularityTask> maybeTask = taskManager.getTask(activeTaskId);
                cleanupTaskDueToDecomission(requestIdsToUserToReschedule, matchingTaskIds, maybeTask.get(), rack);
            }
        }
        if (!foundTask) {
            racks.put(rack, MachineState.DECOMMISSIONED);
        }
    }
    for (Entry<String, Optional<String>> requestIdAndUser : requestIdsToUserToReschedule.entrySet()) {
        final String requestId = requestIdAndUser.getKey();
        LOG.trace("Rescheduling request {} due to decomissions", requestId);
        Optional<String> maybeDeployId = deployManager.getInUseDeployId(requestId);
        if (maybeDeployId.isPresent()) {
            requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, maybeDeployId.get(), start, requestIdAndUser.getValue(), PendingType.DECOMISSIONED_SLAVE_OR_RACK, Optional.<Boolean>absent(), Optional.<String>absent()));
        } else {
            LOG.warn("Not rescheduling a request ({}) because of no active deploy", requestId);
        }
    }
    changeState(slaves, slaveManager);
    changeState(racks, rackManager);
    if (slaves.isEmpty() && racks.isEmpty() && requestIdsToUserToReschedule.isEmpty() && matchingTaskIds.isEmpty()) {
        LOG.trace("Decomission check found nothing");
    } else {
        LOG.info("Found {} decomissioning slaves, {} decomissioning racks, rescheduling {} requests and scheduling {} tasks for cleanup in {}", slaves.size(), racks.size(), requestIdsToUserToReschedule.size(), matchingTaskIds.size(), JavaUtils.duration(start));
    }
}