Examples with CachedOffer com.hubspot.singularity.mesos.SingularityOfferCache.CachedOffer used on opensource projects

Example 1 with CachedOffer

use of com.hubspot.singularity.mesos.SingularityOfferCache.CachedOffer in project Singularity by HubSpot.

the class SingularityMesosSchedulerImpl method resourceOffers.

@Timed
@Override
public void resourceOffers(List<Offer> offers) {
    if (!isRunning()) {
        LOG.info("Scheduler is in state {}, declining {} offer(s)", state.name(), offers.size());
        mesosSchedulerClient.decline(offers.stream().map(Offer::getId).collect(Collectors.toList()));
        return;
    }
    callWithOffersLock(() -> {
        final long start = System.currentTimeMillis();
        lastOfferTimestamp = Optional.of(start);
        LOG.info("Received {} offer(s)", offers.size());
        boolean delclineImmediately = false;
        if (disasterManager.isDisabled(SingularityAction.PROCESS_OFFERS)) {
            LOG.info("Processing offers is currently disabled, declining {} offers", offers.size());
            delclineImmediately = true;
        }
        if (delayWhenStatusUpdateDeltaTooLarge && statusUpdateDeltaAvg.get() > delayWhenDeltaOverMs) {
            LOG.info("Status update delta is too large ({}), declining offers while status updates catch up", statusUpdateDeltaAvg.get());
            delclineImmediately = true;
        }
        if (delclineImmediately) {
            mesosSchedulerClient.decline(offers.stream().map(Offer::getId).collect(Collectors.toList()));
            return;
        }
        if (offerCacheEnabled) {
            if (disasterManager.isDisabled(SingularityAction.CACHE_OFFERS)) {
                offerCache.disableOfferCache();
            } else {
                offerCache.enableOfferCache();
            }
        }
        List<Offer> offersToCheck = new ArrayList<>(offers);
        List<CachedOffer> cachedOfferList = offerCache.checkoutOffers();
        Map<String, CachedOffer> cachedOffers = new HashMap<>();
        for (CachedOffer cachedOffer : cachedOfferList) {
            cachedOffers.put(cachedOffer.getOfferId(), cachedOffer);
            offersToCheck.add(cachedOffer.getOffer());
        }
        offers.parallelStream().forEach((offer) -> {
            String rolesInfo = MesosUtils.getRoles(offer).toString();
            LOG.debug("Received offer ID {} with roles {} from {} ({}) for {} cpu(s), {} memory, {} ports, and {} disk", offer.getId().getValue(), rolesInfo, offer.getHostname(), offer.getAgentId().getValue(), MesosUtils.getNumCpus(offer), MesosUtils.getMemory(offer), MesosUtils.getNumPorts(offer), MesosUtils.getDisk(offer));
            CheckResult checkResult = slaveAndRackManager.checkOffer(offer);
            if (checkResult == CheckResult.NOT_ACCEPTING_TASKS) {
                mesosSchedulerClient.decline(Collections.singletonList(offer.getId()));
                offersToCheck.remove(offer);
                LOG.debug("Will decline offer {}, slave {} is not currently in a state to launch tasks", offer.getId().getValue(), offer.getHostname());
            }
        });
        final Set<OfferID> acceptedOffers = Sets.newHashSetWithExpectedSize(offersToCheck.size());
        try {
            Collection<SingularityOfferHolder> offerHolders = offerScheduler.checkOffers(offersToCheck);
            for (SingularityOfferHolder offerHolder : offerHolders) {
                if (!offerHolder.getAcceptedTasks().isEmpty()) {
                    List<Offer> leftoverOffers = offerHolder.launchTasksAndGetUnusedOffers(mesosSchedulerClient);
                    leftoverOffers.forEach((o) -> {
                        if (cachedOffers.containsKey(o.getId().getValue())) {
                            offerCache.returnOffer(cachedOffers.remove(o.getId().getValue()));
                        } else {
                            offerCache.cacheOffer(start, o);
                        }
                    });
                    List<Offer> offersAcceptedFromSlave = offerHolder.getOffers();
                    offersAcceptedFromSlave.removeAll(leftoverOffers);
                    offersAcceptedFromSlave.stream().filter((offer) -> cachedOffers.containsKey(offer.getId().getValue())).map((o) -> cachedOffers.remove(o.getId().getValue())).forEach(offerCache::useOffer);
                    acceptedOffers.addAll(offersAcceptedFromSlave.stream().map(Offer::getId).collect(Collectors.toList()));
                } else {
                    offerHolder.getOffers().forEach((o) -> {
                        if (cachedOffers.containsKey(o.getId().getValue())) {
                            offerCache.returnOffer(cachedOffers.remove(o.getId().getValue()));
                        } else {
                            offerCache.cacheOffer(start, o);
                        }
                    });
                }
            }
            LOG.info("{} remaining offers not accounted for in offer check", cachedOffers.size());
            cachedOffers.values().forEach(offerCache::returnOffer);
        } catch (Throwable t) {
            LOG.error("Received fatal error while handling offers - will decline all available offers", t);
            mesosSchedulerClient.decline(offersToCheck.stream().filter((o) -> !acceptedOffers.contains(o.getId()) && !cachedOffers.containsKey(o.getId().getValue())).map(Offer::getId).collect(Collectors.toList()));
            offersToCheck.forEach((o) -> {
                if (cachedOffers.containsKey(o.getId().getValue())) {
                    offerCache.returnOffer(cachedOffers.get(o.getId().getValue()));
                }
            });
            throw t;
        }
        LOG.info("Finished handling {} new offer(s) ({}), {} accepted, {} declined/cached", offers.size(), JavaUtils.duration(start), acceptedOffers.size(), offers.size() - acceptedOffers.size());
    }, "resourceOffers");
}

Example 2 with CachedOffer

use of com.hubspot.singularity.mesos.SingularityOfferCache.CachedOffer in project Singularity by HubSpot.

the class SingularitySchedulerPoller method runActionOnPoll.

@Override
public void runActionOnPoll() {
    if (disasterManager.isDisabled(SingularityAction.RUN_SCHEDULER_POLLER)) {
        LOG.warn("Scheduler poller is disabled");
        return;
    }
    lock.runWithOffersLock(() -> {
        List<CachedOffer> cachedOffers = offerCache.checkoutOffers();
        Map<String, CachedOffer> offerIdToCachedOffer = new HashMap<>(cachedOffers.size());
        List<Offer> offers = new ArrayList<>(cachedOffers.size());
        for (CachedOffer cachedOffer : cachedOffers) {
            offerIdToCachedOffer.put(cachedOffer.getOfferId(), cachedOffer);
            offers.add(cachedOffer.getOffer());
        }
        Collection<SingularityOfferHolder> offerHolders = offerScheduler.checkOffers(offers);
        if (offerHolders.isEmpty()) {
            return;
        }
        int acceptedOffers = 0;
        int launchedTasks = 0;
        for (SingularityOfferHolder offerHolder : offerHolders) {
            List<CachedOffer> cachedOffersFromHolder = offerHolder.getOffers().stream().map((o) -> offerIdToCachedOffer.get(o.getId().getValue())).collect(Collectors.toList());
            if (!offerHolder.getAcceptedTasks().isEmpty()) {
                List<Offer> unusedOffers = offerHolder.launchTasksAndGetUnusedOffers(schedulerClient);
                launchedTasks += offerHolder.getAcceptedTasks().size();
                acceptedOffers += cachedOffersFromHolder.size() - unusedOffers.size();
                // Return to the cache those offers which we checked out of the cache, but didn't end up using.
                List<CachedOffer> unusedCachedOffers = unusedOffers.stream().map((o) -> offerIdToCachedOffer.get(o.getId().getValue())).collect(Collectors.toList());
                unusedCachedOffers.forEach((cachedOffer) -> {
                    offerIdToCachedOffer.remove(cachedOffer.getOfferId());
                    offerCache.returnOffer(cachedOffer);
                });
                // Notify the cache of the cached offers that we did use.
                cachedOffersFromHolder.removeAll(unusedCachedOffers);
                cachedOffersFromHolder.forEach((cachedOffer) -> {
                    offerIdToCachedOffer.remove(cachedOffer.getOfferId());
                    offerCache.useOffer(cachedOffer);
                });
            } else {
                cachedOffersFromHolder.forEach((cachedOffer) -> {
                    offerIdToCachedOffer.remove(cachedOffer.getOfferId());
                    offerCache.returnOffer(cachedOffer);
                });
            }
        }
        LOG.info("{} remaining offers not accounted for in offer check", offerIdToCachedOffer.size());
        offerIdToCachedOffer.values().forEach(offerCache::returnOffer);
        LOG.info("Launched {} tasks on {} cached offers (returned {})", launchedTasks, acceptedOffers, offerHolders.size() - acceptedOffers);
    }, getClass().getSimpleName());
}

Example 3 with CachedOffer

use of com.hubspot.singularity.mesos.SingularityOfferCache.CachedOffer in project Singularity by HubSpot.

the class SingularityOfferCache method cacheOffer.

@Override
public void cacheOffer(long timestamp, Offer offer) {
    if (!useOfferCache.get()) {
        schedulerClient.decline(Collections.singletonList(offer.getId()));
        return;
    }
    LOG.debug("Caching offer {} for {}", offer.getId().getValue(), JavaUtils.durationFromMillis(configuration.getCacheOffersForMillis()));
    offerCache.put(offer.getId().getValue(), new CachedOffer(offer));
}

Example 4 with CachedOffer

use of com.hubspot.singularity.mesos.SingularityOfferCache.CachedOffer in project Singularity by HubSpot.

the class SingularityOfferCache method checkoutOffers.

@Override
public List<CachedOffer> checkoutOffers() {
    if (!useOfferCache.get()) {
        return Collections.emptyList();
    }
    // Force Guava cache to perform maintenance operations and reach a consistent state.
    offerCache.cleanUp();
    List<CachedOffer> offers = new ArrayList<>((int) offerCache.size());
    for (CachedOffer cachedOffer : offerCache.asMap().values()) {
        cachedOffer.checkOut();
        offers.add(cachedOffer);
    }
    return offers;
}