use of org.apache.mesos.v1.Protos.Offer in project Singularity by HubSpot.
the class SingularityMesosSchedulerClient method decline.
/**
* Sent by the scheduler to explicitly decline offer(s) received. Note that this is same as sending an
* ACCEPT call with no operations.
*
* @param offerIds
*/
public void decline(List<OfferID> offerIds) {
Builder decline = build().setDecline(Decline.newBuilder().addAllOfferIds(offerIds));
sendCall(decline, Type.DECLINE);
}
use of org.apache.mesos.v1.Protos.Offer in project Singularity by HubSpot.
the class SingularityMesosSchedulerClient method accept.
/**
* Sent by the scheduler when it accepts offer(s) sent by the master. The ACCEPT request includes the type of
* operations (e.g., launch task, launch task group, reserve resources, create volumes) that the scheduler wants to
* perform on the offers. Note that until the scheduler replies (accepts or declines) to an offer, the offer’s
* resources are considered allocated to the offer’s role and to the framework.
*
* @param offerIds
* @param offerOperations
*/
public void accept(List<OfferID> offerIds, List<Offer.Operation> offerOperations) {
Builder accept = build().setAccept(Accept.newBuilder().addAllOfferIds(offerIds).addAllOperations(offerOperations));
sendCall(accept, Type.ACCEPT);
}
use of org.apache.mesos.v1.Protos.Offer in project Singularity by HubSpot.
the class SingularityMesosSchedulerImpl method resourceOffers.
@Timed
@Override
public void resourceOffers(List<Offer> offers) {
if (!isRunning()) {
LOG.info("Scheduler is in state {}, declining {} offer(s)", state.name(), offers.size());
mesosSchedulerClient.decline(offers.stream().map(Offer::getId).collect(Collectors.toList()));
return;
}
callWithOffersLock(() -> {
final long start = System.currentTimeMillis();
lastOfferTimestamp = Optional.of(start);
LOG.info("Received {} offer(s)", offers.size());
boolean delclineImmediately = false;
if (disasterManager.isDisabled(SingularityAction.PROCESS_OFFERS)) {
LOG.info("Processing offers is currently disabled, declining {} offers", offers.size());
delclineImmediately = true;
}
if (delayWhenStatusUpdateDeltaTooLarge && statusUpdateDeltaAvg.get() > delayWhenDeltaOverMs) {
LOG.info("Status update delta is too large ({}), declining offers while status updates catch up", statusUpdateDeltaAvg.get());
delclineImmediately = true;
}
if (delclineImmediately) {
mesosSchedulerClient.decline(offers.stream().map(Offer::getId).collect(Collectors.toList()));
return;
}
if (offerCacheEnabled) {
if (disasterManager.isDisabled(SingularityAction.CACHE_OFFERS)) {
offerCache.disableOfferCache();
} else {
offerCache.enableOfferCache();
}
}
List<Offer> offersToCheck = new ArrayList<>(offers);
List<CachedOffer> cachedOfferList = offerCache.checkoutOffers();
Map<String, CachedOffer> cachedOffers = new HashMap<>();
for (CachedOffer cachedOffer : cachedOfferList) {
cachedOffers.put(cachedOffer.getOfferId(), cachedOffer);
offersToCheck.add(cachedOffer.getOffer());
}
offers.parallelStream().forEach((offer) -> {
String rolesInfo = MesosUtils.getRoles(offer).toString();
LOG.debug("Received offer ID {} with roles {} from {} ({}) for {} cpu(s), {} memory, {} ports, and {} disk", offer.getId().getValue(), rolesInfo, offer.getHostname(), offer.getAgentId().getValue(), MesosUtils.getNumCpus(offer), MesosUtils.getMemory(offer), MesosUtils.getNumPorts(offer), MesosUtils.getDisk(offer));
CheckResult checkResult = slaveAndRackManager.checkOffer(offer);
if (checkResult == CheckResult.NOT_ACCEPTING_TASKS) {
mesosSchedulerClient.decline(Collections.singletonList(offer.getId()));
offersToCheck.remove(offer);
LOG.debug("Will decline offer {}, slave {} is not currently in a state to launch tasks", offer.getId().getValue(), offer.getHostname());
}
});
final Set<OfferID> acceptedOffers = Sets.newHashSetWithExpectedSize(offersToCheck.size());
try {
Collection<SingularityOfferHolder> offerHolders = offerScheduler.checkOffers(offersToCheck);
for (SingularityOfferHolder offerHolder : offerHolders) {
if (!offerHolder.getAcceptedTasks().isEmpty()) {
List<Offer> leftoverOffers = offerHolder.launchTasksAndGetUnusedOffers(mesosSchedulerClient);
leftoverOffers.forEach((o) -> {
if (cachedOffers.containsKey(o.getId().getValue())) {
offerCache.returnOffer(cachedOffers.remove(o.getId().getValue()));
} else {
offerCache.cacheOffer(start, o);
}
});
List<Offer> offersAcceptedFromSlave = offerHolder.getOffers();
offersAcceptedFromSlave.removeAll(leftoverOffers);
offersAcceptedFromSlave.stream().filter((offer) -> cachedOffers.containsKey(offer.getId().getValue())).map((o) -> cachedOffers.remove(o.getId().getValue())).forEach(offerCache::useOffer);
acceptedOffers.addAll(offersAcceptedFromSlave.stream().map(Offer::getId).collect(Collectors.toList()));
} else {
offerHolder.getOffers().forEach((o) -> {
if (cachedOffers.containsKey(o.getId().getValue())) {
offerCache.returnOffer(cachedOffers.remove(o.getId().getValue()));
} else {
offerCache.cacheOffer(start, o);
}
});
}
}
LOG.info("{} remaining offers not accounted for in offer check", cachedOffers.size());
cachedOffers.values().forEach(offerCache::returnOffer);
} catch (Throwable t) {
LOG.error("Received fatal error while handling offers - will decline all available offers", t);
mesosSchedulerClient.decline(offersToCheck.stream().filter((o) -> !acceptedOffers.contains(o.getId()) && !cachedOffers.containsKey(o.getId().getValue())).map(Offer::getId).collect(Collectors.toList()));
offersToCheck.forEach((o) -> {
if (cachedOffers.containsKey(o.getId().getValue())) {
offerCache.returnOffer(cachedOffers.get(o.getId().getValue()));
}
});
throw t;
}
LOG.info("Finished handling {} new offer(s) ({}), {} accepted, {} declined/cached", offers.size(), JavaUtils.duration(start), acceptedOffers.size(), offers.size() - acceptedOffers.size());
}, "resourceOffers");
}
use of org.apache.mesos.v1.Protos.Offer in project Singularity by HubSpot.
the class SingularityMesosOfferSchedulerTest method itAccountsForExpectedTaskUsage.
@Test
public void itAccountsForExpectedTaskUsage() {
initRequest();
double cpuReserved = 2;
double memMbReserved = 1000;
initFirstDeployWithResources(cpuReserved, memMbReserved);
saveAndSchedule(requestManager.getRequest(requestId).get().getRequest().toBuilder().setInstances(Optional.of(1)));
resourceOffers(3);
SingularityTaskId taskId = taskManager.getActiveTaskIds().get(0);
String t1 = taskId.getId();
// 2 cpus used
MesosTaskMonitorObject t1u1 = getTaskMonitor(t1, 10, TimeUnit.MILLISECONDS.toSeconds(taskId.getStartedAt()) + 5, 1000);
mesosClient.setSlaveResourceUsage("host1", Collections.singletonList(t1u1));
usagePoller.runActionOnPoll();
Map<ResourceUsageType, Number> longRunningTasksUsage = new HashMap<>();
longRunningTasksUsage.put(ResourceUsageType.CPU_USED, 0.1);
longRunningTasksUsage.put(ResourceUsageType.MEMORY_BYTES_USED, 0.1);
longRunningTasksUsage.put(ResourceUsageType.DISK_BYTES_USED, 0.1);
SingularitySlaveUsage smallUsage = new SingularitySlaveUsage(0.1, 0.1, Optional.of(10.0), 1, 1, Optional.of(30L), 1, 1, Optional.of(1024L), longRunningTasksUsage, 1, System.currentTimeMillis(), 1, 30000, 10, 0, 0, 0, 0, 107374182);
usageManager.saveSpecificSlaveUsageAndSetCurrent("host1", smallUsage);
usageManager.saveSpecificSlaveUsageAndSetCurrent("host2", smallUsage);
usageManager.saveSpecificSlaveUsageAndSetCurrent("host3", smallUsage);
requestResource.scale(requestId, new SingularityScaleRequest(Optional.of(3), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent()), SingularityUser.DEFAULT_USER);
Assert.assertEquals(2.0, usageManager.getRequestUtilizations().get(requestId).getCpuUsed(), 0.001);
Offer host2Offer = createOffer(6, 30000, 107374182, "host2", "host2");
slaveAndRackManager.checkOffer(host2Offer);
Offer host3Offer = createOffer(6, 30000, 107374182, "host3", "host3");
slaveAndRackManager.checkOffer(host3Offer);
Collection<SingularityOfferHolder> offerHolders = offerScheduler.checkOffers(Arrays.asList(host2Offer, host3Offer));
Assert.assertEquals(2, offerHolders.size());
// A single offer should only ever get a single task even though both have room for both tasks here. Adding a task should reduce the score for the next check
for (SingularityOfferHolder offerHolder : offerHolders) {
Assert.assertEquals(1, offerHolder.getAcceptedTasks().size());
}
}
use of org.apache.mesos.v1.Protos.Offer in project Singularity by HubSpot.
the class SingularitySchedulerPoller method runActionOnPoll.
@Override
public void runActionOnPoll() {
if (disasterManager.isDisabled(SingularityAction.RUN_SCHEDULER_POLLER)) {
LOG.warn("Scheduler poller is disabled");
return;
}
lock.runWithOffersLock(() -> {
List<CachedOffer> cachedOffers = offerCache.checkoutOffers();
Map<String, CachedOffer> offerIdToCachedOffer = new HashMap<>(cachedOffers.size());
List<Offer> offers = new ArrayList<>(cachedOffers.size());
for (CachedOffer cachedOffer : cachedOffers) {
offerIdToCachedOffer.put(cachedOffer.getOfferId(), cachedOffer);
offers.add(cachedOffer.getOffer());
}
Collection<SingularityOfferHolder> offerHolders = offerScheduler.checkOffers(offers);
if (offerHolders.isEmpty()) {
return;
}
int acceptedOffers = 0;
int launchedTasks = 0;
for (SingularityOfferHolder offerHolder : offerHolders) {
List<CachedOffer> cachedOffersFromHolder = offerHolder.getOffers().stream().map((o) -> offerIdToCachedOffer.get(o.getId().getValue())).collect(Collectors.toList());
if (!offerHolder.getAcceptedTasks().isEmpty()) {
List<Offer> unusedOffers = offerHolder.launchTasksAndGetUnusedOffers(schedulerClient);
launchedTasks += offerHolder.getAcceptedTasks().size();
acceptedOffers += cachedOffersFromHolder.size() - unusedOffers.size();
// Return to the cache those offers which we checked out of the cache, but didn't end up using.
List<CachedOffer> unusedCachedOffers = unusedOffers.stream().map((o) -> offerIdToCachedOffer.get(o.getId().getValue())).collect(Collectors.toList());
unusedCachedOffers.forEach((cachedOffer) -> {
offerIdToCachedOffer.remove(cachedOffer.getOfferId());
offerCache.returnOffer(cachedOffer);
});
// Notify the cache of the cached offers that we did use.
cachedOffersFromHolder.removeAll(unusedCachedOffers);
cachedOffersFromHolder.forEach((cachedOffer) -> {
offerIdToCachedOffer.remove(cachedOffer.getOfferId());
offerCache.useOffer(cachedOffer);
});
} else {
cachedOffersFromHolder.forEach((cachedOffer) -> {
offerIdToCachedOffer.remove(cachedOffer.getOfferId());
offerCache.returnOffer(cachedOffer);
});
}
}
LOG.info("{} remaining offers not accounted for in offer check", offerIdToCachedOffer.size());
offerIdToCachedOffer.values().forEach(offerCache::returnOffer);
LOG.info("Launched {} tasks on {} cached offers (returned {})", launchedTasks, acceptedOffers, offerHolders.size() - acceptedOffers);
}, getClass().getSimpleName());
}
Aggregations