Examples with SingularityMesosTaskHolder com.hubspot.singularity.helpers.SingularityMesosTaskHolder used on opensource projects

Example 11 with SingularityMesosTaskHolder

use of com.hubspot.singularity.helpers.SingularityMesosTaskHolder in project Singularity by HubSpot.

the class SingularityOfferHolder method launchTasksAndGetUnusedOffers.

public List<Offer> launchTasksAndGetUnusedOffers(SingularityMesosSchedulerClient schedulerClient) {
    final List<TaskInfo> toLaunch = Lists.newArrayListWithCapacity(acceptedTasks.size());
    final List<SingularityTaskId> taskIds = Lists.newArrayListWithCapacity(acceptedTasks.size());
    for (SingularityMesosTaskHolder taskHolder : acceptedTasks) {
        taskIds.add(taskHolder.getTask().getTaskId());
        toLaunch.add(taskHolder.getMesosTask());
        LOG.debug("Launching {} with offer {}", taskHolder.getTask().getTaskId(), offers.get(0).getId());
        LOG.trace("Launching {} mesos task: {}", taskHolder.getTask().getTaskId(), MesosUtils.formatForLogging(taskHolder.getMesosTask()));
    }
    // At this point, `currentResources` contains a list of unused resources, because we subtracted out the required resources of every task we accepted.
    // Let's try and reclaim offers by trying to pull each offer's list of resources out of the combined pool of leftover resources.
    // n.b., This is currently not optimal. We just look through the offers in this instance and try to reclaim them with no particular priority or order.
    Map<Boolean, List<Offer>> partitionedOffers = offers.stream().collect(Collectors.partitioningBy(offer -> {
        List<Long> ports = MesosUtils.getAllPorts(offer.getResourcesList());
        boolean offerCanBeReclaimedFromUnusedResources = offer.getResourcesList().stream().collect(Collectors.groupingBy(Resource::getRole)).entrySet().stream().map((entry) -> {
            // Now, for each set of offer Resources grouped by role...
            String role = entry.getKey();
            List<Resource> offerResources = entry.getValue();
            Optional<String> maybeRole = (!role.equals("") && !role.equals("*")) ? Optional.of(role) : Optional.absent();
            // ...Check if we can pull the Resources belonging to this offer out of the pool of `currentResources`.
            return MesosUtils.doesOfferMatchResources(maybeRole, MesosUtils.buildResourcesFromMesosResourceList(offerResources, maybeRole), currentResources, ports);
        }).reduce(true, (x, y) -> x && y);
        if (offerCanBeReclaimedFromUnusedResources) {
            // We can reclaim this offer in its entirety! Pull all of its resources out of the combined pool for this SingularityOfferHolder instance.
            LOG.trace("Able to reclaim offer {} from unused resources in OfferHolder from host {}. cpu: {}, mem: {}, disk: {}", offer.getId().getValue(), offer.getHostname(), MesosUtils.getNumCpus(offer), MesosUtils.getMemory(offer), MesosUtils.getDisk(offer));
            currentResources = MesosUtils.subtractResources(currentResources, offer.getResourcesList());
        }
        return offerCanBeReclaimedFromUnusedResources;
    }));
    List<Offer> leftoverOffers = partitionedOffers.get(true);
    List<Offer> neededOffers = partitionedOffers.get(false);
    schedulerClient.accept(neededOffers.stream().map(Offer::getId).collect(Collectors.toList()), Collections.singletonList(Operation.newBuilder().setType(Type.LAUNCH).setLaunch(Launch.newBuilder().addAllTaskInfos(toLaunch).build()).build()));
    LOG.info("{} tasks ({}) launched", taskIds.size(), taskIds);
    return leftoverOffers;
}

Example 12 with SingularityMesosTaskHolder

use of com.hubspot.singularity.helpers.SingularityMesosTaskHolder in project Singularity by HubSpot.

the class SingularityMesosOfferScheduler method acceptTask.

private SingularityMesosTaskHolder acceptTask(SingularityOfferHolder offerHolder, Map<String, Integer> tasksPerOffer, SingularityTaskRequestHolder taskRequestHolder) {
    final SingularityTaskRequest taskRequest = taskRequestHolder.getTaskRequest();
    final SingularityMesosTaskHolder taskHolder = mesosTaskBuilder.buildTask(offerHolder, offerHolder.getCurrentResources(), taskRequest, taskRequestHolder.getTaskResources(), taskRequestHolder.getExecutorResources());
    final SingularityTask zkTask = taskSizeOptimizer.getSizeOptimizedTask(taskHolder);
    LOG.trace("Accepted and built task {}", zkTask);
    LOG.info("Launching task {} slot on slave {} ({})", taskHolder.getTask().getTaskId(), offerHolder.getSlaveId(), offerHolder.getHostname());
    taskManager.createTaskAndDeletePendingTask(zkTask);
    addRequestToMapByOfferHost(tasksPerOffer, offerHolder.getHostname(), taskRequest.getRequest().getId());
    return taskHolder;
}