Examples with Resource org.apache.mesos.v1.Protos.Resource used on opensource projects

Example 6 with Resource

use of org.apache.mesos.v1.Protos.Resource in project Singularity by HubSpot.

the class MesosUtilsTest method testLiteralHostPortSelection.

@Test
public void testLiteralHostPortSelection() {
    String[] rangesNotOverlappingRequestedPorts = { "23:24", "25:25", "31:32", "50:51" };
    int numPorts = 1;
    List<Long> requestedPorts = Arrays.asList(50L, 51L);
    Resource resource = MesosUtils.getPortsResource(numPorts, buildOffer(rangesNotOverlappingRequestedPorts).getResourcesList(), requestedPorts);
    Assert.assertTrue(MesosUtils.getAllPorts(Collections.singletonList(resource)).containsAll(requestedPorts));
    Assert.assertEquals(numPorts + requestedPorts.size(), MesosUtils.getNumPorts(Collections.singletonList(resource)));
    String[] rangesOverlappingRequestPorts = { "23:28" };
    numPorts = 4;
    requestedPorts = Arrays.asList(25L, 27L);
    resource = MesosUtils.getPortsResource(numPorts, buildOffer(rangesOverlappingRequestPorts).getResourcesList(), requestedPorts);
    Assert.assertTrue(MesosUtils.getAllPorts(Collections.singletonList(resource)).containsAll(requestedPorts));
    Assert.assertEquals(numPorts + requestedPorts.size(), MesosUtils.getNumPorts(Collections.singletonList(resource)));
}

Example 7 with Resource

use of org.apache.mesos.v1.Protos.Resource in project Singularity by HubSpot.

the class SingularityOfferHolder method launchTasksAndGetUnusedOffers.

public List<Offer> launchTasksAndGetUnusedOffers(SingularityMesosSchedulerClient schedulerClient) {
    final List<TaskInfo> toLaunch = Lists.newArrayListWithCapacity(acceptedTasks.size());
    final List<SingularityTaskId> taskIds = Lists.newArrayListWithCapacity(acceptedTasks.size());
    for (SingularityMesosTaskHolder taskHolder : acceptedTasks) {
        taskIds.add(taskHolder.getTask().getTaskId());
        toLaunch.add(taskHolder.getMesosTask());
        LOG.debug("Launching {} with offer {}", taskHolder.getTask().getTaskId(), offers.get(0).getId());
        LOG.trace("Launching {} mesos task: {}", taskHolder.getTask().getTaskId(), MesosUtils.formatForLogging(taskHolder.getMesosTask()));
    }
    // At this point, `currentResources` contains a list of unused resources, because we subtracted out the required resources of every task we accepted.
    // Let's try and reclaim offers by trying to pull each offer's list of resources out of the combined pool of leftover resources.
    // n.b., This is currently not optimal. We just look through the offers in this instance and try to reclaim them with no particular priority or order.
    Map<Boolean, List<Offer>> partitionedOffers = offers.stream().collect(Collectors.partitioningBy(offer -> {
        List<Long> ports = MesosUtils.getAllPorts(offer.getResourcesList());
        boolean offerCanBeReclaimedFromUnusedResources = offer.getResourcesList().stream().collect(Collectors.groupingBy(Resource::getRole)).entrySet().stream().map((entry) -> {
            // Now, for each set of offer Resources grouped by role...
            String role = entry.getKey();
            List<Resource> offerResources = entry.getValue();
            Optional<String> maybeRole = (!role.equals("") && !role.equals("*")) ? Optional.of(role) : Optional.absent();
            // ...Check if we can pull the Resources belonging to this offer out of the pool of `currentResources`.
            return MesosUtils.doesOfferMatchResources(maybeRole, MesosUtils.buildResourcesFromMesosResourceList(offerResources, maybeRole), currentResources, ports);
        }).reduce(true, (x, y) -> x && y);
        if (offerCanBeReclaimedFromUnusedResources) {
            // We can reclaim this offer in its entirety! Pull all of its resources out of the combined pool for this SingularityOfferHolder instance.
            LOG.trace("Able to reclaim offer {} from unused resources in OfferHolder from host {}. cpu: {}, mem: {}, disk: {}", offer.getId().getValue(), offer.getHostname(), MesosUtils.getNumCpus(offer), MesosUtils.getMemory(offer), MesosUtils.getDisk(offer));
            currentResources = MesosUtils.subtractResources(currentResources, offer.getResourcesList());
        }
        return offerCanBeReclaimedFromUnusedResources;
    }));
    List<Offer> leftoverOffers = partitionedOffers.get(true);
    List<Offer> neededOffers = partitionedOffers.get(false);
    schedulerClient.accept(neededOffers.stream().map(Offer::getId).collect(Collectors.toList()), Collections.singletonList(Operation.newBuilder().setType(Type.LAUNCH).setLaunch(Launch.newBuilder().addAllTaskInfos(toLaunch).build()).build()));
    LOG.info("{} tasks ({}) launched", taskIds.size(), taskIds);
    return leftoverOffers;
}

Example 8 with Resource

use of org.apache.mesos.v1.Protos.Resource in project Singularity by HubSpot.

the class MesosUtils method getPorts.

public static long[] getPorts(Resource portsResource, int numPorts) {
    long[] ports = new long[numPorts];
    if (numPorts == 0) {
        return ports;
    }
    int idx = 0;
    for (Range r : portsResource.getRanges().getRangeList()) {
        for (long port = r.getBegin(); port <= r.getEnd(); port++) {
            ports[idx++] = port;
            if (idx >= numPorts) {
                return ports;
            }
        }
    }
    return ports;
}

Example 9 with Resource

use of org.apache.mesos.v1.Protos.Resource in project Singularity by HubSpot.

the class MesosUtils method getPortsResource.

public static Resource getPortsResource(int numPorts, List<Resource> resources, List<Long> otherRequestedPorts) {
    List<Long> requestedPorts = new ArrayList<>(otherRequestedPorts);
    Ranges ranges = getRanges(resources, PORTS);
    Preconditions.checkState(ranges.getRangeCount() > 0, "Ports %s should have existed in resources %s", PORTS, formatForLogging(resources));
    Ranges.Builder rangesBldr = Ranges.newBuilder();
    int portsSoFar = 0;
    List<Range> offerRangeList = Lists.newArrayList(ranges.getRangeList());
    Random random = new Random();
    Collections.shuffle(offerRangeList, random);
    if (numPorts > 0) {
        for (Range range : offerRangeList) {
            long rangeStartSelection = Math.max(range.getBegin(), range.getEnd() - (numPorts - portsSoFar + 1));
            if (rangeStartSelection != range.getBegin()) {
                int rangeDelta = (int) (rangeStartSelection - range.getBegin()) + 1;
                rangeStartSelection = random.nextInt(rangeDelta) + range.getBegin();
            }
            long rangeEndSelection = Math.min(range.getEnd(), rangeStartSelection + (numPorts - portsSoFar - 1));
            rangesBldr.addRange(Range.newBuilder().setBegin(rangeStartSelection).setEnd(rangeEndSelection));
            portsSoFar += (rangeEndSelection - rangeStartSelection) + 1;
            List<Long> toRemove = new ArrayList<>();
            for (long port : requestedPorts) {
                if (rangeStartSelection >= port && rangeEndSelection <= port) {
                    toRemove.add(port);
                    portsSoFar--;
                }
            }
            requestedPorts.removeAll(toRemove);
            if (portsSoFar == numPorts) {
                break;
            }
        }
    }
    for (long port : requestedPorts) {
        rangesBldr.addRange(Range.newBuilder().setBegin(port).setEnd(port).build());
    }
    return Resource.newBuilder().setType(Type.RANGES).setName(PORTS).setRanges(rangesBldr).build();
}

Example 10 with Resource

use of org.apache.mesos.v1.Protos.Resource in project Singularity by HubSpot.

the class MesosUtils method getAllPorts.

public static List<Long> getAllPorts(List<Resource> resources) {
    Ranges ranges = getRanges(resources, PORTS);
    final List<Long> ports = Lists.newArrayList();
    if (ranges != null) {
        for (Range range : ranges.getRangeList()) {
            for (long port = range.getBegin(); port <= range.getEnd(); port++) {
                ports.add(port);
            }
        }
    }
    return ports;
}