Examples with Request org.apache.mesos.v1.scheduler.Protos.Call.Request used on opensource projects

Example 6 with Request

use of org.apache.mesos.v1.scheduler.Protos.Call.Request in project Singularity by HubSpot.

the class SingularityOfferPerformanceTestRunner method testSchedulerPerformance.

@Test(timeout = 600000L)
@Ignore
public void testSchedulerPerformance() {
    long start = System.currentTimeMillis();
    int numRequests = 4500;
    int numOffers = 2500;
    Random r = new Random();
    Iterator<Double> cpuIterator = r.doubles(1, 5).iterator();
    Iterator<Double> memoryIterator = r.doubles(15, 20000).iterator();
    Iterator<Double> diskIterator = r.doubles(30, 50000).iterator();
    for (int i = 0; i < numRequests; i++) {
        SingularityRequestBuilder bldr = new SingularityRequestBuilder("request-" + i, RequestType.SERVICE);
        bldr.setInstances(Optional.of(5));
        bldr.setSlavePlacement(Optional.of(SlavePlacement.GREEDY));
        SingularityRequest request = bldr.build();
        saveRequest(request);
        deployRequest(request, cpuIterator.next(), memoryIterator.next());
    }
    List<Offer> offers = new ArrayList<>(numOffers);
    for (int i = 0; i < numOffers; i++) {
        offers.add(createOffer(cpuIterator.next(), memoryIterator.next(), diskIterator.next(), "slave-" + i, "host-" + i));
    }
    System.out.println("Starting after " + JavaUtils.duration(start));
    start = System.currentTimeMillis();
    sms.resourceOffers(offers);
    final long duration = System.currentTimeMillis() - start;
    int activeTasks = taskManager.getActiveTaskIds().size();
    System.out.println(String.format("Launched %s tasks on %s offers in %s", activeTasks, numOffers, JavaUtils.durationFromMillis(duration)));
    start = System.currentTimeMillis();
    ExecutorService statusUpadtes = Executors.newFixedThreadPool(100);
    CompletableFuture<Void>[] updateFutures = new CompletableFuture[taskManager.getActiveTaskIds().size() * 5];
    AtomicInteger i = new AtomicInteger(0);
    for (SingularityTaskId taskId : taskManager.getActiveTaskIds()) {
        updateFutures[i.getAndIncrement()] = CompletableFuture.supplyAsync(() -> {
            statusUpdate(taskManager.getTask(taskId).get(), TaskState.TASK_STAGING);
            return null;
        }, statusUpadtes);
        updateFutures[i.getAndIncrement()] = CompletableFuture.supplyAsync(() -> {
            statusUpdate(taskManager.getTask(taskId).get(), TaskState.TASK_STARTING);
            return null;
        }, statusUpadtes);
        updateFutures[i.getAndIncrement()] = CompletableFuture.supplyAsync(() -> {
            statusUpdate(taskManager.getTask(taskId).get(), TaskState.TASK_RUNNING);
            return null;
        }, statusUpadtes);
        updateFutures[i.getAndIncrement()] = CompletableFuture.supplyAsync(() -> {
            statusUpdate(taskManager.getTask(taskId).get(), TaskState.TASK_FAILED);
            return null;
        }, statusUpadtes);
        updateFutures[i.getAndIncrement()] = CompletableFuture.supplyAsync(() -> {
            statusUpdate(taskManager.getTask(taskId).get(), TaskState.TASK_FAILED);
            return null;
        }, statusUpadtes);
    }
    CompletableFuture.allOf(updateFutures).join();
    System.out.println(String.format("Ran %s status updates in %s", activeTasks * 5, JavaUtils.durationFromMillis(System.currentTimeMillis() - start)));
}

Example 7 with Request

use of org.apache.mesos.v1.scheduler.Protos.Call.Request in project Singularity by HubSpot.

the class SingularityMesosSchedulerClient method kill.

/**
 * Sent by the scheduler to kill a specific task. If the scheduler has a custom executor, the kill is
 * forwarded to the executor; it is up to the executor to kill the task and send a TASK_KILLED
 * (or TASK_FAILED) update. If the task hasn’t yet been delivered to the executor when Mesos master or
 * agent receives the kill request, a TASK_KILLED is generated and the task launch is not forwarded to
 * the executor. Note that if the task belongs to a task group, killing of one task results in all tasks
 * in the task group being killed. Mesos releases the resources for a task once it receives a terminal
 * update for the task. If the task is unknown to the master, a TASK_LOST will be generated.
 *
 * @param taskId
 */
public void kill(TaskID taskId) {
    Builder kill = build().setKill(Kill.newBuilder().setTaskId(taskId));
    sendCall(kill, Type.KILL);
}

Example 8 with Request

use of org.apache.mesos.v1.scheduler.Protos.Call.Request in project Singularity by HubSpot.

the class SingularityMesosSchedulerClient method accept.

/**
 * Sent by the scheduler when it accepts offer(s) sent by the master. The ACCEPT request includes the type of
 * operations (e.g., launch task, launch task group, reserve resources, create volumes) that the scheduler wants to
 * perform on the offers. Note that until the scheduler replies (accepts or declines) to an offer, the offer’s
 * resources are considered allocated to the offer’s role and to the framework.
 *
 * @param offerIds
 * @param offerOperations
 */
public void accept(List<OfferID> offerIds, List<Offer.Operation> offerOperations) {
    Builder accept = build().setAccept(Accept.newBuilder().addAllOfferIds(offerIds).addAllOperations(offerOperations));
    sendCall(accept, Type.ACCEPT);
}

Example 9 with Request

use of org.apache.mesos.v1.scheduler.Protos.Call.Request in project Singularity by HubSpot.

the class SingularityMesosTaskBuilder method buildTask.

public SingularityMesosTaskHolder buildTask(SingularityOfferHolder offerHolder, List<Resource> availableResources, SingularityTaskRequest taskRequest, Resources desiredTaskResources, Resources desiredExecutorResources) {
    final String sanitizedRackId = offerHolder.getSanitizedRackId();
    final String sanitizedHost = offerHolder.getSanitizedHost();
    final SingularityTaskId taskId = new SingularityTaskId(taskRequest.getPendingTask().getPendingTaskId().getRequestId(), taskRequest.getDeploy().getId(), System.currentTimeMillis(), taskRequest.getPendingTask().getPendingTaskId().getInstanceNo(), sanitizedHost, sanitizedRackId);
    final TaskInfo.Builder bldr = TaskInfo.newBuilder().setTaskId(TaskID.newBuilder().setValue(taskId.toString()));
    Optional<long[]> ports = Optional.absent();
    Optional<Resource> portsResource = Optional.absent();
    final Optional<SingularityContainerInfo> containerInfo = taskRequest.getDeploy().getContainerInfo();
    if (desiredTaskResources.getNumPorts() > 0 || hasLiteralPortMapping(containerInfo)) {
        List<Long> requestedPorts = new ArrayList<>();
        if (hasLiteralPortMapping(containerInfo)) {
            requestedPorts.addAll(containerInfo.get().getDocker().get().getLiteralHostPorts());
        }
        portsResource = Optional.of(MesosUtils.getPortsResource(desiredTaskResources.getNumPorts(), availableResources, requestedPorts));
        ports = Optional.of(MesosUtils.getPorts(portsResource.get(), desiredTaskResources.getNumPorts()));
    }
    if (containerInfo.isPresent()) {
        prepareContainerInfo(offerHolder, taskId, bldr, containerInfo.get(), ports);
    }
    if (taskRequest.getDeploy().getCustomExecutorCmd().isPresent()) {
        prepareCustomExecutor(bldr, taskId, taskRequest, offerHolder, ports, desiredExecutorResources);
    } else {
        prepareCommand(bldr, taskId, taskRequest, offerHolder, ports);
    }
    if (portsResource.isPresent()) {
        bldr.addResources(portsResource.get());
    }
    Optional<String> requiredRole = taskRequest.getRequest().getRequiredRole();
    bldr.addResources(MesosUtils.getCpuResource(desiredTaskResources.getCpus(), requiredRole));
    bldr.addResources(MesosUtils.getMemoryResource(desiredTaskResources.getMemoryMb(), requiredRole));
    bldr.addResources(MesosUtils.getDiskResource(desiredTaskResources.getDiskMb(), requiredRole));
    bldr.setAgentId(offerHolder.getOffers().get(0).getAgentId());
    bldr.setName(taskRequest.getRequest().getId());
    final Builder labelsBuilder = Labels.newBuilder();
    // apply request-specific labels, if any
    if (taskRequest.getDeploy().getMesosLabels().isPresent() && !taskRequest.getDeploy().getMesosLabels().get().isEmpty()) {
        for (SingularityMesosTaskLabel label : taskRequest.getDeploy().getMesosLabels().get()) {
            org.apache.mesos.v1.Protos.Label.Builder labelBuilder = Label.newBuilder();
            labelBuilder.setKey(label.getKey());
            if ((label.getValue().isPresent())) {
                labelBuilder.setValue(label.getValue().get());
            }
            labelsBuilder.addLabels(labelBuilder.build());
        }
    }
    // apply task-specific labels, if any
    final int taskInstanceNo = taskRequest.getPendingTask().getPendingTaskId().getInstanceNo();
    if (taskRequest.getDeploy().getMesosTaskLabels().isPresent() && taskRequest.getDeploy().getMesosTaskLabels().get().containsKey(taskInstanceNo) && !taskRequest.getDeploy().getMesosTaskLabels().get().get(taskInstanceNo).isEmpty()) {
        for (SingularityMesosTaskLabel label : taskRequest.getDeploy().getMesosTaskLabels().get().get(taskInstanceNo)) {
            org.apache.mesos.v1.Protos.Label.Builder labelBuilder = Label.newBuilder();
            labelBuilder.setKey(label.getKey());
            if ((label.getValue().isPresent())) {
                labelBuilder.setValue(label.getValue().get());
            }
            labelsBuilder.addLabels(labelBuilder.build());
        }
    }
    bldr.setLabels(labelsBuilder);
    TaskInfo task = bldr.build();
    return new SingularityMesosTaskHolder(new SingularityTask(taskRequest, taskId, offerHolder.getOffers().stream().map((o) -> mesosProtosUtils.offerFromProtos(o)).collect(Collectors.toList()), mesosProtosUtils.taskFromProtos(task), Optional.of(offerHolder.getRackId())), task);
}

Example 10 with Request

use of org.apache.mesos.v1.scheduler.Protos.Call.Request in project Singularity by HubSpot.

the class SingularityScheduler method drainPendingQueue.

@Timed
public void drainPendingQueue() {
    final long start = System.currentTimeMillis();
    final ImmutableList<SingularityPendingRequest> pendingRequests = ImmutableList.copyOf(requestManager.getPendingRequests());
    if (pendingRequests.isEmpty()) {
        LOG.trace("Pending queue was empty");
        return;
    }
    LOG.info("Pending queue had {} requests", pendingRequests.size());
    Map<SingularityDeployKey, List<SingularityPendingRequest>> deployKeyToPendingRequests = pendingRequests.stream().collect(Collectors.groupingBy((request) -> new SingularityDeployKey(request.getRequestId(), request.getDeployId())));
    AtomicInteger totalNewScheduledTasks = new AtomicInteger(0);
    AtomicInteger heldForScheduledActiveTask = new AtomicInteger(0);
    AtomicInteger obsoleteRequests = new AtomicInteger(0);
    deployKeyToPendingRequests.forEach((deployKey, pendingRequestsForDeployKey) -> {
        lock.runWithRequestLock(() -> handlePendingRequestsForDeployKey(obsoleteRequests, heldForScheduledActiveTask, totalNewScheduledTasks, deployKey, pendingRequestsForDeployKey), deployKey.getRequestId(), String.format("%s#%s", getClass().getSimpleName(), "drainPendingQueue"));
    });
    LOG.info("Scheduled {} new tasks ({} obsolete requests, {} held) in {}", totalNewScheduledTasks.get(), obsoleteRequests.get(), heldForScheduledActiveTask.get(), JavaUtils.duration(start));
}