Examples with SingularityTaskRequest com.hubspot.singularity.SingularityTaskRequest used on opensource projects

Example 6 with SingularityTaskRequest

use of com.hubspot.singularity.SingularityTaskRequest in project Singularity by HubSpot.

the class SingularityMesosTaskBuilder method buildTask.

public SingularityMesosTaskHolder buildTask(SingularityOfferHolder offerHolder, List<Resource> availableResources, SingularityTaskRequest taskRequest, Resources desiredTaskResources, Resources desiredExecutorResources) {
    final String sanitizedRackId = offerHolder.getSanitizedRackId();
    final String sanitizedHost = offerHolder.getSanitizedHost();
    final SingularityTaskId taskId = new SingularityTaskId(taskRequest.getPendingTask().getPendingTaskId().getRequestId(), taskRequest.getDeploy().getId(), System.currentTimeMillis(), taskRequest.getPendingTask().getPendingTaskId().getInstanceNo(), sanitizedHost, sanitizedRackId);
    final TaskInfo.Builder bldr = TaskInfo.newBuilder().setTaskId(TaskID.newBuilder().setValue(taskId.toString()));
    Optional<long[]> ports = Optional.absent();
    Optional<Resource> portsResource = Optional.absent();
    final Optional<SingularityContainerInfo> containerInfo = taskRequest.getDeploy().getContainerInfo();
    if (desiredTaskResources.getNumPorts() > 0 || hasLiteralPortMapping(containerInfo)) {
        List<Long> requestedPorts = new ArrayList<>();
        if (hasLiteralPortMapping(containerInfo)) {
            requestedPorts.addAll(containerInfo.get().getDocker().get().getLiteralHostPorts());
        }
        portsResource = Optional.of(MesosUtils.getPortsResource(desiredTaskResources.getNumPorts(), availableResources, requestedPorts));
        ports = Optional.of(MesosUtils.getPorts(portsResource.get(), desiredTaskResources.getNumPorts()));
    }
    if (containerInfo.isPresent()) {
        prepareContainerInfo(offerHolder, taskId, bldr, containerInfo.get(), ports);
    }
    if (taskRequest.getDeploy().getCustomExecutorCmd().isPresent()) {
        prepareCustomExecutor(bldr, taskId, taskRequest, offerHolder, ports, desiredExecutorResources);
    } else {
        prepareCommand(bldr, taskId, taskRequest, offerHolder, ports);
    }
    if (portsResource.isPresent()) {
        bldr.addResources(portsResource.get());
    }
    Optional<String> requiredRole = taskRequest.getRequest().getRequiredRole();
    bldr.addResources(MesosUtils.getCpuResource(desiredTaskResources.getCpus(), requiredRole));
    bldr.addResources(MesosUtils.getMemoryResource(desiredTaskResources.getMemoryMb(), requiredRole));
    bldr.addResources(MesosUtils.getDiskResource(desiredTaskResources.getDiskMb(), requiredRole));
    bldr.setAgentId(offerHolder.getOffers().get(0).getAgentId());
    bldr.setName(taskRequest.getRequest().getId());
    final Builder labelsBuilder = Labels.newBuilder();
    // apply request-specific labels, if any
    if (taskRequest.getDeploy().getMesosLabels().isPresent() && !taskRequest.getDeploy().getMesosLabels().get().isEmpty()) {
        for (SingularityMesosTaskLabel label : taskRequest.getDeploy().getMesosLabels().get()) {
            org.apache.mesos.v1.Protos.Label.Builder labelBuilder = Label.newBuilder();
            labelBuilder.setKey(label.getKey());
            if ((label.getValue().isPresent())) {
                labelBuilder.setValue(label.getValue().get());
            }
            labelsBuilder.addLabels(labelBuilder.build());
        }
    }
    // apply task-specific labels, if any
    final int taskInstanceNo = taskRequest.getPendingTask().getPendingTaskId().getInstanceNo();
    if (taskRequest.getDeploy().getMesosTaskLabels().isPresent() && taskRequest.getDeploy().getMesosTaskLabels().get().containsKey(taskInstanceNo) && !taskRequest.getDeploy().getMesosTaskLabels().get().get(taskInstanceNo).isEmpty()) {
        for (SingularityMesosTaskLabel label : taskRequest.getDeploy().getMesosTaskLabels().get().get(taskInstanceNo)) {
            org.apache.mesos.v1.Protos.Label.Builder labelBuilder = Label.newBuilder();
            labelBuilder.setKey(label.getKey());
            if ((label.getValue().isPresent())) {
                labelBuilder.setValue(label.getValue().get());
            }
            labelsBuilder.addLabels(labelBuilder.build());
        }
    }
    bldr.setLabels(labelsBuilder);
    TaskInfo task = bldr.build();
    return new SingularityMesosTaskHolder(new SingularityTask(taskRequest, taskId, offerHolder.getOffers().stream().map((o) -> mesosProtosUtils.offerFromProtos(o)).collect(Collectors.toList()), mesosProtosUtils.taskFromProtos(task), Optional.of(offerHolder.getRackId())), task);
}

Example 7 with SingularityTaskRequest

use of com.hubspot.singularity.SingularityTaskRequest in project Singularity by HubSpot.

the class SingularityMesosTaskPrioritizer method removeTasksAffectedByPriorityFreeze.

public void removeTasksAffectedByPriorityFreeze(List<SingularityTaskRequest> taskRequests) {
    final Optional<SingularityPriorityFreezeParent> maybePriorityFreeze = priorityManager.getActivePriorityFreeze();
    if (maybePriorityFreeze.isPresent()) {
        final ListIterator<SingularityTaskRequest> iterator = taskRequests.listIterator();
        while (iterator.hasNext()) {
            final SingularityTaskRequest taskRequest = iterator.next();
            final double taskPriorityLevel = priorityManager.getTaskPriorityLevelForRequest(taskRequest.getRequest());
            if (taskPriorityLevel < maybePriorityFreeze.get().getPriorityFreeze().getMinimumPriorityLevel()) {
                LOG.trace("Skipping scheduled task {} because taskPriorityLevel ({}) is less than active priority freeze ({})", taskRequest.getPendingTask().getPendingTaskId(), taskPriorityLevel, maybePriorityFreeze.get().getPriorityFreeze().getMinimumPriorityLevel());
                iterator.remove();
            }
        }
    }
}

Example 8 with SingularityTaskRequest

use of com.hubspot.singularity.SingularityTaskRequest in project Singularity by HubSpot.

the class SingularityScheduler method getDueTasks.

@Timed
public List<SingularityTaskRequest> getDueTasks() {
    final List<SingularityPendingTask> tasks = taskManager.getPendingTasks();
    final long now = System.currentTimeMillis();
    final List<SingularityPendingTask> dueTasks = Lists.newArrayListWithCapacity(tasks.size());
    for (SingularityPendingTask task : tasks) {
        if (task.getPendingTaskId().getNextRunAt() <= now) {
            dueTasks.add(task);
        }
    }
    final List<SingularityTaskRequest> dueTaskRequests = taskRequestManager.getTaskRequests(dueTasks);
    return checkForStaleScheduledTasks(dueTasks, dueTaskRequests);
}

Example 9 with SingularityTaskRequest

use of com.hubspot.singularity.SingularityTaskRequest in project Singularity by HubSpot.

the class SingularityScheduler method checkForStaleScheduledTasks.

private List<SingularityTaskRequest> checkForStaleScheduledTasks(List<SingularityPendingTask> pendingTasks, List<SingularityTaskRequest> taskRequests) {
    final Set<String> foundPendingTaskId = Sets.newHashSetWithExpectedSize(taskRequests.size());
    final Set<String> requestIds = Sets.newHashSetWithExpectedSize(taskRequests.size());
    for (SingularityTaskRequest taskRequest : taskRequests) {
        foundPendingTaskId.add(taskRequest.getPendingTask().getPendingTaskId().getId());
        requestIds.add(taskRequest.getRequest().getId());
    }
    for (SingularityPendingTask pendingTask : pendingTasks) {
        if (!foundPendingTaskId.contains(pendingTask.getPendingTaskId().getId())) {
            LOG.info("Removing stale pending task {}", pendingTask.getPendingTaskId());
            taskManager.deletePendingTask(pendingTask.getPendingTaskId());
        }
    }
    // TODO this check isn't necessary if we keep track better during deploys
    final Map<String, SingularityRequestDeployState> deployStates = deployManager.getRequestDeployStatesByRequestIds(requestIds);
    final List<SingularityTaskRequest> taskRequestsWithValidDeploys = Lists.newArrayListWithCapacity(taskRequests.size());
    for (SingularityTaskRequest taskRequest : taskRequests) {
        SingularityRequestDeployState requestDeployState = deployStates.get(taskRequest.getRequest().getId());
        if (!matchesDeploy(requestDeployState, taskRequest) && !(taskRequest.getRequest().getRequestType() == RequestType.RUN_ONCE)) {
            LOG.info("Removing stale pending task {} because the deployId did not match active/pending deploys {}", taskRequest.getPendingTask().getPendingTaskId(), requestDeployState);
            taskManager.deletePendingTask(taskRequest.getPendingTask().getPendingTaskId());
        } else {
            taskRequestsWithValidDeploys.add(taskRequest);
        }
    }
    return taskRequestsWithValidDeploys;
}

Example 10 with SingularityTaskRequest

use of com.hubspot.singularity.SingularityTaskRequest in project Singularity by HubSpot.

the class SingularitySchedulerTest method testSchedulerPriority.

@Test
public void testSchedulerPriority() {
    final SingularityRequest lowPriorityRequest = new SingularityRequestBuilder("lowPriorityRequest", RequestType.WORKER).setTaskPriorityLevel(Optional.of(.25)).build();
    saveRequest(lowPriorityRequest);
    final SingularityRequest mediumPriorityRequest = new SingularityRequestBuilder("mediumPriorityRequest", RequestType.WORKER).setTaskPriorityLevel(Optional.of(.5)).build();
    saveRequest(mediumPriorityRequest);
    final SingularityRequest highPriorityRequest = new SingularityRequestBuilder("highPriorityRequest", RequestType.WORKER).setTaskPriorityLevel(Optional.of(.75)).build();
    saveRequest(highPriorityRequest);
    final SingularityDeploy lowPriorityDeploy = initAndFinishDeploy(lowPriorityRequest, "lowPriorityDeploy");
    final SingularityDeploy mediumPriorityDeploy = initAndFinishDeploy(mediumPriorityRequest, "mediumPriorityDeploy");
    final SingularityDeploy highPriorityDeploy = initAndFinishDeploy(highPriorityRequest, "highPriorityDeploy");
    // Task requests launched at ~ the same time should be in priority order
    long now = System.currentTimeMillis();
    List<SingularityTaskRequest> requestsByPriority = Arrays.asList(buildTaskRequest(lowPriorityRequest, lowPriorityDeploy, now), buildTaskRequest(mediumPriorityRequest, mediumPriorityDeploy, now), buildTaskRequest(highPriorityRequest, highPriorityDeploy, now));
    List<SingularityTaskRequest> sortedRequestsByPriority = taskPrioritizer.getSortedDueTasks(requestsByPriority);
    Assert.assertEquals(sortedRequestsByPriority.get(0).getRequest().getId(), highPriorityRequest.getId());
    Assert.assertEquals(sortedRequestsByPriority.get(1).getRequest().getId(), mediumPriorityRequest.getId());
    Assert.assertEquals(sortedRequestsByPriority.get(2).getRequest().getId(), lowPriorityRequest.getId());
    // A lower priority task that is long overdue should be run before a higher priority task
    now = System.currentTimeMillis();
    List<SingularityTaskRequest> requestsByOverdueAndPriority = Arrays.asList(// 2 min overdue
    buildTaskRequest(lowPriorityRequest, lowPriorityDeploy, now - 120000), // 60s overdue
    buildTaskRequest(mediumPriorityRequest, mediumPriorityDeploy, now - 30000), // Not overdue
    buildTaskRequest(highPriorityRequest, highPriorityDeploy, now));
    List<SingularityTaskRequest> sortedRequestsByOverdueAndPriority = taskPrioritizer.getSortedDueTasks(requestsByOverdueAndPriority);
    Assert.assertEquals(sortedRequestsByOverdueAndPriority.get(0).getRequest().getId(), lowPriorityRequest.getId());
    Assert.assertEquals(sortedRequestsByOverdueAndPriority.get(1).getRequest().getId(), mediumPriorityRequest.getId());
    Assert.assertEquals(sortedRequestsByOverdueAndPriority.get(2).getRequest().getId(), highPriorityRequest.getId());
}