Examples with TaskID org.apache.mesos.v1.Protos.TaskID used on opensource projects

Example 11 with TaskID

use of org.apache.mesos.v1.Protos.TaskID in project Singularity by HubSpot.

the class SingularityMesosOfferSchedulerTest method itAccountsForExpectedTaskUsage.

@Test
public void itAccountsForExpectedTaskUsage() {
    initRequest();
    double cpuReserved = 2;
    double memMbReserved = 1000;
    initFirstDeployWithResources(cpuReserved, memMbReserved);
    saveAndSchedule(requestManager.getRequest(requestId).get().getRequest().toBuilder().setInstances(Optional.of(1)));
    resourceOffers(3);
    SingularityTaskId taskId = taskManager.getActiveTaskIds().get(0);
    String t1 = taskId.getId();
    // 2 cpus used
    MesosTaskMonitorObject t1u1 = getTaskMonitor(t1, 10, TimeUnit.MILLISECONDS.toSeconds(taskId.getStartedAt()) + 5, 1000);
    mesosClient.setSlaveResourceUsage("host1", Collections.singletonList(t1u1));
    usagePoller.runActionOnPoll();
    Map<ResourceUsageType, Number> longRunningTasksUsage = new HashMap<>();
    longRunningTasksUsage.put(ResourceUsageType.CPU_USED, 0.1);
    longRunningTasksUsage.put(ResourceUsageType.MEMORY_BYTES_USED, 0.1);
    longRunningTasksUsage.put(ResourceUsageType.DISK_BYTES_USED, 0.1);
    SingularitySlaveUsage smallUsage = new SingularitySlaveUsage(0.1, 0.1, Optional.of(10.0), 1, 1, Optional.of(30L), 1, 1, Optional.of(1024L), longRunningTasksUsage, 1, System.currentTimeMillis(), 1, 30000, 10, 0, 0, 0, 0, 107374182);
    usageManager.saveSpecificSlaveUsageAndSetCurrent("host1", smallUsage);
    usageManager.saveSpecificSlaveUsageAndSetCurrent("host2", smallUsage);
    usageManager.saveSpecificSlaveUsageAndSetCurrent("host3", smallUsage);
    requestResource.scale(requestId, new SingularityScaleRequest(Optional.of(3), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent()), SingularityUser.DEFAULT_USER);
    Assert.assertEquals(2.0, usageManager.getRequestUtilizations().get(requestId).getCpuUsed(), 0.001);
    Offer host2Offer = createOffer(6, 30000, 107374182, "host2", "host2");
    slaveAndRackManager.checkOffer(host2Offer);
    Offer host3Offer = createOffer(6, 30000, 107374182, "host3", "host3");
    slaveAndRackManager.checkOffer(host3Offer);
    Collection<SingularityOfferHolder> offerHolders = offerScheduler.checkOffers(Arrays.asList(host2Offer, host3Offer));
    Assert.assertEquals(2, offerHolders.size());
    // A single offer should only ever get a single task even though both have room for both tasks here. Adding a task should reduce the score for the next check
    for (SingularityOfferHolder offerHolder : offerHolders) {
        Assert.assertEquals(1, offerHolder.getAcceptedTasks().size());
    }
}

Example 12 with TaskID

use of org.apache.mesos.v1.Protos.TaskID in project Singularity by HubSpot.

the class SingularitySchedulerTestBase method prepTask.

protected SingularityTask prepTask(SingularityRequest request, SingularityDeploy deploy, long launchTime, int instanceNo, boolean separateHosts, Optional<String> runId) {
    SingularityPendingTask pendingTask = buildPendingTask(request, deploy, launchTime, instanceNo, runId);
    SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask);
    Offer offer;
    if (separateHosts) {
        offer = createOffer(125, 1024, 2048, String.format("slave%s", instanceNo), String.format("host%s", instanceNo));
    } else {
        offer = createOffer(125, 1024, 2048);
    }
    SingularityTaskId taskId = new SingularityTaskId(request.getId(), deploy.getId(), launchTime, instanceNo, offer.getHostname(), "rack1");
    TaskID taskIdProto = TaskID.newBuilder().setValue(taskId.toString()).build();
    TaskInfo taskInfo = TaskInfo.newBuilder().setAgentId(offer.getAgentId()).setExecutor(ExecutorInfo.newBuilder().setExecutorId(ExecutorID.newBuilder().setValue("executorID"))).setTaskId(taskIdProto).setName("name").build();
    SingularityTask task = new SingularityTask(taskRequest, taskId, Collections.singletonList(mesosProtosUtils.offerFromProtos(offer)), mesosProtosUtils.taskFromProtos(taskInfo), Optional.of("rack1"));
    taskManager.savePendingTask(pendingTask);
    return task;
}

Example 13 with TaskID

use of org.apache.mesos.v1.Protos.TaskID in project Singularity by HubSpot.

the class SingularityMesosSchedulerClient method kill.

public void kill(TaskID taskId, AgentID agentId) {
    Builder kill = build().setKill(Kill.newBuilder().setTaskId(taskId).setAgentId(agentId));
    sendCall(kill, Type.KILL);
}

Example 14 with TaskID

use of org.apache.mesos.v1.Protos.TaskID in project Singularity by HubSpot.

the class SingularityMesosTaskBuilder method prepareCustomExecutor.

/**
 * Prepares the Mesos TaskInfo object when using our custom SingularityExecutor.
 */
private void prepareCustomExecutor(final TaskInfo.Builder bldr, final SingularityTaskId taskId, final SingularityTaskRequest task, final SingularityOfferHolder offerHolder, final Optional<long[]> ports, final Resources desiredExecutorResources) {
    CommandInfo.Builder commandBuilder = CommandInfo.newBuilder().setValue(task.getDeploy().getCustomExecutorCmd().get());
    prepareEnvironment(task, taskId, commandBuilder, offerHolder, ports);
    if (task.getDeploy().getUser().isPresent()) {
        commandBuilder.setUser(task.getDeploy().getUser().get());
    }
    prepareMesosUriDownloads(task.getPendingTask().getExtraArtifacts(), commandBuilder);
    bldr.setExecutor(ExecutorInfo.newBuilder().setCommand(commandBuilder.build()).setExecutorId(ExecutorID.newBuilder().setValue(task.getDeploy().getCustomExecutorId().or(idGenerator.getNextExecutorId()))).setSource(// set source to taskId for use in statistics endpoint, TODO: remove
    task.getDeploy().getCustomExecutorSource().or(taskId.getId())).setLabels(Labels.newBuilder().addLabels(Label.newBuilder().setKey("taskId").setValue(taskId.getId()))).addAllResources(buildMesosResources(desiredExecutorResources, task.getRequest().getRequiredRole())).build());
    if (task.getDeploy().getExecutorData().isPresent()) {
        final ExecutorDataBuilder executorDataBldr = task.getDeploy().getExecutorData().get().toBuilder();
        String defaultS3Bucket = "";
        String s3UploaderKeyPattern = "";
        if (configuration.getS3ConfigurationOptional().isPresent()) {
            if (task.getRequest().getGroup().isPresent() && configuration.getS3ConfigurationOptional().get().getGroupOverrides().containsKey(task.getRequest().getGroup().get())) {
                defaultS3Bucket = configuration.getS3ConfigurationOptional().get().getGroupOverrides().get(task.getRequest().getGroup().get()).getS3Bucket();
                LOG.trace("Setting defaultS3Bucket to {} for task {} executorData", defaultS3Bucket, taskId.getId());
            } else {
                defaultS3Bucket = configuration.getS3ConfigurationOptional().get().getS3Bucket();
            }
            s3UploaderKeyPattern = configuration.getS3ConfigurationOptional().get().getS3KeyFormat();
        }
        if (task.getPendingTask().getCmdLineArgsList().isPresent() && !task.getPendingTask().getCmdLineArgsList().get().isEmpty()) {
            LOG.trace("Adding cmd line args {} to task {} executorData", task.getPendingTask().getCmdLineArgsList(), taskId.getId());
            final ImmutableList.Builder<String> extraCmdLineArgsBuilder = ImmutableList.builder();
            if (executorDataBldr.getExtraCmdLineArgs() != null && !executorDataBldr.getExtraCmdLineArgs().isEmpty()) {
                extraCmdLineArgsBuilder.addAll(executorDataBldr.getExtraCmdLineArgs());
            }
            extraCmdLineArgsBuilder.addAll(task.getPendingTask().getCmdLineArgsList().get());
            executorDataBldr.setExtraCmdLineArgs(extraCmdLineArgsBuilder.build());
        }
        List<SingularityS3UploaderFile> uploaderAdditionalFiles = new ArrayList<>();
        if (configuration.getS3ConfigurationOptional().isPresent()) {
            uploaderAdditionalFiles.addAll(configuration.getS3ConfigurationOptional().get().getS3UploaderAdditionalFiles());
        }
        uploaderAdditionalFiles.addAll(task.getPendingTask().getS3UploaderAdditionalFiles());
        Optional<String> maybeS3StorageClass = configuration.getS3ConfigurationOptional().isPresent() ? configuration.getS3ConfigurationOptional().get().getS3StorageClass() : Optional.<String>absent();
        Optional<Long> maybeApplyAfterBytes = configuration.getS3ConfigurationOptional().isPresent() ? configuration.getS3ConfigurationOptional().get().getApplyS3StorageClassAfterBytes() : Optional.<Long>absent();
        if (task.getPendingTask().getRunAsUserOverride().isPresent()) {
            executorDataBldr.setUser(task.getPendingTask().getRunAsUserOverride());
        }
        final SingularityTaskExecutorData executorData = new SingularityTaskExecutorData(executorDataBldr.build(), uploaderAdditionalFiles, defaultS3Bucket, s3UploaderKeyPattern, configuration.getCustomExecutorConfiguration().getServiceLog(), configuration.getCustomExecutorConfiguration().getServiceFinishedTailLog(), task.getRequest().getGroup(), maybeS3StorageClass, maybeApplyAfterBytes);
        try {
            bldr.setData(ByteString.copyFromUtf8(objectMapper.writeValueAsString(executorData)));
        } catch (JsonProcessingException e) {
            LOG.warn("Unable to process executor data {} for task {} as json (trying as string)", executorData, taskId.getId(), e);
            bldr.setData(ByteString.copyFromUtf8(executorData.toString()));
        }
    } else if (task.getDeploy().getCommand().isPresent()) {
        bldr.setData(ByteString.copyFromUtf8(task.getDeploy().getCommand().get()));
    }
}