Examples with Task org.apache.mesos.v1.scheduler.Protos.Call.Reconcile.Task used on opensource projects

Example 11 with Task

use of org.apache.mesos.v1.scheduler.Protos.Call.Reconcile.Task in project Singularity by HubSpot.

the class SingularityMesosTaskBuilderTest method testJobUserPassedAsEnvironmentVariable.

@Test
public void testJobUserPassedAsEnvironmentVariable() {
    final SingularityRequest request = new SingularityRequestBuilder("test", RequestType.WORKER).build();
    final SingularityDeploy deploy = new SingularityDeployBuilder("test", "1").setCommand(Optional.of("/bin/echo hi")).build();
    final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask);
    final SingularityMesosTaskHolder task = builder.buildTask(offerHolder, null, taskRequest, taskResources, executorResources);
    List<Variable> environmentVariables = task.getMesosTask().getCommand().getEnvironment().getVariablesList();
    boolean success = false;
    for (Variable environmentVariable : environmentVariables) {
        success = success || (environmentVariable.getName().equals("STARTED_BY_USER") && environmentVariable.getValue().equals(user));
    }
    assertTrue(success, "Expected env variable STARTED_BY_USER to be set to " + user);
}

Example 12 with Task

use of org.apache.mesos.v1.scheduler.Protos.Call.Reconcile.Task in project Singularity by HubSpot.

the class SingularityMesosTaskBuilderTest method testDockerTask.

@Test
public void testDockerTask() {
    taskResources = new Resources(1, 1, 1, 0);
    final Protos.Resource portsResource = Protos.Resource.newBuilder().setName("ports").setType(Protos.Value.Type.RANGES).setRanges(Protos.Value.Ranges.newBuilder().addRange(Protos.Value.Range.newBuilder().setBegin(31000).setEnd(31000).build()).build()).build();
    final SingularityDockerPortMapping literalMapping = new SingularityDockerPortMapping(Optional.<SingularityPortMappingType>empty(), 80, Optional.of(SingularityPortMappingType.LITERAL), 8080, Optional.<String>empty());
    final SingularityDockerPortMapping offerMapping = new SingularityDockerPortMapping(Optional.<SingularityPortMappingType>empty(), 81, Optional.of(SingularityPortMappingType.FROM_OFFER), 0, Optional.of("udp"));
    final SingularityRequest request = new SingularityRequestBuilder("test", RequestType.WORKER).build();
    final SingularityContainerInfo containerInfo = new SingularityContainerInfo(SingularityContainerType.DOCKER, Optional.of(Arrays.asList(new SingularityVolume("/container", Optional.of("/host"), SingularityDockerVolumeMode.RW), new SingularityVolume("/container/${TASK_REQUEST_ID}/${TASK_DEPLOY_ID}", Optional.of("/host/${TASK_ID}"), SingularityDockerVolumeMode.RO))), Optional.of(new SingularityDockerInfo("docker-image", true, SingularityDockerNetworkType.BRIDGE, Optional.of(Arrays.asList(literalMapping, offerMapping)), Optional.of(false), Optional.<Map<String, String>>of(ImmutableMap.of("env", "var=value")), Optional.empty())));
    final SingularityDeploy deploy = new SingularityDeployBuilder("test", "1").setContainerInfo(Optional.of(containerInfo)).setCommand(Optional.of("/bin/echo")).setArguments(Optional.of(Collections.singletonList("wat"))).build();
    final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask);
    final SingularityMesosTaskHolder task = builder.buildTask(offerHolder, Collections.singletonList(portsResource), taskRequest, taskResources, executorResources);
    assertEquals("/bin/echo", task.getMesosTask().getCommand().getValue());
    assertEquals(1, task.getMesosTask().getCommand().getArgumentsCount());
    assertEquals("wat", task.getMesosTask().getCommand().getArguments(0));
    assertFalse(task.getMesosTask().getCommand().getShell());
    assertEquals(Type.DOCKER, task.getMesosTask().getContainer().getType());
    assertEquals("docker-image", task.getMesosTask().getContainer().getDocker().getImage());
    assertTrue(task.getMesosTask().getContainer().getDocker().getPrivileged());
    assertEquals("/container", task.getMesosTask().getContainer().getVolumes(0).getContainerPath());
    assertEquals("/host", task.getMesosTask().getContainer().getVolumes(0).getHostPath());
    assertEquals(Mode.RW, task.getMesosTask().getContainer().getVolumes(0).getMode());
    Parameter envParameter = Parameter.newBuilder().setKey("env").setValue("var=value").build();
    assertTrue(task.getMesosTask().getContainer().getDocker().getParametersList().contains(envParameter));
    assertEquals(String.format("/container/%s/%s", task.getTask().getTaskRequest().getDeploy().getRequestId(), task.getTask().getTaskRequest().getDeploy().getId()), task.getMesosTask().getContainer().getVolumes(1).getContainerPath());
    assertEquals(String.format("/host/%s", task.getMesosTask().getTaskId().getValue()), task.getMesosTask().getContainer().getVolumes(1).getHostPath());
    assertEquals(Mode.RO, task.getMesosTask().getContainer().getVolumes(1).getMode());
    assertEquals(80, task.getMesosTask().getContainer().getDocker().getPortMappings(0).getContainerPort());
    assertEquals(8080, task.getMesosTask().getContainer().getDocker().getPortMappings(0).getHostPort());
    assertEquals("tcp", task.getMesosTask().getContainer().getDocker().getPortMappings(0).getProtocol());
    assertTrue(MesosUtils.getAllPorts(task.getMesosTask().getResourcesList()).contains(8080L));
    assertEquals(81, task.getMesosTask().getContainer().getDocker().getPortMappings(1).getContainerPort());
    assertEquals(31000, task.getMesosTask().getContainer().getDocker().getPortMappings(1).getHostPort());
    assertEquals("udp", task.getMesosTask().getContainer().getDocker().getPortMappings(1).getProtocol());
    assertEquals(Protos.ContainerInfo.DockerInfo.Network.BRIDGE, task.getMesosTask().getContainer().getDocker().getNetwork());
}

Example 13 with Task

use of org.apache.mesos.v1.scheduler.Protos.Call.Reconcile.Task in project Singularity by HubSpot.

the class SingularityTaskReconciliation method checkReconciliation.

private void checkReconciliation(final long reconciliationStart, final Collection<SingularityTaskId> remainingTaskIds, final int numTimes, final Histogram histogram) {
    final List<SingularityTaskStatusHolder> taskStatusHolders = taskManager.getLastActiveTaskStatusesFor(remainingTaskIds);
    final List<MesosTaskStatusObject> taskStatuses = Lists.newArrayListWithCapacity(taskStatusHolders.size());
    for (SingularityTaskStatusHolder taskStatusHolder : taskStatusHolders) {
        if (taskStatusHolder.getServerId().equals(serverId) && taskStatusHolder.getServerTimestamp() > reconciliationStart) {
            histogram.update(taskStatusHolder.getServerTimestamp() - reconciliationStart);
            continue;
        }
        if (taskStatusHolder.getTaskStatus().isPresent()) {
            LOG.debug("Re-requesting task status for {}", taskStatusHolder.getTaskId());
            taskStatuses.add(taskStatusHolder.getTaskStatus().get());
        } else {
            TaskStatus.Builder fakeTaskStatusBuilder = TaskStatus.newBuilder().setTaskId(TaskID.newBuilder().setValue(taskStatusHolder.getTaskId().getId())).setState(TaskState.TASK_STARTING);
            if (taskStatusHolder.getAgentId().isPresent()) {
                fakeTaskStatusBuilder.setAgentId(AgentID.newBuilder().setValue(taskStatusHolder.getAgentId().get()));
            }
            LOG.info("Task {} didn't have a TaskStatus yet, submitting fake status", taskStatusHolder.getTaskId());
            taskStatuses.add(mesosProtosUtils.taskStatusFromProtos(fakeTaskStatusBuilder.build()));
        }
    }
    if (taskStatuses.isEmpty()) {
        LOG.info("Task reconciliation ended after {} checks and {}", numTimes, JavaUtils.duration(reconciliationStart));
        final Snapshot snapshot = histogram.getSnapshot();
        stateManager.saveTaskReconciliationStatistics(new SingularityTaskReconciliationStatistics(reconciliationStart, System.currentTimeMillis() - reconciliationStart, numTimes, histogram.getCount(), snapshot.getMax(), snapshot.getMean(), snapshot.getMin(), snapshot.getMedian(), snapshot.get75thPercentile(), snapshot.get95thPercentile(), snapshot.get98thPercentile(), snapshot.get99thPercentile(), snapshot.get999thPercentile(), snapshot.getStdDev()));
        isRunningReconciliation.set(false);
        return;
    }
    LOG.info("Requesting reconciliation of {} taskStatuses, task reconciliation has been running for {}", taskStatuses.size(), JavaUtils.duration(reconciliationStart));
    schedulerClient.reconcile(taskStatuses.stream().map(t -> Task.newBuilder().setTaskId(MesosProtosUtils.toTaskId(t.getTaskId())).setAgentId(MesosProtosUtils.toAgentId(t.getAgentId())).build()).collect(Collectors.toList()));
    scheduleReconciliationCheck(reconciliationStart, remainingTaskIds, numTimes, histogram);
}

Example 14 with Task

use of org.apache.mesos.v1.scheduler.Protos.Call.Reconcile.Task in project Singularity by HubSpot.

the class SingularityTaskShellCommandDispatchPoller method runActionOnPoll.

@Override
public void runActionOnPoll() {
    final long start = System.currentTimeMillis();
    final List<SingularityTaskShellCommandRequest> shellRequests = taskManager.getAllQueuedTaskShellCommandRequests();
    if (!schedulerClient.isRunning()) {
        LOG.warn("Unable to process shell requests because scheduler driver isn't present ({} tasks waiting)", shellRequests.size());
        return;
    }
    if (shellRequests.isEmpty()) {
        LOG.trace("No shell requests to send.");
        return;
    }
    for (SingularityTaskShellCommandRequest shellRequest : shellRequests) {
        Optional<SingularityTask> task = taskManager.getTask(shellRequest.getTaskId());
        if (!task.isPresent() || !taskManager.isActiveTask(shellRequest.getTaskId())) {
            LOG.info("Skipping shell request {} because {} didn't exist or isn't active", shellRequest, shellRequest.getTaskId());
            continue;
        }
        final ExecutorID executorId = MesosProtosUtils.toExecutorId(task.get().getMesosTask().getExecutor().getExecutorId());
        final AgentID slaveId = MesosProtosUtils.toAgentId(task.get().getMesosTask().getAgentId());
        final byte[] bytes = transcoder.toBytes(shellRequest);
        schedulerClient.frameworkMessage(executorId, slaveId, bytes);
        LOG.info("Sent {} ({} bytes) to {} on {}", shellRequest, bytes.length, executorId, slaveId);
        taskManager.saveTaskShellCommandRequestToTask(shellRequest);
        taskManager.deleteTaskShellCommandRequestFromQueue(shellRequest);
    }
    LOG.info("Sent {} shell requests to executors in {}", shellRequests.size(), JavaUtils.duration(start));
}

Example 15 with Task

use of org.apache.mesos.v1.scheduler.Protos.Call.Reconcile.Task in project Singularity by HubSpot.

the class SingularitySchedulerTest method testTaskOddities.

@Test
public void testTaskOddities() {
    // test unparseable status update
    TaskStatus.Builder bldr = TaskStatus.newBuilder().setTaskId(TaskID.newBuilder().setValue("task")).setAgentId(AgentID.newBuilder().setValue("agent1")).setState(TaskState.TASK_RUNNING);
    // should not throw exception:
    sms.statusUpdate(bldr.build()).join();
    initRequest();
    initFirstDeploy();
    SingularityTask taskOne = launchTask(request, firstDeploy, 1, TaskState.TASK_STARTING);
    taskManager.deleteTaskHistory(taskOne.getTaskId());
    Assertions.assertTrue(taskManager.isActiveTask(taskOne.getTaskId()));
    statusUpdate(taskOne, TaskState.TASK_RUNNING);
    statusUpdate(taskOne, TaskState.TASK_FAILED);
    Assertions.assertTrue(!taskManager.isActiveTask(taskOne.getTaskId()));
    Assertions.assertEquals(2, taskManager.getTaskHistoryUpdates(taskOne.getTaskId()).size());
}