Examples with SingularityTaskShellCommandUpdate com.hubspot.singularity.SingularityTaskShellCommandUpdate used on opensource projects

Example 1 with SingularityTaskShellCommandUpdate

use of com.hubspot.singularity.SingularityTaskShellCommandUpdate in project Singularity by HubSpot.

the class SingularityTaskShellCommandTest method testTaskShellCommandPersistence.

@Test
public void testTaskShellCommandPersistence() {
    initRequest();
    initFirstDeploy();
    SingularityTask task = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING);
    // test bad command first:
    setShellCommandsConfiguration();
    // bad shell cmd
    try {
        taskResource.runShellCommand(singularityUser, task.getTaskId().getId(), new SingularityShellCommand("test-cmd", Optional.of(Arrays.asList("one", "two")), user, Optional.empty()));
    } catch (WebApplicationException exception) {
        assertEquals(400, exception.getResponse().getStatus());
    }
    // bad option
    try {
        taskResource.runShellCommand(singularityUser, task.getTaskId().getId(), new SingularityShellCommand("d1", Optional.of(Arrays.asList("one", "two")), user, Optional.empty()));
    } catch (WebApplicationException exception) {
        assertEquals(400, exception.getResponse().getStatus());
    }
    SingularityTaskShellCommandRequest firstShellRequest = taskResource.runShellCommand(singularityUser, task.getTaskId().getId(), new SingularityShellCommand("d1", Optional.of(Arrays.asList("o1", "o2")), user, Optional.empty()));
    try {
        Thread.sleep(3);
    } catch (Exception e) {
    }
    SingularityTaskShellCommandRequest secondShellRequest = taskResource.runShellCommand(singularityUser, task.getTaskId().getId(), new SingularityShellCommand("d2", Optional.<List<String>>empty(), user, Optional.empty()));
    assertEquals(2, taskManager.getAllQueuedTaskShellCommandRequests().size());
    dispatchPoller.runActionOnPoll();
    assertEquals(0, taskManager.getAllQueuedTaskShellCommandRequests().size());
    assertEquals(2, taskManager.getTaskShellCommandRequestsForTask(task.getTaskId()).size());
    mesosScheduler.message(Event.Message.newBuilder().setExecutorId(MesosProtosUtils.toExecutorId(task.getMesosTask().getExecutor().getExecutorId())).setAgentId(MesosProtosUtils.toAgentId(task.getMesosTask().getAgentId())).setData(ByteString.copyFrom(updateTranscoder.toBytes(new SingularityTaskShellCommandUpdate(firstShellRequest.getId(), System.currentTimeMillis(), Optional.<String>of("hi"), Optional.<String>empty(), UpdateType.STARTED)))).build());
    mesosScheduler.message(Event.Message.newBuilder().setExecutorId(MesosProtosUtils.toExecutorId(task.getMesosTask().getExecutor().getExecutorId())).setAgentId(MesosProtosUtils.toAgentId(task.getMesosTask().getAgentId())).setData(ByteString.copyFrom(updateTranscoder.toBytes(new SingularityTaskShellCommandUpdate(new SingularityTaskShellCommandRequestId(task.getTaskId(), "wat", System.currentTimeMillis()), System.currentTimeMillis(), Optional.<String>of("hi"), Optional.<String>empty(), UpdateType.STARTED)))).build());
    mesosScheduler.message(Event.Message.newBuilder().setExecutorId(MesosProtosUtils.toExecutorId(task.getMesosTask().getExecutor().getExecutorId())).setAgentId(MesosProtosUtils.toAgentId(task.getMesosTask().getAgentId())).setData(ByteString.copyFrom(updateTranscoder.toBytes(new SingularityTaskShellCommandUpdate(new SingularityTaskShellCommandRequestId(new SingularityTaskId("makingitup", "did", System.currentTimeMillis(), 1, "host", "rack"), "wat", System.currentTimeMillis()), System.currentTimeMillis(), Optional.<String>of("hi"), Optional.<String>empty(), UpdateType.STARTED)))).build());
    assertEquals(true, taskManager.getTaskHistory(task.getTaskId()).get().getShellCommandHistory().get(1).getShellUpdates().get(0).getUpdateType() == UpdateType.STARTED);
    assertEquals(1, taskManager.getTaskShellCommandUpdates(firstShellRequest.getId()).size());
    assertEquals(0, taskManager.getTaskShellCommandUpdates(secondShellRequest.getId()).size());
}

Example 2 with SingularityTaskShellCommandUpdate

use of com.hubspot.singularity.SingularityTaskShellCommandUpdate in project Singularity by HubSpot.

the class SingularityTaskShellCommandTest method testRunCommandBeforeBounceKill.

@Test
public void testRunCommandBeforeBounceKill() {
    setShellCommandsConfiguration();
    initRequest();
    initFirstDeploy();
    launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING);
    requestResource.bounce(requestId, Optional.of(new SingularityBounceRequest(Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.of(new SingularityShellCommand("d1", Optional.of(Arrays.asList("o1", "o2")), user, Optional.empty())))), singularityUser);
    cleaner.drainCleanupQueue();
    List<SingularityTaskCleanup> taskCleanups = taskManager.getCleanupTasks();
    Assertions.assertTrue(taskCleanups.get(0).getRunBeforeKillId().isPresent());
    SingularityTaskShellCommandRequestId shellCommandRequestId = taskCleanups.get(0).getRunBeforeKillId().get();
    cleaner.drainCleanupQueue();
    Assertions.assertEquals(1, taskManager.getCleanupTaskIds().size());
    launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING);
    Assertions.assertEquals(2, taskManager.getActiveTaskIdsForRequest(requestId).size());
    cleaner.drainCleanupQueue();
    Assertions.assertEquals(2, taskManager.getActiveTaskIdsForRequest(requestId).size());
    taskManager.saveTaskShellCommandUpdate(new SingularityTaskShellCommandUpdate(shellCommandRequestId, System.currentTimeMillis(), Optional.<String>empty(), Optional.<String>empty(), UpdateType.ACKED));
    cleaner.drainCleanupQueue();
    Assertions.assertEquals(2, taskManager.getActiveTaskIdsForRequest(requestId).size());
    taskManager.saveTaskShellCommandUpdate(new SingularityTaskShellCommandUpdate(shellCommandRequestId, System.currentTimeMillis(), Optional.<String>empty(), Optional.<String>empty(), UpdateType.FINISHED));
    cleaner.drainCleanupQueue();
    Assertions.assertEquals(1, taskManager.getKilledTaskIdRecords().size());
}

Example 3 with SingularityTaskShellCommandUpdate

use of com.hubspot.singularity.SingularityTaskShellCommandUpdate in project Singularity by HubSpot.

the class SingularityExecutorShellCommandUpdater method sendUpdate.

// TODO thread?
public void sendUpdate(UpdateType updateType, Optional<String> message, Optional<String> outputFilename) {
    SingularityTaskShellCommandUpdate update = new SingularityTaskShellCommandUpdate(shellRequest.getId(), System.currentTimeMillis(), message, outputFilename, updateType);
    try {
        byte[] data = objectMapper.writeValueAsBytes(update);
        task.getLog().info("Sending update {} ({}) for shell command {}", updateType, message.orElse(""), shellRequest.getId());
        task.getDriver().sendFrameworkMessage(data);
    } catch (JsonProcessingException e) {
        task.getLog().error("Unable to serialize update {}", update, e);
    }
}

Example 4 with SingularityTaskShellCommandUpdate

use of com.hubspot.singularity.SingularityTaskShellCommandUpdate in project Singularity by HubSpot.

the class SingularityCleaner method shouldKillTask.

private boolean shouldKillTask(SingularityTaskCleanup taskCleanup, List<SingularityTaskId> activeTaskIds, Set<SingularityTaskId> cleaningTasks, Multiset<SingularityDeployKey> incrementalCleaningTasks) {
    final Optional<SingularityRequestWithState> requestWithState = requestManager.getRequest(taskCleanup.getTaskId().getRequestId());
    if (!requestWithState.isPresent()) {
        LOG.debug("Killing a task {} immediately because the request was missing", taskCleanup);
        return true;
    }
    final SingularityRequest request = requestWithState.get().getRequest();
    if (taskCleanup.getRunBeforeKillId().isPresent()) {
        List<SingularityTaskShellCommandUpdate> shellCommandUpdates = taskManager.getTaskShellCommandUpdates(taskCleanup.getRunBeforeKillId().get());
        boolean finished = false;
        for (SingularityTaskShellCommandUpdate update : shellCommandUpdates) {
            if (update.getUpdateType().isFinished()) {
                finished = true;
                break;
            }
        }
        if (!finished) {
            LOG.debug("Waiting for pre-kill shell command {} to finish before killing task", taskCleanup.getRunBeforeKillId());
            return false;
        }
    }
    if (taskCleanup.getCleanupType().shouldKillTaskInstantly(request)) {
        LOG.debug("Killing a task {} immediately because of its cleanup type", taskCleanup);
        return true;
    }
    // If pausing, must be a long-running task to kill here
    if (requestWithState.get().getState() == RequestState.PAUSED && (!(taskCleanup.getCleanupType() == TaskCleanupType.PAUSING) || request.isLongRunning())) {
        LOG.debug("Killing a task {} immediately because the request was paused", taskCleanup);
        return true;
    }
    if (!request.isLongRunning()) {
        final long timeSinceCleanup = System.currentTimeMillis() - taskCleanup.getTimestamp();
        final long maxWaitTime = request.getKillOldNonLongRunningTasksAfterMillis().orElse(killNonLongRunningTasksInCleanupAfterMillis);
        final boolean tooOld = (maxWaitTime < 1) || (timeSinceCleanup > maxWaitTime);
        if (!tooOld) {
            LOG.trace("Not killing a non-longRunning task {}, running time since cleanup {} (max wait time is {})", taskCleanup, timeSinceCleanup, maxWaitTime);
        } else {
            LOG.debug("Killing a non-longRunning task {} - running time since cleanup {} exceeded max wait time {}", taskCleanup, timeSinceCleanup, maxWaitTime);
        }
        return tooOld;
    }
    final String requestId = request.getId();
    final Optional<SingularityRequestDeployState> deployState = deployManager.getRequestDeployState(requestId);
    if (taskCleanup.getCleanupType() == TaskCleanupType.DECOMISSIONING && deployState.get().getPendingDeploy().isPresent() && deployState.get().getPendingDeploy().get().getDeployId().equals(taskCleanup.getTaskId().getDeployId())) {
        final long timeSinceCleanup = System.currentTimeMillis() - taskCleanup.getTimestamp();
        final long maxWaitTime = configuration.getPendingDeployHoldTaskDuringDecommissionMillis();
        final boolean tooOld = (maxWaitTime < 1) || (timeSinceCleanup > maxWaitTime);
        if (!tooOld) {
            LOG.trace("Not killing {} - part of pending deploy - running time since cleanup {} (max wait time is {})", taskCleanup, timeSinceCleanup, maxWaitTime);
            return false;
        } else {
            LOG.debug("Killing {} - part of pending deploy but running time since cleanup {} exceeded max wait time {}", taskCleanup, timeSinceCleanup, maxWaitTime);
            return true;
        }
    }
    if (!deployState.isPresent() || !deployState.get().getActiveDeploy().isPresent()) {
        LOG.debug("Killing a task {} immediately because there is no active deploy state {}", taskCleanup, deployState);
        return true;
    }
    final String activeDeployId = deployState.get().getActiveDeploy().get().getDeployId();
    final String matchingTasksDeployId = taskCleanup.getCleanupType() == TaskCleanupType.INCREMENTAL_DEPLOY_CANCELLED || taskCleanup.getCleanupType() == TaskCleanupType.INCREMENTAL_DEPLOY_FAILED ? activeDeployId : taskCleanup.getTaskId().getDeployId();
    // check to see if there are enough active tasks out there that have been active for long enough that we can safely shut this task down.
    final List<SingularityTaskId> matchingTasks = new ArrayList<>();
    for (SingularityTaskId taskId : activeTaskIds) {
        if (!taskId.getRequestId().equals(requestId) || !taskId.getDeployId().equals(matchingTasksDeployId)) {
            continue;
        }
        if (cleaningTasks.contains(taskId)) {
            continue;
        }
        matchingTasks.add(taskId);
    }
    // For an incremental bounce or incremental deploy cleanup, shut down old tasks as new ones are started
    final SingularityDeployKey key = SingularityDeployKey.fromTaskId(taskCleanup.getTaskId());
    if (taskCleanup.getCleanupType() == TaskCleanupType.INCREMENTAL_BOUNCE) {
        return shouldKillIncrementalBounceTask(request, taskCleanup, matchingTasksDeployId, matchingTasks, key, incrementalCleaningTasks);
    } else if (isIncrementalDeployCleanup(taskCleanup)) {
        return shouldKillIncrementalDeployCleanupTask(request, taskCleanup, matchingTasksDeployId, matchingTasks, key, incrementalCleaningTasks);
    } else {
        if (matchingTasks.size() < request.getInstancesSafe()) {
            LOG.trace("Not killing a task {} yet, only {} matching out of a required {}", taskCleanup, matchingTasks.size(), request.getInstancesSafe());
            return false;
        }
    }
    final Optional<SingularityDeploy> deploy = deployManager.getDeploy(requestId, activeDeployId);
    final DeployHealth deployHealth = deploy.isPresent() ? deployHealthHelper.getDeployHealth(requestWithState.get().getRequest(), deploy.get(), matchingTasks, false) : DeployHealth.UNHEALTHY;
    switch(deployHealth) {
        case HEALTHY:
            for (SingularityTaskId taskId : matchingTasks) {
                DeployHealth lbHealth = getLbHealth(request, taskId);
                if (lbHealth != DeployHealth.HEALTHY) {
                    LOG.trace("Not killing a task {}, waiting for new replacement tasks to be added to LB (current state: {})", taskCleanup, lbHealth);
                    return false;
                }
            }
            LOG.debug("Killing a task {}, at least {} replacement tasks are healthy [{}]", taskCleanup, request.getInstancesSafe(), matchingTasks);
            return true;
        case WAITING:
        case UNHEALTHY:
        default:
            LOG.trace("Not killing a task {}, waiting for new replacement tasks to be healthy (current state: {})", taskCleanup, deployHealth);
            return false;
    }
}

Example 5 with SingularityTaskShellCommandUpdate

use of com.hubspot.singularity.SingularityTaskShellCommandUpdate in project Singularity by HubSpot.

the class SingularityMesosFrameworkMessageHandler method handleMessage.

public void handleMessage(Protos.ExecutorID executorId, Protos.AgentID agentId, byte[] data) {
    SingularityTaskShellCommandUpdate shellUpdate = null;
    try {
        shellUpdate = commandUpdateTranscoder.fromBytes(data);
        SingularityCreateResult saved = taskManager.saveTaskShellCommandUpdate(shellUpdate);
        LOG.debug("Saved {} with result {}", shellUpdate, saved);
    } catch (SingularityTranscoderException ste) {
        LOG.warn("Framework message {} not a commandUpdate", new String(data, UTF_8));
    } catch (Exception e) {
        LOG.error("While processing framework message {}", shellUpdate, e);
    }
}