Examples with SingularityExecutorTask com.hubspot.singularity.executor.task.SingularityExecutorTask used on opensource projects

Example 1 with SingularityExecutorTask

use of com.hubspot.singularity.executor.task.SingularityExecutorTask in project Singularity by HubSpot.

the class SingularityExecutorMonitor method shutdown.

public void shutdown(ExecutorDriver driver) {
    if (!alreadyShutDown.compareAndSet(false, true)) {
        LOG.info("Already ran shut down process");
        return;
    }
    LOG.info("Shutdown requested with driver {}", driver);
    threadChecker.getExecutorService().shutdown();
    processBuilderPool.shutdown();
    runningProcessPool.shutdown();
    cgroupCfsWatcherService.shutdown();
    for (SingularityExecutorTask task : tasks.values()) {
        if (!task.wasKilled()) {
            task.getLog().info("Executor shutting down - requested task kill with state: {}", requestKill(task.getTaskId()));
        }
    }
    processKiller.getExecutorService().shutdown();
    for (Entry<String, ListeningExecutorService> taskIdToShellCommandPool : taskToShellCommandPool.entrySet()) {
        // in case
        LOG.warn("Shutting down abandoned pool for {}", taskIdToShellCommandPool.getKey());
        taskIdToShellCommandPool.getValue().shutdown();
    }
    cgroupCheckers.values().forEach(SingularityExecutorCgroupCfsChecker::close);
    exitChecker.shutdown();
    final long start = System.currentTimeMillis();
    JavaUtils.awaitTerminationWithLatch(latch, "threadChecker", threadChecker.getExecutorService(), configuration.getShutdownTimeoutWaitMillis());
    JavaUtils.awaitTerminationWithLatch(latch, "processBuilder", processBuilderPool, configuration.getShutdownTimeoutWaitMillis());
    JavaUtils.awaitTerminationWithLatch(latch, "runningProcess", runningProcessPool, configuration.getShutdownTimeoutWaitMillis());
    JavaUtils.awaitTerminationWithLatch(latch, "processKiller", processKiller.getExecutorService(), configuration.getShutdownTimeoutWaitMillis());
    LOG.info("Awaiting shutdown of all thread pools for a max of {}", JavaUtils.durationFromMillis(configuration.getShutdownTimeoutWaitMillis()));
    try {
        latch.await();
    } catch (InterruptedException e) {
        LOG.warn("While awaiting shutdown of executor services", e);
    }
    LOG.info("Waited {} for shutdown of thread pools, now waiting {} before exiting...", JavaUtils.duration(start), JavaUtils.durationFromMillis(configuration.getStopDriverAfterMillis()));
    try {
        Thread.sleep(configuration.getStopDriverAfterMillis());
    } catch (Throwable t) {
        LOG.warn("While waiting to exit", t);
    }
    LOG.info("Stopping driver {}", driver);
    Status status = driver.stop();
    LOG.info("Driver stopped with status {}", status);
}

Example 2 with SingularityExecutorTask

use of com.hubspot.singularity.executor.task.SingularityExecutorTask in project Singularity by HubSpot.

the class SingularityExecutorMonitor method requestKill.

public KillState requestKill(String taskId, Optional<String> user, boolean destroy) {
    final Optional<SingularityExecutorTask> maybeTask = Optional.ofNullable(tasks.get(taskId));
    if (!maybeTask.isPresent()) {
        return KillState.DIDNT_EXIST;
    }
    final SingularityExecutorTask task = maybeTask.get();
    if (!destroy && task.wasForceDestroyed()) {
        task.getLog().debug("Already force destroyed, will not issue additional kill");
        return KillState.DESTROYING_PROCESS;
    }
    task.getLog().info("Executor asked to kill {}", taskId);
    ListenableFuture<ProcessBuilder> processBuilderFuture = null;
    SingularityExecutorTaskProcessCallable runningProcess = null;
    task.getLock().lock();
    boolean wasKilled = task.wasKilled();
    try {
        if (!wasKilled) {
            task.markKilled(user);
        }
        processBuilderFuture = processBuildingTasks.get(task.getTaskId());
        runningProcess = processRunningTasks.get(task.getTaskId());
    } finally {
        task.getLock().unlock();
    }
    if (processBuilderFuture != null) {
        task.getLog().info("Canceling process builder future for {}", taskId);
        CancelThread cancelThread = new CancelThread(processBuilderFuture, task);
        cancelThread.start();
        return KillState.INTERRUPTING_PRE_PROCESS;
    }
    if (runningProcess != null) {
        if (destroy) {
            if (user.isPresent()) {
                task.getLog().info("Destroying process with pid {} for task {} by request from user {}", runningProcess.getCurrentPid(), taskId, user.get());
            } else {
                task.getLog().info("Destroying process with pid {} for task {}", runningProcess.getCurrentPid(), taskId);
            }
            task.markForceDestroyed(user);
            runningProcess.signalKillToProcessIfActive();
            return KillState.DESTROYING_PROCESS;
        }
        if (processKiller.isKillInProgress(taskId)) {
            task.getLog().info("Kill already in progress for task {}", taskId);
            return KillState.KILLING_PROCESS;
        }
        if (user.isPresent()) {
            task.getLog().info("Killing process for task {} by request from {}", taskId, user.get());
        } else {
            task.getLog().info("Killing process for task {}", taskId);
        }
        processKiller.submitKillRequest(runningProcess);
        return KillState.KILLING_PROCESS;
    }
    return KillState.INCONSISTENT_STATE;
}

Example 3 with SingularityExecutorTask

use of com.hubspot.singularity.executor.task.SingularityExecutorTask in project Singularity by HubSpot.

the class SingularityExecutor method launchTask.

/**
 * Invoked when a task has been launched on this executor (initiated
 * via Scheduler::launchTasks). Note that this task can be realized
 * with a thread, a process, or some simple computation, however, no
 * other callbacks will be invoked on this executor until this
 * callback has returned.
 */
@Override
public void launchTask(final ExecutorDriver executorDriver, final Protos.TaskInfo taskInfo) {
    final String taskId = taskInfo.getTaskId().getValue();
    LOG.info("Asked to launch task {}", taskId);
    try {
        final ch.qos.logback.classic.Logger taskLog = taskBuilder.buildTaskLogger(taskId, taskInfo.getExecutor().getExecutorId().getValue());
        final SingularityExecutorTask task = taskBuilder.buildTask(taskId, executorDriver, taskInfo, taskLog);
        SubmitState submitState = monitor.submit(task);
        switch(submitState) {
            case REJECTED:
                LOG.warn("Can't launch task {}, it was rejected (probably due to shutdown)", taskInfo);
                break;
            case TASK_ALREADY_EXISTED:
                LOG.error("Can't launch task {}, already had a task with that ID", taskInfo);
                break;
            case SUBMITTED:
                task.getLog().info("Launched task {} with data {}", taskId, task.getExecutorData());
                break;
        }
    } catch (Throwable t) {
        LOG.error("Unexpected exception starting task {}", taskId, t);
        TaskState state = t instanceof ArtifactVerificationException ? TaskState.TASK_FAILED : TaskState.TASK_LOST;
        executorUtils.sendStatusUpdate(executorDriver, taskInfo.getTaskId(), state, String.format("Unexpected exception while launching task %s - %s", taskId, t.getMessage()), LOG);
    }
}

Example 4 with SingularityExecutorTask

use of com.hubspot.singularity.executor.task.SingularityExecutorTask in project Singularity by HubSpot.

the class SingularityExecutorMesosFrameworkMessageHandler method handleShellRequest.

private void handleShellRequest(SingularityTaskShellCommandRequest shellRequest) {
    Optional<SingularityExecutorTask> matchingTask = monitor.getTask(shellRequest.getTaskId().getId());
    if (!matchingTask.isPresent()) {
        LOG.warn("Missing task for {}, ignoring shell request", shellRequest.getTaskId());
        return;
    }
    matchingTask.get().getLog().info("Received shell request {}", shellRequest);
    SingularityExecutorShellCommandUpdater updater = new SingularityExecutorShellCommandUpdater(objectMapper, shellRequest, matchingTask.get());
    Optional<SingularityExecutorTaskProcessCallable> taskProcess = monitor.getTaskProcess(shellRequest.getTaskId().getId());
    if (!taskProcess.isPresent()) {
        updater.sendUpdate(UpdateType.INVALID, Optional.of("No task process found"), Optional.<String>empty());
        return;
    }
    SingularityExecutorShellCommandRunner shellRunner = new SingularityExecutorShellCommandRunner(shellRequest, executorConfiguration, matchingTask.get(), taskProcess.get(), monitor.getShellCommandExecutorServiceForTask(shellRequest.getTaskId().getId()), updater);
    shellRunner.start();
}

Example 5 with SingularityExecutorTask

use of com.hubspot.singularity.executor.task.SingularityExecutorTask in project Singularity by HubSpot.

the class SingularityExecutorTaskBuilder method buildTask.

public SingularityExecutorTask buildTask(String taskId, ExecutorDriver driver, TaskInfo taskInfo, Logger log) {
    SingularityTaskExecutorData taskExecutorData = readExecutorData(jsonObjectMapper, taskInfo);
    if (executorConfiguration.isVerifyAssignedPorts()) {
        checkAssignedPorts(taskInfo);
    }
    SingularityExecutorTaskDefinition taskDefinition = new SingularityExecutorTaskDefinition(taskId, taskExecutorData, MesosUtils.getTaskDirectoryPath(taskId).toString(), executorPid, taskExecutorData.getServiceLog(), Files.getFileExtension(taskExecutorData.getServiceLog()), taskExecutorData.getServiceFinishedTailLog(), executorConfiguration.getTaskAppDirectory(), executorConfiguration.getExecutorBashLog(), executorConfiguration.getLogrotateStateFile(), executorConfiguration.getSignatureVerifyOut());
    jsonObjectFileHelper.writeObject(taskDefinition, executorConfiguration.getTaskDefinitionPath(taskId), log);
    return new SingularityExecutorTask(driver, executorUtils, baseConfiguration, executorConfiguration, taskDefinition, executorPid, artifactFetcher, taskInfo, templateManager, log, jsonObjectFileHelper, dockerUtils, jsonObjectMapper, s3Configuration);
}