Examples with SingularityTaskHistoryUpdate com.hubspot.singularity.SingularityTaskHistoryUpdate used on opensource projects

Example 11 with SingularityTaskHistoryUpdate

use of com.hubspot.singularity.SingularityTaskHistoryUpdate in project Singularity by HubSpot.

the class SingularitySchedulerTestBase method runLaunchedTasks.

protected void runLaunchedTasks() {
    for (SingularityTaskId taskId : taskManager.getActiveTaskIds()) {
        Collection<SingularityTaskHistoryUpdate> updates = taskManager.getTaskHistoryUpdates(taskId);
        SimplifiedTaskState currentState = SingularityTaskHistoryUpdate.getCurrentState(updates);
        switch(currentState) {
            case UNKNOWN:
            case WAITING:
                statusUpdate(taskManager.getTask(taskId).get(), TaskState.TASK_RUNNING);
                break;
            case DONE:
            case RUNNING:
                break;
        }
    }
}

Example 12 with SingularityTaskHistoryUpdate

use of com.hubspot.singularity.SingularityTaskHistoryUpdate in project Singularity by HubSpot.

the class SingularityMesosStatusUpdateHandler method unsafeProcessStatusUpdate.

private void unsafeProcessStatusUpdate(Protos.TaskStatus status, SingularityTaskId taskIdObj) {
    final String taskId = status.getTaskId().getValue();
    long timestamp = System.currentTimeMillis();
    if (status.hasTimestamp()) {
        timestamp = (long) (status.getTimestamp() * 1000);
    }
    long now = System.currentTimeMillis();
    long delta = now - timestamp;
    LOG.debug("Update: task {} is now {} ({}) at {} (delta: {})", taskId, status.getState(), status.getMessage(), timestamp, JavaUtils.durationFromMillis(delta));
    statusUpdateDeltas.put(now, delta);
    final SingularityTaskStatusHolder newTaskStatusHolder = new SingularityTaskStatusHolder(taskIdObj, Optional.of(mesosProtosUtils.taskStatusFromProtos(status)), System.currentTimeMillis(), serverId, Optional.<String>absent());
    final Optional<SingularityTaskStatusHolder> previousTaskStatusHolder = taskManager.getLastActiveTaskStatus(taskIdObj);
    final ExtendedTaskState taskState = MesosUtils.fromTaskState(status.getState());
    if (isDuplicateOrIgnorableStatusUpdate(previousTaskStatusHolder, newTaskStatusHolder)) {
        LOG.trace("Ignoring status update {} to {}", taskState, taskIdObj);
        saveNewTaskStatusHolder(taskIdObj, newTaskStatusHolder, taskState);
        return;
    }
    if (status.getState() == TaskState.TASK_LOST) {
        lostTasksMeter.mark();
        if (configuration.getDisasterDetection().isEnabled()) {
            taskLostReasons.add(status.getReason());
        }
    }
    final Optional<SingularityTask> task = taskManager.getTask(taskIdObj);
    final boolean isActiveTask = taskManager.isActiveTask(taskId);
    if (isActiveTask && !taskState.isDone()) {
        if (task.isPresent()) {
            final Optional<SingularityPendingDeploy> pendingDeploy = deployManager.getPendingDeploy(taskIdObj.getRequestId());
            Optional<SingularityRequestWithState> requestWithState = Optional.absent();
            if (taskState == ExtendedTaskState.TASK_RUNNING) {
                requestWithState = requestManager.getRequest(taskIdObj.getRequestId());
                healthchecker.enqueueHealthcheck(task.get(), pendingDeploy, requestWithState);
            }
            if (!pendingDeploy.isPresent() || !pendingDeploy.get().getDeployMarker().getDeployId().equals(taskIdObj.getDeployId())) {
                if (!requestWithState.isPresent()) {
                    requestWithState = requestManager.getRequest(taskIdObj.getRequestId());
                }
                newTaskChecker.enqueueNewTaskCheck(task.get(), requestWithState, healthchecker);
            }
        } else {
            final String message = String.format("Task %s is active but is missing task data", taskId);
            exceptionNotifier.notify(message);
            LOG.error(message);
        }
    }
    final Optional<String> statusMessage = getStatusMessage(status, task);
    final SingularityTaskHistoryUpdate taskUpdate = new SingularityTaskHistoryUpdate(taskIdObj, timestamp, taskState, statusMessage, status.hasReason() ? Optional.of(status.getReason().name()) : Optional.<String>absent());
    final SingularityCreateResult taskHistoryUpdateCreateResult = taskManager.saveTaskHistoryUpdate(taskUpdate);
    logSupport.checkDirectoryAndContainerId(taskIdObj);
    if (taskState.isDone()) {
        healthchecker.cancelHealthcheck(taskId);
        newTaskChecker.cancelNewTaskCheck(taskId);
        taskManager.deleteKilledRecord(taskIdObj);
        slaveAndRackManager.checkStateAfterFinishedTask(taskIdObj, status.getAgentId().getValue(), leaderCache);
        scheduler.handleCompletedTask(task, taskIdObj, isActiveTask, timestamp, taskState, taskHistoryUpdateCreateResult, status);
    }
    saveNewTaskStatusHolder(taskIdObj, newTaskStatusHolder, taskState);
}

Example 13 with SingularityTaskHistoryUpdate

use of com.hubspot.singularity.SingularityTaskHistoryUpdate in project Singularity by HubSpot.

the class SingularityDeployHealthHelper method getNoHealthcheckDeployHealth.

private DeployHealth getNoHealthcheckDeployHealth(final Optional<SingularityDeploy> deploy, final Collection<SingularityTaskId> matchingActiveTasks) {
    final Map<SingularityTaskId, List<SingularityTaskHistoryUpdate>> taskUpdates = taskManager.getTaskHistoryUpdates(matchingActiveTasks);
    for (SingularityTaskId taskId : matchingActiveTasks) {
        Collection<SingularityTaskHistoryUpdate> updates = taskUpdates.get(taskId);
        SimplifiedTaskState currentState = SingularityTaskHistoryUpdate.getCurrentState(updates);
        switch(currentState) {
            case UNKNOWN:
            case WAITING:
                return DeployHealth.WAITING;
            case DONE:
                LOG.warn("Unexpectedly found an active task ({}) in done state: {}}", taskId, updates);
                return DeployHealth.UNHEALTHY;
            case RUNNING:
                if (!isRunningTaskHealthy(deploy, updates, taskId)) {
                    return DeployHealth.WAITING;
                }
        }
    }
    return DeployHealth.HEALTHY;
}

Example 14 with SingularityTaskHistoryUpdate

use of com.hubspot.singularity.SingularityTaskHistoryUpdate in project Singularity by HubSpot.

the class SingularityMailPoller method shouldSendTaskFinishedMail.

private ShouldSendMailState shouldSendTaskFinishedMail(SingularityTaskId taskId, Optional<SingularityRequestWithState> requestWithState, Optional<SingularityTaskHistory> taskHistory) {
    if (!requestWithState.isPresent()) {
        LOG.warn("No request found for {}, can't send task finished email", taskId);
        return ShouldSendMailState.ERROR;
    }
    if (!taskHistory.isPresent()) {
        taskHistory = historyManager.getTaskHistory(taskId.getId());
    }
    if (!taskHistory.isPresent()) {
        LOG.warn("No task history found for {}, can't send task finished email", taskId);
        return ShouldSendMailState.ERROR;
    }
    if (taskMetadataConfiguration.getSendTaskCompletedMailOnceMetadataTypeIsAvailable().isPresent()) {
        for (SingularityTaskMetadata taskMetadata : taskHistory.get().getTaskMetadata()) {
            if (taskMetadata.getType().equals(taskMetadataConfiguration.getSendTaskCompletedMailOnceMetadataTypeIsAvailable().get())) {
                LOG.debug("Sending task finished email for {} because metadata type {} is present", taskId, taskMetadataConfiguration.getSendTaskCompletedMailOnceMetadataTypeIsAvailable().get());
                return ShouldSendMailState.SEND;
            }
        }
    }
    // check to see if it's too soon.
    if (taskMetadataConfiguration.getWaitToSendTaskCompletedMailBufferMillis() > 0) {
        Optional<SingularityTaskHistoryUpdate> lastUpdate = taskHistory.get().getLastTaskUpdate();
        if (!lastUpdate.isPresent()) {
            LOG.warn("Missing last update for {}, can't send task finished email", taskId);
            return ShouldSendMailState.ERROR;
        }
        final long timeSinceLastUpdate = System.currentTimeMillis() - lastUpdate.get().getTimestamp();
        if (timeSinceLastUpdate < taskMetadataConfiguration.getWaitToSendTaskCompletedMailBufferMillis()) {
            LOG.debug("Not sending task finished email for {} because last update was {} ago, buffer is {}", taskId, JavaUtils.durationFromMillis(timeSinceLastUpdate), JavaUtils.durationFromMillis(taskMetadataConfiguration.getWaitToSendTaskCompletedMailBufferMillis()));
            return ShouldSendMailState.WAIT;
        }
    }
    return ShouldSendMailState.SEND;
}

Example 15 with SingularityTaskHistoryUpdate

use of com.hubspot.singularity.SingularityTaskHistoryUpdate in project Singularity by HubSpot.

the class SingularityExecutorCleanup method cleanTask.

private TaskCleanupResult cleanTask(SingularityExecutorTaskDefinition taskDefinition, Optional<SingularityTaskHistory> taskHistory) {
    SingularityExecutorTaskLogManager logManager = new SingularityExecutorTaskLogManager(taskDefinition, templateManager, baseConfiguration, executorConfiguration, LOG, jsonObjectFileHelper);
    SingularityExecutorTaskCleanup taskCleanup = new SingularityExecutorTaskCleanup(logManager, executorConfiguration, taskDefinition, LOG, dockerUtils);
    boolean cleanupTaskAppDirectory = !taskDefinition.getExecutorData().getPreserveTaskSandboxAfterFinish().or(Boolean.FALSE);
    if (taskHistory.isPresent()) {
        final Optional<SingularityTaskHistoryUpdate> lastUpdate = JavaUtils.getLast(taskHistory.get().getTaskUpdates());
        if (lastUpdate.isPresent() && lastUpdate.get().getTaskState().isFailed()) {
            final long delta = System.currentTimeMillis() - lastUpdate.get().getTimestamp();
            if (delta < cleanupConfiguration.getCleanupAppDirectoryOfFailedTasksAfterMillis()) {
                LOG.info("Not cleaning up task app directory of {} because only {} has elapsed since it failed (will cleanup after {})", taskDefinition.getTaskId(), JavaUtils.durationFromMillis(delta), JavaUtils.durationFromMillis(cleanupConfiguration.getCleanupAppDirectoryOfFailedTasksAfterMillis()));
                cleanupTaskAppDirectory = false;
            }
        }
    }
    if (taskDefinition.shouldLogrotateLogFile()) {
        checkForUncompressedLogrotatedFile(taskDefinition);
    }
    boolean isDocker = (taskHistory.isPresent() && taskHistory.get().getTask().getTaskRequest().getDeploy().getContainerInfo().isPresent() && taskHistory.get().getTask().getTaskRequest().getDeploy().getContainerInfo().get().getType() == SingularityContainerType.DOCKER);
    return taskCleanup.cleanup(cleanupTaskAppDirectory, isDocker);
}