Examples with TaskState com.netflix.titus.api.jobmanager.model.job.TaskState used on opensource projects

Example 11 with TaskState

use of com.netflix.titus.api.jobmanager.model.job.TaskState in project titus-control-plane by Netflix.

the class DifferenceResolverUtils method findTaskStateTimeouts.

/**
 * Find all tasks that are stuck in a specific state. The number of {@link ChangeAction changes} will be limited
 * by the {@link TokenBucket stuckInStateRateLimiter}
 */
public static List<ChangeAction> findTaskStateTimeouts(ReconciliationEngine<JobManagerReconcilerEvent> engine, JobView runningJobView, JobManagerConfiguration configuration, JobServiceRuntime runtime, JobStore jobStore, VersionSupplier versionSupplier, TokenBucket stuckInStateRateLimiter, TitusRuntime titusRuntime) {
    Clock clock = titusRuntime.getClock();
    List<ChangeAction> actions = new ArrayList<>();
    runningJobView.getJobHolder().getChildren().forEach(taskHolder -> {
        Task task = taskHolder.getEntity();
        TaskState taskState = task.getStatus().getState();
        if (JobFunctions.isBatchJob(runningJobView.getJob()) && taskState == TaskState.Started) {
            Job<BatchJobExt> batchJob = runningJobView.getJob();
            // We expect runtime limit to be always set, so this is just extra safety measure.
            long runtimeLimitMs = Math.max(BatchJobExt.RUNTIME_LIMIT_MIN, batchJob.getJobDescriptor().getExtensions().getRuntimeLimitMs());
            long deadline = task.getStatus().getTimestamp() + runtimeLimitMs;
            if (deadline < clock.wallTime()) {
                actions.add(KillInitiatedActions.reconcilerInitiatedTaskKillInitiated(engine, task, runtime, jobStore, versionSupplier, TaskStatus.REASON_RUNTIME_LIMIT_EXCEEDED, "Task running too long (runtimeLimit=" + runtimeLimitMs + "ms)", titusRuntime));
            }
            return;
        }
        TaskTimeoutChangeActions.TimeoutStatus timeoutStatus = TaskTimeoutChangeActions.getTimeoutStatus(taskHolder, clock);
        switch(timeoutStatus) {
            case Ignore:
            case Pending:
                break;
            case NotSet:
                long timeoutMs = -1;
                switch(taskState) {
                    case Launched:
                        timeoutMs = configuration.getTaskInLaunchedStateTimeoutMs();
                        break;
                    case StartInitiated:
                        timeoutMs = isBatch(runningJobView.getJob()) ? configuration.getBatchTaskInStartInitiatedStateTimeoutMs() : configuration.getServiceTaskInStartInitiatedStateTimeoutMs();
                        break;
                    case KillInitiated:
                        timeoutMs = configuration.getTaskInKillInitiatedStateTimeoutMs();
                        break;
                }
                if (timeoutMs > 0) {
                    actions.add(TaskTimeoutChangeActions.setTimeout(taskHolder.getId(), task.getStatus().getState(), timeoutMs, clock));
                }
                break;
            case TimedOut:
                if (!stuckInStateRateLimiter.tryTake()) {
                    break;
                }
                if (task.getStatus().getState() == TaskState.KillInitiated) {
                    int attempts = TaskTimeoutChangeActions.getKillInitiatedAttempts(taskHolder) + 1;
                    if (attempts >= configuration.getTaskKillAttempts()) {
                        actions.add(BasicTaskActions.updateTaskInRunningModel(task.getId(), V3JobOperations.Trigger.Reconciler, configuration, engine, taskParam -> Optional.of(taskParam.toBuilder().withStatus(taskParam.getStatus().toBuilder().withState(TaskState.Finished).withReasonCode(TaskStatus.REASON_STUCK_IN_KILLING_STATE).withReasonMessage("stuck in " + taskState + "state").build()).build()), "TimedOut in KillInitiated state", versionSupplier, titusRuntime, JobManagerConstants.RECONCILER_CALLMETADATA.toBuilder().withCallReason("Kill initiated").build()));
                    } else {
                        actions.add(TaskTimeoutChangeActions.incrementTaskKillAttempt(task.getId(), configuration.getTaskInKillInitiatedStateTimeoutMs(), clock));
                        actions.add(KillInitiatedActions.reconcilerInitiatedTaskKillInitiated(engine, task, runtime, jobStore, versionSupplier, TaskStatus.REASON_STUCK_IN_KILLING_STATE, "Another kill attempt (" + (attempts + 1) + ')', titusRuntime));
                    }
                } else {
                    actions.add(KillInitiatedActions.reconcilerInitiatedTaskKillInitiated(engine, task, runtime, jobStore, versionSupplier, TaskStatus.REASON_STUCK_IN_STATE, "Task stuck in " + taskState + " state", titusRuntime));
                }
                break;
        }
    });
    return actions;
}

Example 12 with TaskState

use of com.netflix.titus.api.jobmanager.model.job.TaskState in project titus-control-plane by Netflix.

the class ServiceDifferenceResolver method applyStore.

private List<ChangeAction> applyStore(ReconciliationEngine<JobManagerReconcilerEvent> engine, ServiceJobView refJobView, EntityHolder storeJob, AtomicInteger allowedNewTasks) {
    if (!storeWriteRetryInterceptor.executionLimits(storeJob)) {
        return Collections.emptyList();
    }
    List<ChangeAction> actions = new ArrayList<>();
    EntityHolder refJobHolder = refJobView.getJobHolder();
    Job<ServiceJobExt> refJob = refJobHolder.getEntity();
    if (!refJobHolder.getEntity().equals(storeJob.getEntity())) {
        actions.add(storeWriteRetryInterceptor.apply(BasicJobActions.updateJobInStore(engine, jobStore)));
    }
    boolean isJobTerminating = refJob.getStatus().getState() == JobState.KillInitiated;
    for (EntityHolder referenceTaskHolder : refJobHolder.getChildren()) {
        ServiceJobTask refTask = referenceTaskHolder.getEntity();
        Optional<EntityHolder> storeHolder = storeJob.findById(referenceTaskHolder.getId());
        ServiceJobTask storeTask = storeHolder.get().getEntity();
        boolean refAndStoreInSync = areEquivalent(storeHolder.get(), referenceTaskHolder);
        boolean shouldRetry = !isJobTerminating && refTask.getStatus().getState() == TaskState.Finished && !refTask.getStatus().getReasonCode().equals(TaskStatus.REASON_SCALED_DOWN) && allowedNewTasks.get() > 0;
        if (refAndStoreInSync) {
            TaskState currentTaskState = refTask.getStatus().getState();
            if (currentTaskState == TaskState.Finished) {
                if (isJobTerminating || isScaledDown(storeTask) || hasEnoughTasksRunning(refJobView)) {
                    actions.add(removeFinishedServiceTaskAction(jobStore, storeTask));
                } else if (shouldRetry && TaskRetryers.shouldRetryNow(referenceTaskHolder, clock)) {
                    createNewTaskAction(refJobView, Optional.of(referenceTaskHolder), Collections.emptyList(), Collections.emptyList()).ifPresent(actions::add);
                }
            }
        } else {
            Task task = referenceTaskHolder.getEntity();
            CallMetadata callMetadata = RECONCILER_CALLMETADATA.toBuilder().withCallReason("Writing runtime state changes to store").build();
            actions.add(storeWriteRetryInterceptor.apply(BasicTaskActions.writeReferenceTaskToStore(jobStore, engine, task.getId(), callMetadata, titusRuntime)));
        }
        // Both current and delayed retries are counted
        if (shouldRetry) {
            allowedNewTasks.decrementAndGet();
        }
    }
    return actions;
}

Example 13 with TaskState

use of com.netflix.titus.api.jobmanager.model.job.TaskState in project titus-control-plane by Netflix.

the class TaskTerminationExecutor method checkTaskIsRunningOrThrowAnException.

private Pair<Job<?>, Task> checkTaskIsRunningOrThrowAnException(String taskId) {
    Optional<Pair<Job<?>, Task>> jobAndTask = jobOperations.findTaskById(taskId);
    if (!jobAndTask.isPresent()) {
        throw EvictionException.taskNotFound(taskId);
    }
    Task task = jobAndTask.get().getRight();
    TaskState state = task.getStatus().getState();
    if (state == TaskState.Accepted) {
        throw EvictionException.taskNotScheduledYet(task);
    }
    if (!TaskState.isBefore(state, TaskState.KillInitiated)) {
        throw EvictionException.taskAlreadyStopped(task);
    }
    return jobAndTask.get();
}

Example 14 with TaskState

use of com.netflix.titus.api.jobmanager.model.job.TaskState in project titus-control-plane by Netflix.

the class JobScenarioBuilder method expectJobToScaleDown.

public JobScenarioBuilder expectJobToScaleDown() {
    JobDescriptor.JobDescriptorExt ext = getJob().getJobDescriptor().getExtensions();
    Preconditions.checkState(ext instanceof ServiceJobExt, "Not a service job %s", jobId);
    int size = ((ServiceJobExt) ext).getCapacity().getDesired();
    logger.info("[{}] Expect job {} to scale down to the desired size {}...", discoverActiveTest(), jobId, size);
    Stopwatch stopWatch = Stopwatch.createStarted();
    await().timeout(TIMEOUT_MS, TimeUnit.MILLISECONDS).until(() -> {
        List<TaskHolder> lastTaskHolders = getLastTaskHolders();
        return lastTaskHolders.stream().filter(t -> {
            TaskState state = t.getTaskScenarioBuilder().getTask().getStatus().getState();
            return state != TaskState.KillInitiated;
        }).count() <= size;
    });
    logger.info("[{}] Expected job {} scale down to the desired size {} completed in {}ms", discoverActiveTest(), jobId, size, stopWatch.elapsed(TimeUnit.MILLISECONDS));
    return this;
}

Example 15 with TaskState

use of com.netflix.titus.api.jobmanager.model.job.TaskState in project titus-control-plane by Netflix.

the class TaskScenarioBuilder method expectStateAndReasonUpdateSkipOther.

public TaskScenarioBuilder expectStateAndReasonUpdateSkipOther(TaskStatus.TaskState expectedState, String reasonCode) {
    logger.info("[{}] Expecting task state and reason {}/{} (skipping other)...", discoverActiveTest(), expectedState, reasonCode);
    Stopwatch stopWatch = Stopwatch.createStarted();
    TaskState expectedCoreState = toCoreTaskState(expectedState);
    expectTaskUpdate(task -> task.getStatus().getState().ordinal() <= expectedCoreState.ordinal(), task -> {
        com.netflix.titus.api.jobmanager.model.job.TaskStatus taskStatus = task.getStatus();
        return taskStatus.getState() == expectedCoreState && Objects.equals(reasonCode, taskStatus.getReasonCode());
    }, "Expected state: " + expectedCoreState);
    logger.info("[{}] Expected task state {} received in {}[ms]", discoverActiveTest(), expectedState, stopWatch.elapsed(TimeUnit.MILLISECONDS));
    return this;
}