use of com.netflix.titus.common.runtime.TitusRuntime in project titus-control-plane by Netflix.
the class StreamDataReplicatorPerf method main.
public static void main(String[] args) throws InterruptedException {
TitusRuntime titusRuntime = TitusRuntimes.internal();
JobManagementClient client = Mockito.mock(JobManagementClient.class);
JobConnectorConfiguration configuration = Mockito.mock(JobConnectorConfiguration.class);
Mockito.when(client.observeJobs(ArgumentMatchers.any())).thenAnswer(invocation -> Flux.defer(() -> {
JobManagerEvent jobUpdateEvent = JobUpdateEvent.newJob(JOB, JobManagerConstants.GRPC_REPLICATOR_CALL_METADATA);
JobManagerEvent taskUpdateEvent = TaskUpdateEvent.newTask(JOB, TASK, JobManagerConstants.GRPC_REPLICATOR_CALL_METADATA);
return Flux.just(jobUpdateEvent, JobManagerEvent.snapshotMarker()).concatWith(Flux.interval(Duration.ofSeconds(1)).take(1).map(tick -> taskUpdateEvent)).concatWith(Flux.interval(Duration.ofSeconds(1)).take(1).flatMap(tick -> Flux.error(new RuntimeException("Simulated error"))));
}));
JobDataReplicator replicator = new JobDataReplicatorProvider(configuration, client, JobSnapshotFactories.newDefault(titusRuntime), titusRuntime).get();
replicator.events().subscribe(System.out::println);
Thread.sleep(3600_000);
}
use of com.netflix.titus.common.runtime.TitusRuntime in project titus-control-plane by Netflix.
the class BasicTaskActions method writeReferenceTaskToStore.
/**
* Write updated task record to a store. If a task is completed, remove it from the scheduling service.
* This command calls {@link JobStore#updateTask(Task)}, which assumes that the task record was created already.
*/
public static TitusChangeAction writeReferenceTaskToStore(JobStore titusStore, ReconciliationEngine<JobManagerReconcilerEvent> engine, String taskId, CallMetadata callMetadata, TitusRuntime titusRuntime) {
return TitusChangeAction.newAction("writeReferenceTaskToStore").trigger(V3JobOperations.Trigger.Reconciler).id(taskId).summary("Persisting task to the store").callMetadata(callMetadata).changeWithModelUpdate(self -> {
Optional<EntityHolder> taskHolder = engine.getReferenceView().findById(taskId);
if (!taskHolder.isPresent()) {
// Should never happen
titusRuntime.getCodeInvariants().inconsistent("Reference task with id %s not found.", taskId);
return Observable.empty();
}
Task referenceTask = taskHolder.get().getEntity();
return titusStore.updateTask(referenceTask).andThen(Observable.fromCallable(() -> {
TitusModelAction modelUpdateAction = TitusModelAction.newModelUpdate(self).taskUpdate(storeRoot -> {
EntityHolder storedHolder = EntityHolder.newRoot(referenceTask.getId(), referenceTask);
return Pair.of(storeRoot.addChild(storedHolder), storedHolder);
});
return ModelActionHolder.store(modelUpdateAction);
}));
});
}
use of com.netflix.titus.common.runtime.TitusRuntime in project titus-control-plane by Netflix.
the class KillInitiatedActions method reconcilerInitiatedAllTasksKillInitiated.
/**
* For all active tasks, send terminate command to the compute provider, and change their state to {@link TaskState#KillInitiated}.
* This method is used for internal state reconciliation.
*/
public static List<ChangeAction> reconcilerInitiatedAllTasksKillInitiated(ReconciliationEngine<JobManagerReconcilerEvent> engine, JobServiceRuntime runtime, JobStore jobStore, String reasonCode, String reason, int concurrencyLimit, VersionSupplier versionSupplier, TitusRuntime titusRuntime) {
List<ChangeAction> result = new ArrayList<>();
EntityHolder runningView = engine.getRunningView();
Set<String> runningTaskIds = new HashSet<>();
runningView.getChildren().forEach(taskHolder -> runningTaskIds.add(taskHolder.<Task>getEntity().getId()));
// Immediately finish Accepted tasks, which are not yet in the running model.
for (EntityHolder entityHolder : engine.getReferenceView().getChildren()) {
if (result.size() >= concurrencyLimit) {
return result;
}
Task task = entityHolder.getEntity();
TaskState state = task.getStatus().getState();
if (state == TaskState.Accepted && !runningTaskIds.contains(task.getId())) {
result.add(BasicTaskActions.updateTaskAndWriteItToStore(task.getId(), engine, taskRef -> JobFunctions.changeTaskStatus(taskRef, TaskState.Finished, reasonCode, reason, titusRuntime.getClock()), jobStore, V3JobOperations.Trigger.Reconciler, reason, versionSupplier, titusRuntime, JobManagerConstants.RECONCILER_CALLMETADATA.toBuilder().withCallReason(reason).build()));
}
}
// Move running tasks to KillInitiated state
for (EntityHolder taskHolder : runningView.getChildren()) {
if (result.size() >= concurrencyLimit) {
return result;
}
Task task = taskHolder.getEntity();
TaskState state = task.getStatus().getState();
if (state != TaskState.KillInitiated && state != TaskState.Finished) {
result.add(reconcilerInitiatedTaskKillInitiated(engine, task, runtime, jobStore, versionSupplier, reasonCode, reason, titusRuntime));
}
}
return result;
}
use of com.netflix.titus.common.runtime.TitusRuntime in project titus-control-plane by Netflix.
the class KillInitiatedActions method userInitiateTaskKillAction.
/**
* Change a task to {@link TaskState#KillInitiated} state, store it, and send the kill command to the compute provider.
* All models are updated when both operations complete.
* This method is used for user initiated kill operations, so the store operation happens before response is sent back to the user.
*/
public static ChangeAction userInitiateTaskKillAction(ReconciliationEngine<JobManagerReconcilerEvent> engine, JobServiceRuntime executionContext, JobStore jobStore, VersionSupplier versionSupplier, String taskId, boolean shrink, boolean preventMinSizeUpdate, String reasonCode, String reason, TitusRuntime titusRuntime, CallMetadata callMetadata) {
return TitusChangeAction.newAction("userInitiateTaskKill").id(taskId).trigger(V3JobOperations.Trigger.API).summary(reason).callMetadata(callMetadata).changeWithModelUpdates(self -> JobEntityHolders.toTaskObservable(engine, taskId, titusRuntime).flatMap(task -> {
TaskState taskState = task.getStatus().getState();
if (taskState == TaskState.KillInitiated || taskState == TaskState.Finished) {
return Observable.just(Collections.<ModelActionHolder>emptyList());
}
if (shrink) {
Job<ServiceJobExt> job = engine.getReferenceView().getEntity();
Capacity capacity = job.getJobDescriptor().getExtensions().getCapacity();
if (preventMinSizeUpdate && capacity.getDesired() <= capacity.getMin()) {
return Observable.<List<ModelActionHolder>>error(JobManagerException.terminateAndShrinkNotAllowed(job, task));
}
}
Task taskWithKillInitiated = VersionSuppliers.nextVersion(JobFunctions.changeTaskStatus(task, TaskState.KillInitiated, reasonCode, reason, titusRuntime.getClock()), versionSupplier);
Callable<List<ModelActionHolder>> modelUpdateActions = () -> JobEntityHolders.expectTask(engine, task.getId(), titusRuntime).map(current -> {
List<ModelActionHolder> updateActions = new ArrayList<>();
TitusModelAction stateUpdateAction = TitusModelAction.newModelUpdate(self).taskUpdate(taskWithKillInitiated);
updateActions.addAll(ModelActionHolder.allModels(stateUpdateAction));
if (shrink) {
TitusModelAction shrinkAction = createShrinkAction(self, versionSupplier);
updateActions.add(ModelActionHolder.reference(shrinkAction));
}
return updateActions;
}).orElse(Collections.emptyList());
return jobStore.updateTask(taskWithKillInitiated).andThen(createKillAction(executionContext, task)).andThen(Observable.fromCallable(modelUpdateActions));
}));
}
use of com.netflix.titus.common.runtime.TitusRuntime in project titus-control-plane by Netflix.
the class DifferenceResolverUtils method findTaskStateTimeouts.
/**
* Find all tasks that are stuck in a specific state. The number of {@link ChangeAction changes} will be limited
* by the {@link TokenBucket stuckInStateRateLimiter}
*/
public static List<ChangeAction> findTaskStateTimeouts(ReconciliationEngine<JobManagerReconcilerEvent> engine, JobView runningJobView, JobManagerConfiguration configuration, JobServiceRuntime runtime, JobStore jobStore, VersionSupplier versionSupplier, TokenBucket stuckInStateRateLimiter, TitusRuntime titusRuntime) {
Clock clock = titusRuntime.getClock();
List<ChangeAction> actions = new ArrayList<>();
runningJobView.getJobHolder().getChildren().forEach(taskHolder -> {
Task task = taskHolder.getEntity();
TaskState taskState = task.getStatus().getState();
if (JobFunctions.isBatchJob(runningJobView.getJob()) && taskState == TaskState.Started) {
Job<BatchJobExt> batchJob = runningJobView.getJob();
// We expect runtime limit to be always set, so this is just extra safety measure.
long runtimeLimitMs = Math.max(BatchJobExt.RUNTIME_LIMIT_MIN, batchJob.getJobDescriptor().getExtensions().getRuntimeLimitMs());
long deadline = task.getStatus().getTimestamp() + runtimeLimitMs;
if (deadline < clock.wallTime()) {
actions.add(KillInitiatedActions.reconcilerInitiatedTaskKillInitiated(engine, task, runtime, jobStore, versionSupplier, TaskStatus.REASON_RUNTIME_LIMIT_EXCEEDED, "Task running too long (runtimeLimit=" + runtimeLimitMs + "ms)", titusRuntime));
}
return;
}
TaskTimeoutChangeActions.TimeoutStatus timeoutStatus = TaskTimeoutChangeActions.getTimeoutStatus(taskHolder, clock);
switch(timeoutStatus) {
case Ignore:
case Pending:
break;
case NotSet:
long timeoutMs = -1;
switch(taskState) {
case Launched:
timeoutMs = configuration.getTaskInLaunchedStateTimeoutMs();
break;
case StartInitiated:
timeoutMs = isBatch(runningJobView.getJob()) ? configuration.getBatchTaskInStartInitiatedStateTimeoutMs() : configuration.getServiceTaskInStartInitiatedStateTimeoutMs();
break;
case KillInitiated:
timeoutMs = configuration.getTaskInKillInitiatedStateTimeoutMs();
break;
}
if (timeoutMs > 0) {
actions.add(TaskTimeoutChangeActions.setTimeout(taskHolder.getId(), task.getStatus().getState(), timeoutMs, clock));
}
break;
case TimedOut:
if (!stuckInStateRateLimiter.tryTake()) {
break;
}
if (task.getStatus().getState() == TaskState.KillInitiated) {
int attempts = TaskTimeoutChangeActions.getKillInitiatedAttempts(taskHolder) + 1;
if (attempts >= configuration.getTaskKillAttempts()) {
actions.add(BasicTaskActions.updateTaskInRunningModel(task.getId(), V3JobOperations.Trigger.Reconciler, configuration, engine, taskParam -> Optional.of(taskParam.toBuilder().withStatus(taskParam.getStatus().toBuilder().withState(TaskState.Finished).withReasonCode(TaskStatus.REASON_STUCK_IN_KILLING_STATE).withReasonMessage("stuck in " + taskState + "state").build()).build()), "TimedOut in KillInitiated state", versionSupplier, titusRuntime, JobManagerConstants.RECONCILER_CALLMETADATA.toBuilder().withCallReason("Kill initiated").build()));
} else {
actions.add(TaskTimeoutChangeActions.incrementTaskKillAttempt(task.getId(), configuration.getTaskInKillInitiatedStateTimeoutMs(), clock));
actions.add(KillInitiatedActions.reconcilerInitiatedTaskKillInitiated(engine, task, runtime, jobStore, versionSupplier, TaskStatus.REASON_STUCK_IN_KILLING_STATE, "Another kill attempt (" + (attempts + 1) + ')', titusRuntime));
}
} else {
actions.add(KillInitiatedActions.reconcilerInitiatedTaskKillInitiated(engine, task, runtime, jobStore, versionSupplier, TaskStatus.REASON_STUCK_IN_STATE, "Task stuck in " + taskState + " state", titusRuntime));
}
break;
}
});
return actions;
}
Aggregations